database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Fix](Json type) correct cast result for json type (#34764)

Browse Source
This commit is contained in:
lihangyu
2024-05-17 17:16:14 +08:00
committed by yiguolei
parent 81bcb9d490
commit e3e5f18f26
16 changed files with 307 additions and 216 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
be/src/util/jsonb_document.h
View File

@ -498,7 +498,7 @@ public:
bool isNull() const { return (type_ == JsonbType::T_Null); }
bool isTrue() const { return (type_ == JsonbType::T_True); }
bool isFalse() const { return (type_ == JsonbType::T_False); }
bool isInt() const { return isInt8() || isInt16() || isInt32() || isInt64(); }
bool isInt() const { return isInt8() || isInt16() || isInt32() || isInt64() || isInt128(); }
bool isInt8() const { return (type_ == JsonbType::T_Int8); }
bool isInt16() const { return (type_ == JsonbType::T_Int16); }
bool isInt32() const { return (type_ == JsonbType::T_Int32); }

79
be/src/vec/functions/function_cast.h
View File

@ -89,6 +89,7 @@
#include "vec/data_types/data_type_struct.h"
#include "vec/data_types/data_type_time.h"
#include "vec/data_types/data_type_time_v2.h"
#include "vec/data_types/serde/data_type_serde.h"
#include "vec/functions/function.h"
#include "vec/functions/function_convert_tz.h"
#include "vec/functions/function_helpers.h"
@ -762,29 +763,43 @@ struct ConvertImplGenericToJsonb {
auto column_string = ColumnString::create();
JsonbWriter writer;
ColumnUInt8::MutablePtr col_null_map_to = ColumnUInt8::create(col_from.size());
ColumnUInt8::Container* vec_null_map_to = &col_null_map_to->get_data();
DataTypeSerDe::FormatOptions format_options;
format_options.converted_from_string = true;
DataTypeSerDeSPtr from_serde = type.get_serde();
DataTypeSerDeSPtr to_serde = data_type_to->get_serde();
auto col_to = data_type_to->create_column();
auto tmp_col = ColumnString::create();
vectorized::DataTypeSerDe::FormatOptions options;
for (size_t i = 0; i < input_rows_count; i++) {
// convert to string
tmp_col->clear();
VectorBufferWriter write_buffer(*tmp_col.get());
type.to_string(col_from, i, write_buffer);
Status st =
from_serde->serialize_column_to_json(col_from, i, i + 1, write_buffer, options);
// if serialized failed, will return null
(*vec_null_map_to)[i] = !st.ok();
if (!st.ok()) {
col_to->insert_default();
continue;
}
write_buffer.commit();
writer.reset();
auto str_ref = tmp_col->get_data_at(0);
ReadBuffer read_buffer((char*)(str_ref.data), str_ref.size);
Slice data((char*)(str_ref.data), str_ref.size);
// first try to parse string
Status st = data_type_to->from_string(read_buffer, column_string.get());
st = to_serde->deserialize_one_cell_from_json(*col_to, data, format_options);
// if parsing failed, will return null
(*vec_null_map_to)[i] = !st.ok();
if (!st.ok()) {
// write raw string to jsonb
writer.writeStartString();
writer.writeString(str_ref.data, str_ref.size);
writer.writeEndString();
column_string->insert_data(writer.getOutput()->getBuffer(),
writer.getOutput()->getSize());
col_to->insert_default();
}
}
block.replace_by_position(result, std::move(column_string));
block.replace_by_position(
result, ColumnNullable::create(std::move(col_to), std::move(col_null_map_to)));
return Status::OK();
}
};
@ -845,39 +860,13 @@ struct ConvertImplFromJsonb {
null_map[i] = 1;
res[i] = 0;
}
} else if constexpr (type_index == TypeIndex::Int8) {
if (value->isInt8()) {
res[i] = (int8_t)((const JsonbIntVal*)value)->val();
} else {
null_map[i] = 1;
res[i] = 0;
}
} else if constexpr (type_index == TypeIndex::Int16) {
if (value->isInt8() || value->isInt16()) {
res[i] = (int16_t)((const JsonbIntVal*)value)->val();
} else {
null_map[i] = 1;
res[i] = 0;
}
} else if constexpr (type_index == TypeIndex::Int32) {
if (value->isInt8() || value->isInt16() || value->isInt32()) {
res[i] = (int32_t)((const JsonbIntVal*)value)->val();
} else {
null_map[i] = 1;
res[i] = 0;
}
} else if constexpr (type_index == TypeIndex::Int64) {
if (value->isInt8() || value->isInt16() || value->isInt32() ||
value->isInt64()) {
res[i] = (int64_t)((const JsonbIntVal*)value)->val();
} else {
null_map[i] = 1;
res[i] = 0;
}
} else if constexpr (type_index == TypeIndex::Int128) {
if (value->isInt8() || value->isInt16() || value->isInt32() ||
value->isInt64() || value->isInt128()) {
res[i] = (int128_t)((const JsonbIntVal*)value)->val();
} else if constexpr (type_index == TypeIndex::Int8 ||
type_index == TypeIndex::Int16 ||
type_index == TypeIndex::Int32 ||
type_index == TypeIndex::Int64 ||
type_index == TypeIndex::Int128) {
if (value->isInt()) {
res[i] = ((const JsonbIntVal*)value)->val();
} else {
null_map[i] = 1;
res[i] = 0;
@ -885,8 +874,7 @@ struct ConvertImplFromJsonb {
} else if constexpr (type_index == TypeIndex::Float64) {
if (value->isDouble()) {
res[i] = ((const JsonbDoubleVal*)value)->val();
} else if (value->isInt8() || value->isInt16() || value->isInt32() ||
value->isInt64()) {
} else if (value->isInt()) {
res[i] = ((const JsonbIntVal*)value)->val();
} else {
null_map[i] = 1;
@ -2082,7 +2070,6 @@ private:
const auto& col_with_type_and_name = block.get_by_position(arguments[0]);
auto& from_type = col_with_type_and_name.type;
auto& col_from = col_with_type_and_name.column;
// set variant root column/type to from column/type
auto variant = ColumnObject::create(true /*always nullable*/);
variant->create_root(from_type, col_from->assume_mutable());

15
be/test/vec/function/function_jsonb_test.cpp
View File

@ -1271,11 +1271,11 @@ TEST(FunctionJsonbTEST, JsonbCastToOtherTest) {
{{STRING("null"), static_cast<int8_t>(TypeIndex::Int8)}, Null()},
{{STRING("true"), static_cast<int8_t>(TypeIndex::Int8)}, Null()},
{{STRING("false"), static_cast<int8_t>(TypeIndex::Int8)}, Null()},
{{STRING("100"), static_cast<int8_t>(TypeIndex::Int8)}, TINYINT(100)}, //int8
{{STRING("10000"), static_cast<int8_t>(TypeIndex::Int8)}, Null()}, // int16
{{STRING("1000000000"), static_cast<int8_t>(TypeIndex::Int8)}, Null()}, // int32
{{STRING("100"), static_cast<int8_t>(TypeIndex::Int8)}, TINYINT(100)}, //int8
{{STRING("10000"), static_cast<int8_t>(TypeIndex::Int8)}, TINYINT(16)}, // int16
{{STRING("1000000000"), static_cast<int8_t>(TypeIndex::Int8)}, TINYINT(0)}, // int32
{{STRING("1152921504606846976"), static_cast<int8_t>(TypeIndex::Int8)},
Null()}, // int64
TINYINT(0)}, // int64
{{STRING("6.18"), static_cast<int8_t>(TypeIndex::Int8)}, Null()}, // double
{{STRING(R"("abcd")"), static_cast<int8_t>(TypeIndex::Int8)}, Null()}, // string
{{STRING("{}"), static_cast<int8_t>(TypeIndex::Int8)}, Null()}, // empty object
@ -1306,9 +1306,10 @@ TEST(FunctionJsonbTEST, JsonbCastToOtherTest) {
{{STRING("false"), static_cast<int16_t>(TypeIndex::Int16)}, Null()},
{{STRING("100"), static_cast<int16_t>(TypeIndex::Int16)}, SMALLINT(100)}, //int8
{{STRING("10000"), static_cast<int16_t>(TypeIndex::Int16)}, SMALLINT(10000)}, // int16
{{STRING("1000000000"), static_cast<int16_t>(TypeIndex::Int16)}, Null()}, // int32
{{STRING("1000000000"), static_cast<int16_t>(TypeIndex::Int16)},
SMALLINT(-13824)}, // int32
{{STRING("1152921504606846976"), static_cast<int16_t>(TypeIndex::Int16)},
Null()}, // int64
SMALLINT(0)}, // int64
{{STRING("6.18"), static_cast<int16_t>(TypeIndex::Int16)}, Null()}, // double
{{STRING(R"("abcd")"), static_cast<int16_t>(TypeIndex::Int16)}, Null()}, // string
{{STRING("{}"), static_cast<int16_t>(TypeIndex::Int16)}, Null()}, // empty object
@ -1342,7 +1343,7 @@ TEST(FunctionJsonbTEST, JsonbCastToOtherTest) {
{{STRING("1000000000"), static_cast<int32_t>(TypeIndex::Int32)},
INT(1000000000)}, // int32
{{STRING("1152921504606846976"), static_cast<int32_t>(TypeIndex::Int32)},
Null()}, // int64
INT(0)}, // int64
{{STRING("6.18"), static_cast<int32_t>(TypeIndex::Int32)}, Null()}, // double
{{STRING(R"("abcd")"), static_cast<int32_t>(TypeIndex::Int32)}, Null()}, // string
{{STRING("{}"), static_cast<int32_t>(TypeIndex::Int32)}, Null()}, // empty object

43
regression-test/data/datatype_p0/json/json_cast.out Normal file
View File

@ -0,0 +1,43 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !sql1 --
10
-- !sql2 --
23
-- !sql3 --
-28649
-- !sql4 --
-679213870
-- !sql5 --
1234
-- !sql6 --
-46
-- !sql7 --
true
-- !sql8 --
\N
-- !sql9 --
1000.1111
-- !sql10 --
\N
-- !sql11 --
["CXO0N: 1045901740","HMkTa: 1348450505","44 HHD: 915015173","j9WoJ: -1517316688"]
-- !sql12 --
111111
-- !sql13 --
111111
-- !sql14 --
1.1111

52
regression-test/data/json_p0/test_json_load_and_function.out
View File

@ -5151,8 +5151,8 @@
4 false \N
5 100 100
6 10000 10000
7 1000000000 \N
8 1152921504606846976 \N
7 1000000000 -13824
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -5166,9 +5166,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
30 -9223372036854775808 \N
31 18446744073709551615 \N
29 12524337771678448270 -17778
30 -9223372036854775808 0
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -5178,7 +5178,7 @@
5 100 100
6 10000 10000
7 1000000000 1000000000
8 1152921504606846976 \N
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -5192,9 +5192,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
30 -9223372036854775808 \N
31 18446744073709551615 \N
29 12524337771678448270 -1209615730
30 -9223372036854775808 0
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -5218,9 +5218,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
29 12524337771678448270 -5922406302031103346
30 -9223372036854775808 -9223372036854775808
31 18446744073709551615 \N
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -5244,9 +5244,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
29 12524337771678448270 1.2524337771678448E19
30 -9223372036854775808 -9.223372036854776E18
31 18446744073709551615 \N
31 18446744073709551615 1.8446744073709552E19
-- !select --
1 \N \N
@ -5307,8 +5307,8 @@
4 false \N
5 100 100
6 10000 10000
7 1000000000 \N
8 1152921504606846976 \N
7 1000000000 -13824
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -5322,9 +5322,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
30 -9223372036854775808 \N
31 18446744073709551615 \N
29 12524337771678448270 -17778
30 -9223372036854775808 0
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -5334,7 +5334,7 @@
5 100 100
6 10000 10000
7 1000000000 1000000000
8 1152921504606846976 \N
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -5348,9 +5348,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
30 -9223372036854775808 \N
31 18446744073709551615 \N
29 12524337771678448270 -1209615730
30 -9223372036854775808 0
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -5374,9 +5374,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
29 12524337771678448270 -5922406302031103346
30 -9223372036854775808 -9223372036854775808
31 18446744073709551615 \N
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -5400,9 +5400,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
29 12524337771678448270 1.25243377716784e+19
30 -9223372036854775808 -9.22337203685478e+18
31 18446744073709551615 \N
31 18446744073709551615 1.84467440737096e+19
-- !select --
1 \N \N

12
regression-test/data/json_p0/test_json_load_unique_key_and_function.out
View File

@ -3959,8 +3959,8 @@
4 false \N
5 100 100
6 10000 10000
7 1000000000 \N
8 1152921504606846976 \N
7 1000000000 -13824
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -3983,7 +3983,7 @@
5 100 100
6 10000 10000
7 1000000000 1000000000
8 1152921504606846976 \N
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -4097,8 +4097,8 @@
4 false \N
5 100 100
6 10000 10000
7 1000000000 \N
8 1152921504606846976 \N
7 1000000000 -13824
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -4121,7 +4121,7 @@
5 100 100
6 10000 10000
7 1000000000 1000000000
8 1152921504606846976 \N
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N

52
regression-test/data/jsonb_p0/test_jsonb_load_and_function.out
View File

@ -6942,8 +6942,8 @@
4 false \N
5 100 100
6 10000 10000
7 1000000000 \N
8 1152921504606846976 \N
7 1000000000 -13824
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -6957,9 +6957,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
30 -9223372036854775808 \N
31 18446744073709551615 \N
29 12524337771678448270 -17778
30 -9223372036854775808 0
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -6969,7 +6969,7 @@
5 100 100
6 10000 10000
7 1000000000 1000000000
8 1152921504606846976 \N
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -6983,9 +6983,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
30 -9223372036854775808 \N
31 18446744073709551615 \N
29 12524337771678448270 -1209615730
30 -9223372036854775808 0
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -7009,9 +7009,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
29 12524337771678448270 -5922406302031103346
30 -9223372036854775808 -9223372036854775808
31 18446744073709551615 \N
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -7035,9 +7035,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
29 12524337771678448270 1.2524337771678448E19
30 -9223372036854775808 -9.223372036854776E18
31 18446744073709551615 \N
31 18446744073709551615 1.8446744073709552E19
-- !select --
1 \N \N
@ -7098,8 +7098,8 @@
4 false \N
5 100 100
6 10000 10000
7 1000000000 \N
8 1152921504606846976 \N
7 1000000000 -13824
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -7113,9 +7113,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
30 -9223372036854775808 \N
31 18446744073709551615 \N
29 12524337771678448270 -17778
30 -9223372036854775808 0
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -7125,7 +7125,7 @@
5 100 100
6 10000 10000
7 1000000000 1000000000
8 1152921504606846976 \N
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -7139,9 +7139,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
30 -9223372036854775808 \N
31 18446744073709551615 \N
29 12524337771678448270 -1209615730
30 -9223372036854775808 0
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -7165,9 +7165,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
29 12524337771678448270 -5922406302031103346
30 -9223372036854775808 -9223372036854775808
31 18446744073709551615 \N
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -7191,9 +7191,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
29 12524337771678448270 1.25243377716784e+19
30 -9223372036854775808 -9.22337203685478e+18
31 18446744073709551615 \N
31 18446744073709551615 1.84467440737096e+19
-- !select --
1 \N \N

12
regression-test/data/jsonb_p0/test_jsonb_load_unique_key_and_function.out
View File

@ -5546,8 +5546,8 @@
4 false \N
5 100 100
6 10000 10000
7 1000000000 \N
8 1152921504606846976 \N
7 1000000000 -13824
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -5570,7 +5570,7 @@
5 100 100
6 10000 10000
7 1000000000 1000000000
8 1152921504606846976 \N
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -5684,8 +5684,8 @@
4 false \N
5 100 100
6 10000 10000
7 1000000000 \N
8 1152921504606846976 \N
7 1000000000 -13824
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -5708,7 +5708,7 @@
5 100 100
6 10000 10000
7 1000000000 1000000000
8 1152921504606846976 \N
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N

52
regression-test/data/nereids_function_p0/scalar_function/J.out
View File

@ -6942,8 +6942,8 @@
4 false \N
5 100 100
6 10000 10000
7 1000000000 \N
8 1152921504606846976 \N
7 1000000000 -13824
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -6957,9 +6957,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
30 -9223372036854775808 \N
31 18446744073709551615 \N
29 12524337771678448270 -17778
30 -9223372036854775808 0
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -6969,7 +6969,7 @@
5 100 100
6 10000 10000
7 1000000000 1000000000
8 1152921504606846976 \N
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -6983,9 +6983,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
30 -9223372036854775808 \N
31 18446744073709551615 \N
29 12524337771678448270 -1209615730
30 -9223372036854775808 0
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -7009,9 +7009,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
29 12524337771678448270 -5922406302031103346
30 -9223372036854775808 -9223372036854775808
31 18446744073709551615 \N
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -7035,9 +7035,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
29 12524337771678448270 1.2524337771678448E19
30 -9223372036854775808 -9.223372036854776E18
31 18446744073709551615 \N
31 18446744073709551615 1.8446744073709552E19
-- !select --
1 \N \N
@ -7098,8 +7098,8 @@
4 false \N
5 100 100
6 10000 10000
7 1000000000 \N
8 1152921504606846976 \N
7 1000000000 -13824
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -7113,9 +7113,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
30 -9223372036854775808 \N
31 18446744073709551615 \N
29 12524337771678448270 -17778
30 -9223372036854775808 0
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -7125,7 +7125,7 @@
5 100 100
6 10000 10000
7 1000000000 1000000000
8 1152921504606846976 \N
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -7139,9 +7139,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
30 -9223372036854775808 \N
31 18446744073709551615 \N
29 12524337771678448270 -1209615730
30 -9223372036854775808 0
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -7165,9 +7165,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
29 12524337771678448270 -5922406302031103346
30 -9223372036854775808 -9223372036854775808
31 18446744073709551615 \N
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -7191,9 +7191,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
29 12524337771678448270 1.25243377716784e+19
30 -9223372036854775808 -9.22337203685478e+18
31 18446744073709551615 \N
31 18446744073709551615 1.84467440737096e+19
-- !select --
1 \N \N

52
regression-test/data/nereids_p0/json_p0/test_json_load_and_function.out
View File

@ -5148,8 +5148,8 @@
4 false \N
5 100 100
6 10000 10000
7 1000000000 \N
8 1152921504606846976 \N
7 1000000000 -13824
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -5163,9 +5163,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
30 -9223372036854775808 \N
31 18446744073709551615 \N
29 12524337771678448270 -17778
30 -9223372036854775808 0
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -5175,7 +5175,7 @@
5 100 100
6 10000 10000
7 1000000000 1000000000
8 1152921504606846976 \N
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -5189,9 +5189,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
30 -9223372036854775808 \N
31 18446744073709551615 \N
29 12524337771678448270 -1209615730
30 -9223372036854775808 0
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -5215,9 +5215,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
29 12524337771678448270 -5922406302031103346
30 -9223372036854775808 -9223372036854775808
31 18446744073709551615 \N
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -5241,9 +5241,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
29 12524337771678448270 1.2524337771678448E19
30 -9223372036854775808 -9.223372036854776E18
31 18446744073709551615 \N
31 18446744073709551615 1.8446744073709552E19
-- !select --
1 \N \N
@ -5304,8 +5304,8 @@
4 false \N
5 100 100
6 10000 10000
7 1000000000 \N
8 1152921504606846976 \N
7 1000000000 -13824
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -5319,9 +5319,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
30 -9223372036854775808 \N
31 18446744073709551615 \N
29 12524337771678448270 -17778
30 -9223372036854775808 0
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -5331,7 +5331,7 @@
5 100 100
6 10000 10000
7 1000000000 1000000000
8 1152921504606846976 \N
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -5345,9 +5345,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
30 -9223372036854775808 \N
31 18446744073709551615 \N
29 12524337771678448270 -1209615730
30 -9223372036854775808 0
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -5371,9 +5371,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
29 12524337771678448270 -5922406302031103346
30 -9223372036854775808 -9223372036854775808
31 18446744073709551615 \N
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -5397,9 +5397,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
29 12524337771678448270 1.25243377716784e+19
30 -9223372036854775808 -9.22337203685478e+18
31 18446744073709551615 \N
31 18446744073709551615 1.84467440737096e+19
-- !select --
1 \N \N

12
regression-test/data/nereids_p0/json_p0/test_json_load_unique_key_and_function.out
View File

@ -3959,8 +3959,8 @@
4 false \N
5 100 100
6 10000 10000
7 1000000000 \N
8 1152921504606846976 \N
7 1000000000 -13824
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -3983,7 +3983,7 @@
5 100 100
6 10000 10000
7 1000000000 1000000000
8 1152921504606846976 \N
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -4097,8 +4097,8 @@
4 false \N
5 100 100
6 10000 10000
7 1000000000 \N
8 1152921504606846976 \N
7 1000000000 -13824
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -4121,7 +4121,7 @@
5 100 100
6 10000 10000
7 1000000000 1000000000
8 1152921504606846976 \N
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N

52
regression-test/data/nereids_p0/jsonb_p0/test_jsonb_load_and_function.out
View File

@ -6942,8 +6942,8 @@
4 false \N
5 100 100
6 10000 10000
7 1000000000 \N
8 1152921504606846976 \N
7 1000000000 -13824
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -6957,9 +6957,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
30 -9223372036854775808 \N
31 18446744073709551615 \N
29 12524337771678448270 -17778
30 -9223372036854775808 0
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -6969,7 +6969,7 @@
5 100 100
6 10000 10000
7 1000000000 1000000000
8 1152921504606846976 \N
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -6983,9 +6983,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
30 -9223372036854775808 \N
31 18446744073709551615 \N
29 12524337771678448270 -1209615730
30 -9223372036854775808 0
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -7009,9 +7009,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
29 12524337771678448270 -5922406302031103346
30 -9223372036854775808 -9223372036854775808
31 18446744073709551615 \N
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -7035,9 +7035,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
29 12524337771678448270 1.2524337771678448E19
30 -9223372036854775808 -9.223372036854776E18
31 18446744073709551615 \N
31 18446744073709551615 1.8446744073709552E19
-- !select --
1 \N \N
@ -7098,8 +7098,8 @@
4 false \N
5 100 100
6 10000 10000
7 1000000000 \N
8 1152921504606846976 \N
7 1000000000 -13824
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -7113,9 +7113,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
30 -9223372036854775808 \N
31 18446744073709551615 \N
29 12524337771678448270 -17778
30 -9223372036854775808 0
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -7125,7 +7125,7 @@
5 100 100
6 10000 10000
7 1000000000 1000000000
8 1152921504606846976 \N
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -7139,9 +7139,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
30 -9223372036854775808 \N
31 18446744073709551615 \N
29 12524337771678448270 -1209615730
30 -9223372036854775808 0
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -7165,9 +7165,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
29 12524337771678448270 -5922406302031103346
30 -9223372036854775808 -9223372036854775808
31 18446744073709551615 \N
31 18446744073709551615 -1
-- !select --
1 \N \N
@ -7191,9 +7191,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
29 12524337771678448270 1.25243377716784e+19
30 -9223372036854775808 -9.22337203685478e+18
31 18446744073709551615 \N
31 18446744073709551615 1.84467440737096e+19
-- !select --
1 \N \N

12
regression-test/data/nereids_p0/jsonb_p0/test_jsonb_load_unique_key_and_function.out
View File

@ -5546,8 +5546,8 @@
4 false \N
5 100 100
6 10000 10000
7 1000000000 \N
8 1152921504606846976 \N
7 1000000000 -13824
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -5570,7 +5570,7 @@
5 100 100
6 10000 10000
7 1000000000 1000000000
8 1152921504606846976 \N
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -5684,8 +5684,8 @@
4 false \N
5 100 100
6 10000 10000
7 1000000000 \N
8 1152921504606846976 \N
7 1000000000 -13824
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
@ -5708,7 +5708,7 @@
5 100 100
6 10000 10000
7 1000000000 1000000000
8 1152921504606846976 \N
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N

17
regression-test/data/variant_p0/load.out
View File

@ -212,11 +212,14 @@
[123]
-- !sql_25 --
50000 55000.00000000545 6150000
50000 54999.9999999998 6150000
-- !sql_26 --
5000
-- !sql_27 --
16
-- !sql_29_1 --
1 {"kxxxx":123} {"xxxxyyyy":123}
1 {"kyyyy":"123"} {"kxkxkxkx":[123]}
@ -322,3 +325,15 @@
-- !sql_31 --
kaana
-- !sql_39 --
1 array
2 string
3 bigint
4 double
-- !sql_39 --
["CXO0N: 1045901740", "HMkTa: 1348450505", "44 HHD: 915015173", "j9WoJ: -1517316688"]
-- !sql_39 --
[1]

34
regression-test/suites/datatype_p0/json/json_cast.groovy Normal file
View File

@ -0,0 +1,34 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_json_type_cast", "p0") {
qt_sql1 "SELECT CAST(CAST(10 AS JSON) as INT)"
qt_sql2 "SELECT CAST(CAST(102423 AS JSON) as TINYINT)"
qt_sql3 "SELECT CAST(CAST(102423 AS JSON) as SMALLINT)"
qt_sql4 "SELECT CAST(CAST(102400001234 AS JSON) as INT)"
qt_sql5 "SELECT CAST(CAST(102400001234 AS JSON) as SMALLINT)"
qt_sql6 "SELECT CAST(CAST(102400001234 AS JSON) as TINYINT)"
qt_sql7 "SELECT CAST(CAST(102400001234 AS JSON) as BOOLEAN)"
qt_sql8 "SELECT CAST(CAST(1000.1111 AS JSON) as INT)"
qt_sql9 "SELECT CAST(CAST(1000.1111 AS JSON) as DOUBLE)"
qt_sql10 "SELECT CAST(CAST(1000.1111 AS JSON) as BOOLEAN)"
qt_sql11 """select cast('["CXO0N: 1045901740", "HMkTa: 1348450505", "44 HHD: 915015173", "j9WoJ: -1517316688"]' as json);"""
qt_sql12 """select cast("111111" as json)"""
qt_sql13 """select cast(111111 as json)"""
qt_sql14 """select cast(1.1111 as json)"""
}

25
regression-test/suites/variant_p0/load.groovy
View File

@ -206,13 +206,14 @@ suite("regression_test_variant", "nonConcurrent"){
load_json_data.call(table_name, """${getS3Url() + '/load/ghdata_sample.json'}""")
qt_sql_26 "select count() from ${table_name}"
// FIXME: this case it not passed
// // 8. json empty string
// // table_name = "empty_string"
// // create_table table_name
// // sql """INSERT INTO empty_string VALUES (1, ''), (2, '{"k1": 1, "k2": "v1"}'), (3, '{}'), (4, '{"k1": 2}');"""
// // sql """INSERT INTO empty_string VALUES (3, null), (4, '{"k1": 1, "k2": "v1"}'), (3, '{}'), (4, '{"k1": 2}');"""
// // qt_sql_27 "SELECT * FROM ${table_name} ORDER BY k;"
// 8. json empty string
table_name = "empty_string"
create_table table_name
sql """INSERT INTO empty_string VALUES (1, ''), (2, '{"k1": 1, "k2": "v1"}'), (3, '{}'), (4, '{"k1": 2}');"""
sql """INSERT INTO empty_string VALUES (3, null), (4, '{"k1": 1, "k2": "v1"}'), (3, '{}'), (4, '{"k1": 2}');"""
sql """INSERT INTO empty_string VALUES (3, null), (4, null), (3, '{}'), (4, '{"k1": 2}');"""
sql """INSERT INTO empty_string VALUES (3, ''), (4, null), (3, '{}'), (4, null);"""
qt_sql_27 "SELECT count() FROM ${table_name};"
// // // 9. btc data
// // table_name = "btcdata"
@ -393,6 +394,16 @@ suite("regression_test_variant", "nonConcurrent"){
qt_sql_31 """select cast(v['xxxx'] as string) from sparse_columns where cast(v['xxxx'] as string) != 'null' order by k limit 1;"""
sql "truncate table sparse_columns"
set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95")
// test cast
table_name = "variant_cast"
create_table.call(table_name, "DUPLICATE", "1")
sql """
insert into variant_cast values(1,'["CXO0N: 1045901740", "HMkTa: 1348450505", "44 HHD: 915015173", "j9WoJ: -1517316688"]'),(2,'"[1]"'),(3,'123456'),(4,'1.11111')
"""
qt_sql_39 "select k, json_type(cast(v as json), '\$') from variant_cast order by k"
qt_sql_39 "select cast(v as array<text>) from variant_cast where k = 1 order by k"
qt_sql_39 "select cast(v as string) from variant_cast where k = 2 order by k"
} finally {
// reset flags
set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95")