database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](vstream load) Fix the default value insertion problem when importing json (#12601)

Browse Source 
* [fix](vstream load) Fix the default value insertion problem when importing json

* update
This commit is contained in:
yinzhijian
2022-09-16 09:54:45 +08:00
committed by GitHub
parent a97f63141e
commit 2a063355ad
3 changed files with 207 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
be/src/vec/exec/vjson_scanner.cpp
View File

@ -232,8 +232,9 @@ Status VJsonReader::_set_column_value(rapidjson::Value& objectValue,
return Status::OK();
}
int nullcount = 0;
int ctx_idx = 0;
bool has_valid_value = false;
size_t cur_row_count = columns[0]->size();
for (auto slot_desc : slot_descs) {
if (!slot_desc->is_materialized()) {
continue;
@ -259,12 +260,11 @@ Status VJsonReader::_set_column_value(rapidjson::Value& objectValue,
if (!(*valid)) {
return Status::OK();
}
has_valid_value = true;
} else { // not found
if (slot_desc->is_nullable()) {
auto* nullable_column = reinterpret_cast<vectorized::ColumnNullable*>(column_ptr);
nullable_column->insert_default();
nullcount++;
} else {
// When the entire row has no valid value, this row should be filtered,
// so the default value cannot be directly inserted here
if (!slot_desc->is_nullable()) {
RETURN_IF_ERROR(_append_error_msg(
objectValue,
"The column `{}` is not nullable, but it's not found in jsondata.",
@ -273,12 +273,29 @@ Status VJsonReader::_set_column_value(rapidjson::Value& objectValue,
}
}
}
if (nullcount == slot_descs.size()) {
if (!has_valid_value) {
RETURN_IF_ERROR(_append_error_msg(objectValue, "All fields is null, this is a invalid row.",
"", valid));
return Status::OK();
}
ctx_idx = 0;
int nullcount = 0;
// fill missing slot
for (auto slot_desc : slot_descs) {
if (!slot_desc->is_materialized()) {
continue;
}
int dest_index = ctx_idx++;
auto* column_ptr = columns[dest_index].get();
if (column_ptr->size() < cur_row_count + 1) {
DCHECK(column_ptr->size() == cur_row_count);
column_ptr->assume_mutable()->insert_default();
++nullcount;
}
DCHECK(column_ptr->size() == cur_row_count + 1);
}
// There is at least one valid value here
DCHECK(nullcount < columns.size());
*valid = true;
return Status::OK();
}
@ -583,6 +600,7 @@ Status VSIMDJsonReader::_set_column_value(simdjson::ondemand::value objectValue,
auto object_val = objectValue.get_object();
size_t cur_row_count = block.rows();
bool has_valid_value = false;
// iterate through object, simdjson::ondemond will parsing on the fly
for (auto field : object_val) {
std::string_view key;
@ -596,6 +614,11 @@ Status VSIMDJsonReader::_set_column_value(simdjson::ondemand::value objectValue,
if (!(*valid)) {
return Status::OK();
}
has_valid_value = true;
}
if (!has_valid_value) {
RETURN_IF_ERROR(_append_error_msg("All fields is null, this is a invalid row.", "", valid));
return Status::OK();
}
int nullcount = 0;
@ -603,11 +626,11 @@ Status VSIMDJsonReader::_set_column_value(simdjson::ondemand::value objectValue,
for (const auto& column_type_name : block) {
auto column = column_type_name.column;
if (column->size() < cur_row_count + 1) {
assert(column->size() == cur_row_count);
DCHECK(column->size() == cur_row_count);
column->assume_mutable()->insert_default();
++nullcount;
}
assert(column->size() == cur_row_count + 1);
DCHECK(column->size() == cur_row_count + 1);
}
if (nullcount == block.columns()) {
RETURN_IF_ERROR(_append_error_msg("All fields is null, this is a invalid row.", "", valid));

62
regression-test/data/load_p0/stream_load/json_column_match.json Normal file
View File

@ -0,0 +1,62 @@
[ {"hh10": "7395.231067", "hh3": "1000", "hh2": "100", "hh1": "10", "hh0": "1", "hh7": "ynqnzeowymt", "hh6": "t", "hh5": "2010-01-01", "hh4": "2011-01-01 00:00:00", "hh9": "180.998031", "hh8": "38.638843"},
{"hh10": "2080.504502", "hh3": "2000", "hh2": "200", "hh1": "20", "hh0": "2", "hh7": "hfkfwlr", "hh6": "f", "hh5": "2010-01-02", "hh4": "2012-01-01 00:00:00", "hh9": "539.922834", "hh8": "506.044046"},
{"hh10": "4605.253205", "hh3": "3000", "hh2": "300", "hh1": "30", "hh0": "3", "hh7": "uoclasp", "hh6": "t", "hh5": "2010-01-03", "hh4": "2013-01-01 00:00:00", "hh9": "577.044148", "hh8": "377.793209"},
{"hh10": "7291.703724", "hh3": "4000", "hh2": "400", "hh1": "40", "hh0": "4", "hh7": "iswngzeodfhptjzgswsddt", "hh6": "n", "hh5": "2010-01-04", "hh4": "2014-01-01 00:00:00", "hh9": "919.067864", "hh8": "871.354536"},
{"hh10": "3903.906901", "hh3": "5000", "hh2": "500", "hh1": "50", "hh0": "5", "hh7": "sqodagzlyrmcelyxgcgcsfuxadcdt", "hh6": "a", "hh5": "2010-01-05", "hh4": "2015-01-01 00:00:00", "hh9": "929.660783", "hh8": "462.067902"},
{"hh10": "6514.405051", "hh3": "6000", "hh2": "600", "hh1": "60", "hh0": "6", "hh7": "obdrei", "hh6": "m", "hh5": "2010-01-06", "hh4": "2016-01-01 00:00:00", "hh9": "882.708491", "hh8": "921.867848"},
{"hh10": "8604.198677", "hh3": "7000", "hh2": "700", "hh1": "70", "hh0": "7", "hh7": "cuobdhvrgkugknj", "hh6": "a", "hh5": "2010-01-07", "hh4": "2017-01-01 00:00:00", "hh9": "209.420112", "hh8": "141.656421"},
{"hh10": "7784.859446", "hh3": "8000", "hh2": "800", "hh1": "80", "hh0": "8", "hh7": "phcxztwgjllhmj", "hh6": "z", "hh5": "2010-01-08", "hh4": "2018-01-01 00:00:00", "hh9": "285.664871", "hh8": "762.813376"},
{"hh10": "4846.735593", "hh3": "9000", "hh2": "900", "hh1": "90", "hh0": "9", "hh7": "nbarqjwilbkelk", "hh6": "b", "hh5": "2010-01-09", "hh4": "2019-01-01 00:00:00", "hh9": "535.285510", "hh8": "92.702403"},
{"hh10": "7996.434686", "hh3": "10000", "hh2": "1000", "hh1": "100", "hh0": "10", "hh7": "zucprgdnlgzzfl", "hh6": "s", "hh5": "2010-01-10", "hh4": "2020-01-01 00:00:00", "hh9": "155.861217", "hh8": "26.874738"},
{"hh10": "3821.278705", "hh3": "11000", "hh2": "1100", "hh1": "110", "hh0": "11", "hh7": "zabbwiimqemk", "hh6": "j", "hh5": "2010-01-11", "hh4": "2021-01-01 00:00:00", "hh9": "558.999245", "hh8": "369.819134"},
{"hh10": "9334.300973", "hh3": "12000", "hh2": "1200", "hh1": "120", "hh0": "12", "hh7": "elvkhacywhdzrazcvyunkjajnx", "hh6": "d", "hh5": "2010-01-12", "hh4": "2022-01-01 00:00:00", "hh9": "181.877653", "hh8": "536.045037"},
{"hh10": "9103.804406", "hh3": "13000", "hh2": "1300", "hh1": "130", "hh0": "13", "hh7": "cylgmxlmkkrkk", "hh6": "x", "hh5": "2010-01-13", "hh4": "2023-01-01 00:00:00", "hh9": "909.667873", "hh8": "990.836863"},
{"hh10": "9823.062496", "hh3": "14000", "hh2": "1400", "hh1": "140", "hh0": "14", "hh7": "vpzwml", "hh6": "s", "hh5": "2010-01-14", "hh4": "2024-01-01 00:00:00", "hh9": "295.541079", "hh8": "763.615167"},
{"hh10": "5859.241974", "hh3": "15000", "hh2": "1500", "hh1": "150", "hh0": "15", "hh7": "yvvstxajxtgrimmrveljjbwo", "hh6": "f", "hh5": "2010-01-15", "hh4": "2025-01-01 00:00:00", "hh9": "391.329557", "hh8": "380.419929"},
{"hh10": "9430.138013", "hh3": "16000", "hh2": "1600", "hh1": "160", "hh0": "16", "hh7": "yxtowesbeqyejvpfhkixpdw", "hh6": "b", "hh5": "2010-01-16", "hh4": "2026-01-01 00:00:00", "hh9": "328.666079", "hh8": "312.933953"},
{"hh10": "3135.292842", "hh3": "17000", "hh2": "1700", "hh1": "170", "hh0": "17", "hh7": "nrrzcgygjplgttf", "hh6": "d", "hh5": "2010-01-17", "hh4": "2027-01-01 00:00:00", "hh9": "799.528036", "hh8": "472.844369"},
{"hh10": "9855.813711", "hh3": "18000", "hh2": "1800", "hh1": "180", "hh0": "18", "hh7": "wfknyaxplas", "hh6": "e", "hh5": "2010-01-18", "hh4": "2028-01-01 00:00:00", "hh9": "844.025534", "hh8": "971.549437"},
{"hh10": "3086.573874", "hh3": "19000", "hh2": "1900", "hh1": "190", "hh0": "19", "hh7": "qtznw", "hh6": "f", "hh5": "2010-01-19", "hh4": "2029-01-01 00:00:00", "hh9": "502.584396", "hh8": "37.568959"},
{"hh10": "2698.863638", "hh3": "20000", "hh2": "2000", "hh1": "200", "hh0": "20", "hh7": "nwznneizhtmzemy", "hh6": "i", "hh5": "2010-01-20", "hh4": "2030-01-01 00:00:00", "hh9": "972.561324", "hh8": "632.512198"},
{"hh10": "5185.373039", "hh3": "21000", "hh2": "2100", "hh1": "210", "hh0": "21", "hh7": "sqqeanrmafdoheeizljifwsj", "hh6": "q", "hh5": "2010-01-21", "hh4": "2031-01-01 00:00:00", "hh9": "933.754744", "hh8": "942.888716"},
{"hh10": "6221.348291", "hh3": "22000", "hh2": "2200", "hh1": "220", "hh0": "22", "hh7": "tirlgqmwrchusfyxitxttvxlioknz", "hh6": "h", "hh5": "2010-01-22", "hh4": "2032-01-01 00:00:00", "hh9": "69.032071", "hh8": "654.955955"},
{"hh10": "7481.574657", "hh3": "23000", "hh2": "2300", "hh1": "230", "hh0": "23", "hh7": "imnrusvqy", "hh6": "s", "hh5": "2010-01-23", "hh4": "2033-01-01 00:00:00", "hh9": "24.626005", "hh8": "468.780338"},
{"hh10": "3585.208809", "hh3": "24000", "hh2": "2400", "hh1": "240", "hh0": "24", "hh7": "irpbe", "hh6": "s", "hh5": "2010-01-24", "hh4": "2034-01-01 00:00:00", "hh9": "396.750845", "hh8": "537.906124"},
{"hh10": "6982.796060", "hh3": "25000", "hh2": "2500", "hh1": "250", "hh0": "25", "hh7": "drpjerrdlel", "hh6": "w", "hh5": "2010-01-25", "hh4": "2035-01-01 00:00:00", "hh9": "202.007097", "hh8": "662.123247"},
{"hh10": "3046.839203", "hh3": "26000", "hh2": "2600", "hh1": "260", "hh0": "26", "hh7": "vmzutsaifmlimicshgjpsvhiowjnq", "hh6": "v", "hh5": "2010-01-26", "hh4": "2036-01-01 00:00:00", "hh9": "64.115478", "hh8": "115.145598"},
{"hh10": "9921.135045", "hh3": "27000", "hh2": "2700", "hh1": "270", "hh0": "27", "hh7": "yrxspxgcwgbnjnmqkcido", "hh6": "d", "hh5": "2010-01-27", "hh4": "2037-01-01 00:00:00", "hh9": "519.383701", "hh8": "362.227207"},
{"hh10": "4633.258698", "hh3": "28000", "hh2": "2800", "hh1": "280", "hh0": "28", "hh7": "udfzebgnnxfjnoujtvlib", "hh6": "w", "hh5": "2010-01-28", "hh4": "2038-01-01 00:00:00", "hh9": "998.471600", "hh8": "443.030856"},
{"hh10": "5465.010345", "hh3": "29000", "hh2": "2900", "hh1": "290", "hh0": "29", "hh7": "jfgvke", "hh6": "p", "hh5": "2010-01-29", "hh4": "2039-01-01 00:00:00", "hh9": "248.265099", "hh8": "8.602754"},
{"hh10": "2579.668636", "hh3": "30000", "hh2": "3000", "hh1": "300", "hh0": "30", "hh7": "tbuvpobzluhbwkljlhwnrkrhowybk", "hh6": "d", "hh5": "2010-01-30", "hh4": "2040-01-01 00:00:00", "hh9": "976.266490", "hh8": "949.575593"},
{"v6": "7395.231067", "k4": "1000", "k3": "100", "k2": "10", "k1": "1", "v3": "ynqnzeowymt", "v2": "t", "v1": "2010-01-01", "k5": "2011-01-01 00:00:00", "v5": "180.998031", "v4": "38.638843"},
{"v6": "2080.504502", "k4": "2000", "k3": "200", "k2": "20", "k1": "2", "v3": "hfkfwlr", "v2": "f", "v1": "2010-01-02", "k5": "2012-01-01 00:00:00", "v5": "539.922834", "v4": "506.044046"},
{"v6": "4605.253205", "k4": "3000", "k3": "300", "k2": "30", "k1": "3", "v3": "uoclasp", "v2": "t", "v1": "2010-01-03", "k5": "2013-01-01 00:00:00", "v5": "577.044148", "v4": "377.793209"},
{"v6": "7291.703724", "k4": "4000", "k3": "400", "k2": "40", "k1": "4", "v3": "iswngzeodfhptjzgswsddt", "v2": "n", "v1": "2010-01-04", "k5": "2014-01-01 00:00:00", "v5": "919.067864", "v4": "871.354536"},
{"v6": "3903.906901", "k4": "5000", "k3": "500", "k2": "50", "k1": "5", "v3": "sqodagzlyrmcelyxgcgcsfuxadcdt", "v2": "a", "v1": "2010-01-05", "k5": "2015-01-01 00:00:00", "v5": "929.660783", "v4": "462.067902"},
{"v6": "6514.405051", "k4": "6000", "k3": "600", "k2": "60", "k1": "6", "v3": "obdrei", "v2": "m", "v1": "2010-01-06", "k5": "2016-01-01 00:00:00", "v5": "882.708491", "v4": "921.867848"},
{"v6": "8604.198677", "k4": "7000", "k3": "700", "k2": "70", "k1": "7", "v3": "cuobdhvrgkugknj", "v2": "a", "v1": "2010-01-07", "k5": "2017-01-01 00:00:00", "v5": "209.420112", "v4": "141.656421"},
{"v6": "7784.859446", "k4": "8000", "k3": "800", "k2": "80", "k1": "8", "v3": "phcxztwgjllhmj", "v2": "z", "v1": "2010-01-08", "k5": "2018-01-01 00:00:00", "v5": "285.664871", "v4": "762.813376"},
{"v6": "4846.735593", "k4": "9000", "k3": "900", "k2": "90", "k1": "9", "v3": "nbarqjwilbkelk", "v2": "b", "v1": "2010-01-09", "k5": "2019-01-01 00:00:00", "v5": "535.285510", "v4": "92.702403"},
{"v6": "7996.434686", "k4": "10000", "k3": "1000", "k2": "100", "k1": "10", "v3": "zucprgdnlgzzfl", "v2": "s", "v1": "2010-01-10", "k5": "2020-01-01 00:00:00", "v5": "155.861217", "v4": "26.874738"},
{"v6": "3821.278705", "k4": "11000", "k3": "1100", "k2": "110", "k1": "11", "v3": "zabbwiimqemk", "v2": "j", "v1": "2010-01-11", "k5": "2021-01-01 00:00:00", "v5": "558.999245", "v4": "369.819134"},
{"v6": "9334.300973", "k4": "12000", "k3": "1200", "k2": "120", "k1": "12", "v3": "elvkhacywhdzrazcvyunkjajnx", "v2": "d", "v1": "2010-01-12", "k5": "2022-01-01 00:00:00", "v5": "181.877653", "v4": "536.045037"},
{"v6": "9103.804406", "k4": "13000", "k3": "1300", "k2": "130", "k1": "13", "v3": "cylgmxlmkkrkk", "v2": "x", "v1": "2010-01-13", "k5": "2023-01-01 00:00:00", "v5": "909.667873", "v4": "990.836863"},
{"v6": "9823.062496", "k4": "14000", "k3": "1400", "k2": "140", "k1": "14", "v3": "vpzwml", "v2": "s", "v1": "2010-01-14", "k5": "2024-01-01 00:00:00", "v5": "295.541079", "v4": "763.615167"},
{"v6": "5859.241974", "k4": "15000", "k3": "1500", "k2": "150", "k1": "15", "v3": "yvvstxajxtgrimmrveljjbwo", "v2": "f", "v1": "2010-01-15", "k5": "2025-01-01 00:00:00", "v5": "391.329557", "v4": "380.419929"},
{"v6": "9430.138013", "k4": "16000", "k3": "1600", "k2": "160", "k1": "16", "v3": "yxtowesbeqyejvpfhkixpdw", "v2": "b", "v1": "2010-01-16", "k5": "2026-01-01 00:00:00", "v5": "328.666079", "v4": "312.933953"},
{"v6": "3135.292842", "k4": "17000", "k3": "1700", "k2": "170", "k1": "17", "v3": "nrrzcgygjplgttf", "v2": "d", "v1": "2010-01-17", "k5": "2027-01-01 00:00:00", "v5": "799.528036", "v4": "472.844369"},
{"v6": "9855.813711", "k4": "18000", "k3": "1800", "k2": "180", "k1": "18", "v3": "wfknyaxplas", "v2": "e", "v1": "2010-01-18", "k5": "2028-01-01 00:00:00", "v5": "844.025534", "v4": "971.549437"},
{"v6": "3086.573874", "k4": "19000", "k3": "1900", "k2": "190", "k1": "19", "v3": "qtznw", "v2": "f", "v1": "2010-01-19", "k5": "2029-01-01 00:00:00", "v5": "502.584396", "v4": "37.568959"},
{"v6": "2698.863638", "k4": "20000", "k3": "2000", "k2": "200", "k1": "20", "v3": "nwznneizhtmzemy", "v2": "i", "v1": "2010-01-20", "k5": "2030-01-01 00:00:00", "v5": "972.561324", "v4": "632.512198"},
{"v6": "5185.373039", "k4": "21000", "k3": "2100", "k2": "210", "k1": "21", "v3": "sqqeanrmafdoheeizljifwsj", "v2": "q", "v1": "2010-01-21", "k5": "2031-01-01 00:00:00", "v5": "933.754744", "v4": "942.888716"},
{"v6": "6221.348291", "k4": "22000", "k3": "2200", "k2": "220", "k1": "22", "v3": "tirlgqmwrchusfyxitxttvxlioknz", "v2": "h", "v1": "2010-01-22", "k5": "2032-01-01 00:00:00", "v5": "69.032071", "v4": "654.955955"},
{"v6": "7481.574657", "k4": "23000", "k3": "2300", "k2": "230", "k1": "23", "v3": "imnrusvqy", "v2": "s", "v1": "2010-01-23", "k5": "2033-01-01 00:00:00", "v5": "24.626005", "v4": "468.780338"},
{"v6": "3585.208809", "k4": "24000", "k3": "2400", "k2": "240", "k1": "24", "v3": "irpbe", "v2": "s", "v1": "2010-01-24", "k5": "2034-01-01 00:00:00", "v5": "396.750845", "v4": "537.906124"},
{"v6": "6982.796060", "k4": "25000", "k3": "2500", "k2": "250", "k1": "25", "v3": "drpjerrdlel", "v2": "w", "v1": "2010-01-25", "k5": "2035-01-01 00:00:00", "v5": "202.007097", "v4": "662.123247"},
{"v6": "3046.839203", "k4": "26000", "k3": "2600", "k2": "260", "k1": "26", "v3": "vmzutsaifmlimicshgjpsvhiowjnq", "v2": "v", "v1": "2010-01-26", "k5": "2036-01-01 00:00:00", "v5": "64.115478", "v4": "115.145598"},
{"v6": "9921.135045", "k4": "27000", "k3": "2700", "k2": "270", "k1": "27", "v3": "yrxspxgcwgbnjnmqkcido", "v2": "d", "v1": "2010-01-27", "k5": "2037-01-01 00:00:00", "v5": "519.383701", "v4": "362.227207"},
{"v6": "4633.258698", "k4": "28000", "k3": "2800", "k2": "280", "k1": "28", "v3": "udfzebgnnxfjnoujtvlib", "v2": "w", "v1": "2010-01-28", "k5": "2038-01-01 00:00:00", "v5": "998.471600", "v4": "443.030856"},
{"v6": "5465.010345", "k4": "29000", "k3": "2900", "k2": "290", "k1": "29", "v3": "jfgvke", "v2": "p", "v1": "2010-01-29", "k5": "2039-01-01 00:00:00", "v5": "248.265099", "v4": "8.602754"},
{"v6": "2579.668636", "k4": "30000", "k3": "3000", "k2": "300", "k1": "30", "v3": "tbuvpobzluhbwkljlhwnrkrhowybk", "v2": "d", "v1": "2010-01-30", "k5": "2040-01-01 00:00:00", "v5": "976.266490", "v4": "949.575593"}
]

112
regression-test/suites/load_p0/stream_load/load_json_column_exclude_schema_without_jsonpath.groovy Normal file
View File

@ -0,0 +1,112 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_load_json_column_exclude_schema_without_jsonpath", "p0") {
// define a sql table
def testTable = "tbl_test_json_load"
def dbName = "test_query_db"
def create_test_table = {enable_vectorized_flag ->
if (enable_vectorized_flag) {
sql """ set enable_vectorized_engine = true """
} else {
sql """ set enable_vectorized_engine = false """
}
def result1 = sql """
CREATE TABLE IF NOT EXISTS ${testTable} (
k1 TINYINT NULL,
k2 SMALLINT NULL,
k3 INT NULL,
k4 BIGINT NULL,
k5 DATETIME NULL,
v1 DATE REPLACE NULL,
v2 CHAR REPLACE NULL,
v3 VARCHAR(4096) REPLACE NULL,
v4 FLOAT SUM NULL,
v5 DOUBLE SUM NULL,
v6 DECIMAL(20,7) SUM NULL
) ENGINE=OLAP
AGGREGATE KEY(k1,k2,k3,k4,k5)
DISTRIBUTED BY HASH(`k1`) BUCKETS 5
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"storage_format" = "V2"
)
"""
}
def load_array_data = {table_name, strip_flag, read_flag, format_flag, exprs, json_paths,
json_root, where_expr, fuzzy_flag, column_sep, file_name ->
// load the json data
streamLoad {
table table_name
// set http request header params
set 'strip_outer_array', strip_flag
set 'read_json_by_line', read_flag
set 'format', format_flag
set 'columns', exprs
set 'jsonpaths', json_paths
set 'json_root', json_root
set 'where', where_expr
set 'fuzzy_parse', fuzzy_flag
set 'column_separator', column_sep
set 'max_filter_ratio', '0.4'
file file_name // import json file
time 10000 // limit inflight 10s
// if declared a check callback, the default check condition will ignore.
// So you must check all condition
check { result, exception, startTime, endTime ->
if (exception != null) {
throw exception
}
log.info("Stream load result: ${result}".toString())
def json = parseJson(result)
assertEquals("fail", json.Status.toLowerCase())
assertEquals(json.NumberTotalRows, json.NumberLoadedRows + json.NumberFilteredRows)
assertTrue(json.NumberLoadedRows == 30);
assertTrue(json.NumberFilteredRows == 30);
}
}
}
// case1: import array data in json format and enable vectorized engine
try {
sql "DROP TABLE IF EXISTS ${testTable}"
create_test_table.call(true)
load_array_data.call(testTable, 'true', '', 'json', '', '', '', '', '', '', 'json_column_match.json')
} finally {
try_sql("DROP TABLE IF EXISTS ${testTable}")
}
// case2: import array data in json format and disable vectorized engine
try {
sql "DROP TABLE IF EXISTS ${testTable}"
create_test_table.call(false)
load_array_data.call(testTable, 'true', '', 'json', '', '', '', '', '', '', 'json_column_match.json')
} finally {
try_sql("DROP TABLE IF EXISTS ${testTable}")
}
}