database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Bug] Load data is error in json load

Browse Source
This commit is contained in:
worker24h
2020-05-28 17:28:33 +08:00
committed by GitHub
parent 8f71c7a331
commit e76f712bb3
4 changed files with 19 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
be/src/exec/json_scanner.cpp
View File

@ -267,15 +267,11 @@ size_t JsonReader::get_data_by_jsonpath(const std::vector<SlotDescriptor*>& slot
}
// if jsonValues is null, because not match in jsondata.
rapidjson::Value* json_values = JsonFunctions::get_json_object_from_parsed_json(path.GetString(), &_json_doc, _json_doc.GetAllocator());
rapidjson::Value* json_values = JsonFunctions::get_json_array_from_parsed_json(path.GetString(), &_json_doc, _json_doc.GetAllocator());
if (json_values == nullptr) {
return -1;
}
if (json_values->IsArray()) {
max_lines = std::max(max_lines, (size_t)json_values->Size());
} else {
max_lines = std::max(max_lines, (size_t)1);
}
max_lines = std::max(max_lines, (size_t)json_values->Size());
_jmap.emplace(slot_descs[i]->col_name(), json_values);
}
@ -376,7 +372,7 @@ Status JsonReader::set_tuple_value(rapidjson::Value& objectValue, Tuple* tuple,
/**
* handle input a simple json
* For example:
* case 1. {"RECORDS": [{"colunm1":"value1", "colunm2":10}, {"colunm1":"value2", "colunm2":30}]}
* case 1. [{"colunm1":"value1", "colunm2":10}, {"colunm1":"value2", "colunm2":30}]
* case 2. {"colunm1":"value1", "colunm2":10}
*/
Status JsonReader::handle_simple_json(Tuple* tuple, const std::vector<SlotDescriptor*>& slot_descs, MemPool* tuple_pool, bool* eof) {
@ -497,7 +493,7 @@ Status JsonReader::handle_flat_array_complex_json(Tuple* tuple, const std::vecto
}
// if jsonValues is null, because not match in jsondata.
rapidjson::Value* json_values = JsonFunctions::get_json_object_from_parsed_json(path.GetString(), &objectValue, _json_doc.GetAllocator());
rapidjson::Value* json_values = JsonFunctions::get_json_array_from_parsed_json(path.GetString(), &objectValue, _json_doc.GetAllocator());
if (json_values == nullptr) {
if (slot_descs[i]->is_nullable()) {
tuple->set_null(slot_descs[i]->null_indicator_offset());

9
be/src/exprs/json_functions.cpp
View File

@ -263,7 +263,7 @@ rapidjson::Value* JsonFunctions::get_json_object(
}
rapidjson::Value* JsonFunctions::get_json_object_from_parsed_json (
rapidjson::Value* JsonFunctions::get_json_array_from_parsed_json (
const std::string& path_string,
rapidjson::Value* document,
rapidjson::Document::AllocatorType& mem_allocator) {
@ -293,6 +293,13 @@ rapidjson::Value* JsonFunctions::get_json_object_from_parsed_json (
rapidjson::Value* root = match_value(parsed_paths, document, mem_allocator, true);
if (root == document) {// not found
return nullptr;
} else if (!root->IsArray()) {
rapidjson::Value* array_obj = nullptr;
array_obj = static_cast<rapidjson::Value*>(
mem_allocator.Malloc(sizeof(rapidjson::Value)));
array_obj->SetArray();
array_obj->PushBack(*root, mem_allocator);
return array_obj;
}
return root;
}

3
be/src/exprs/json_functions.h
View File

@ -72,8 +72,9 @@ public:
/**
* The `document` parameter must be has parsed.
* return Value Is Array object
*/
static rapidjson::Value* get_json_object_from_parsed_json(
static rapidjson::Value* get_json_array_from_parsed_json(
const std::string& path_string,
rapidjson::Value* document,
rapidjson::Document::AllocatorType& mem_allocator);

10
be/test/exprs/json_function_test.cpp
View File

@ -194,13 +194,13 @@ TEST_F(JsonFunctionTest, json_path1)
ASSERT_TRUE(false);
}
rapidjson::Value* res3;
res3 = JsonFunctions::get_json_object_from_parsed_json("$.[*].keyname.ip", &jsonDoc, jsonDoc.GetAllocator());
res3 = JsonFunctions::get_json_array_from_parsed_json("$.[*].keyname.ip", &jsonDoc, jsonDoc.GetAllocator());
ASSERT_TRUE(res3->IsArray());
for (int i = 0; i < res3->Size(); i++) {
std::cout<< (*res3)[i].GetString() << std::endl;
}
res3 = JsonFunctions::get_json_object_from_parsed_json("$.[*].k1", &jsonDoc, jsonDoc.GetAllocator());
res3 = JsonFunctions::get_json_array_from_parsed_json("$.[*].k1", &jsonDoc, jsonDoc.GetAllocator());
ASSERT_TRUE(res3->IsArray());
for (int i = 0; i < res3->Size(); i++) {
std::cout<< (*res3)[i].GetString() << std::endl;
@ -216,7 +216,7 @@ TEST_F(JsonFunctionTest, json_path_get_nullobject)
ASSERT_TRUE(false);
}
rapidjson::Value* res3 = JsonFunctions::get_json_object_from_parsed_json("$.[*].b", &jsonDoc, jsonDoc.GetAllocator());
rapidjson::Value* res3 = JsonFunctions::get_json_array_from_parsed_json("$.[*].b", &jsonDoc, jsonDoc.GetAllocator());
ASSERT_TRUE(res3->IsArray());
ASSERT_EQ(res3->Size(), 3);
for (int i = 0; i < res3->Size(); i++) {
@ -239,7 +239,7 @@ TEST_F(JsonFunctionTest, json_path_test)
ASSERT_TRUE(false);
}
rapidjson::Value* res3 = JsonFunctions::get_json_object_from_parsed_json("$.[*].a", &jsonDoc, jsonDoc.GetAllocator());
rapidjson::Value* res3 = JsonFunctions::get_json_array_from_parsed_json("$.[*].a", &jsonDoc, jsonDoc.GetAllocator());
ASSERT_TRUE(res3->IsArray());
ASSERT_EQ(res3->Size(), 2);
for (int i = 0; i < res3->Size(); i++) {
@ -258,7 +258,7 @@ TEST_F(JsonFunctionTest, json_path_test)
ASSERT_TRUE(false);
}
rapidjson::Value* res3 = JsonFunctions::get_json_object_from_parsed_json("$.a", &jsonDoc, jsonDoc.GetAllocator());
rapidjson::Value* res3 = JsonFunctions::get_json_array_from_parsed_json("$.a", &jsonDoc, jsonDoc.GetAllocator());
ASSERT_TRUE(res3->IsArray());
ASSERT_EQ(res3->Size(), 2);
for (int i = 0; i < res3->Size(); i++) {