database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Fix](json reader) fix rapidjson array->PushBack may take ownership… (#21988)

Browse Source 
With bellow json path
`["$.data","$.data.datatimestamp"]`

After `array_obj->PushBack` the `data` field owner will be taken from array_obj, and lead to null values for json path `$.data.datatimestamp`

Rapidjson doc:
```
//! Append a GenericValue at the end of the array.
  \note The ownership of \c value will be transferred to this array on success.
 */
GenericValue& PushBack(GenericValue& value, Allocator& allocator);
```
This commit is contained in:
lihangyu
2023-07-21 17:02:01 +08:00
committed by GitHub
parent d1c5025bce
commit 40299d280d
4 changed files with 35 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
be/src/exprs/json_functions.cpp
View File

@ -172,11 +172,13 @@ rapidjson::Value* JsonFunctions::get_json_array_from_parsed_json(
rapidjson::Value* root = match_value(parsed_paths, document, mem_allocator, true);
if (root == nullptr || root == document) { // not found
return nullptr;
} else if (!root->IsArray()) {
} else if (!root->IsArray() && wrap_explicitly) {
rapidjson::Value* array_obj = nullptr;
array_obj = static_cast<rapidjson::Value*>(mem_allocator.Malloc(sizeof(rapidjson::Value)));
array_obj->SetArray();
array_obj->PushBack(*root, mem_allocator);
rapidjson::Value copy;
copy.CopyFrom(*root, mem_allocator);
array_obj->PushBack(std::move(copy), mem_allocator);
// set `wrap_explicitly` to true, so that the caller knows that this Array is wrapped actively.
*wrap_explicitly = true;
return array_obj;

3
regression-test/data/load_p0/stream_load/test_json_load.out
View File

@ -202,3 +202,6 @@
-- !select1 --
John 30 New York {"email":"john@example.com","phone":"+1-123-456-7890"}
-- !select22 --
11324 1321313082437 1678834024274 20230315 {"base_mac_value_null":24,"base_1_value_respiratoryrate":11,"base_3_value_heartrate":51,"base_3_status_onoroutofbed":3,"base_null_count_circulation":84,"base_1_status_onoroutofbed":3,"base_1_value_heartrate":51,"base_3_value_respiratoryrate":11,"base_3_value_bodyactivityenergy":43652,"base_2_value_respiratoryrate":11,"base_2_value_bodyactivityenergy":28831,"base_2_status_onoroutofbed":3,"base_1_value_bodyactivityenergy":56758,"base_2_value_heartrate":51,"tsltype":"properties","datatimestamp":1678834024274,"command":"0105","macaddress":"405EE1805029"}

1
regression-test/data/load_p0/stream_load/with_jsonpath.json Normal file
View File

@ -0,0 +1 @@
{"data":{"base_mac_value_null":24,"base_1_value_respiratoryrate":11,"base_3_value_heartrate":51,"base_3_status_onoroutofbed":3,"base_null_count_circulation":84,"base_1_status_onoroutofbed":3,"base_1_value_heartrate":51,"base_3_value_respiratoryrate":11,"base_3_value_bodyactivityenergy":43652,"base_2_value_respiratoryrate":11,"base_2_value_bodyactivityenergy":28831,"base_2_status_onoroutofbed":3,"base_1_value_bodyactivityenergy":56758,"base_2_value_heartrate":51,"tsltype":"properties","datatimestamp":1678834024274,"command":"0105","macaddress":"405EE1805029"},"deviceid":1321313082437,"productid":11324}

27
regression-test/suites/load_p0/stream_load/test_json_load.groovy
View File

@ -588,6 +588,33 @@ suite("test_json_load", "p0") {
try_sql("DROP TABLE IF EXISTS ${testTable}")
}
// case22: nested and it's member with jsonpath
try {
testTable = "test_json_load"
sql "DROP TABLE IF EXISTS ${testTable}"
sql """CREATE TABLE IF NOT EXISTS ${testTable}
(
`productid` bigint NOT NULL COMMENT "productid",
`deviceid` bigint NOT NULL COMMENT "deviceid",
`datatimestamp` string NULL COMMENT "datatimestamp",
`dt` int NULL COMMENT "dt",
`data` string
)
DUPLICATE KEY(`productid`, `deviceid`)
DISTRIBUTED BY RANDOM BUCKETS auto
properties(
"replication_num" = "1"
);
"""
1
load_json_data.call("${testTable}", 'with_jsonpath', '', 'true', 'json', """productid, deviceid, data, datatimestamp, dt=from_unixtime(substr(datatimestamp,1,10),'%Y%m%d')""",
'["$.productid","$.deviceid","$.data","$.data.datatimestamp"]', '', '', '', 'with_jsonpath.json')
qt_select22 "select * from ${testTable}"
} finally {
try_sql("DROP TABLE IF EXISTS ${testTable}")
}
// if 'enableHdfs' in regression-conf.groovy has been set to true,
// the test will run these case as below.
if (enableHdfs()) {