hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[patch 421->425->434] [json depth config] json depth config set [100, 1024]

Browse Source
This commit is contained in:
obdev 2024-09-20 07:16:36 +00:00 committed by ob-robot
parent 3c2667cf2e
commit 9929e8cc82
28 changed files with 152 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
deps/oblib/src/lib/json_type/ob_json_base.cpp vendored
View File

@ -3881,20 +3881,16 @@ int ObIJsonBase::print(ObJsonBuffer &j_buf, bool is_quoted, bool is_pretty, uint
}
case ObJsonNodeType::J_ARRAY: {
if (ObJsonParser::is_json_doc_over_depth(++depth)) {
ret = OB_ERR_JSON_OUT_OF_DEPTH;
LOG_WARN("current json over depth", K(ret), K(depth), K(j_type));
} else if (OB_FAIL(print_array(j_buf, depth, is_pretty))) {
++depth;
if (OB_FAIL(SMART_CALL(print_array(j_buf, depth, is_pretty)))) {
LOG_WARN("fail to change jarray to string", K(ret), K(is_quoted), K(is_pretty), K(depth));
}
break;
}
case ObJsonNodeType::J_OBJECT: {
if (ObJsonParser::is_json_doc_over_depth(++depth)) {
ret = OB_ERR_JSON_OUT_OF_DEPTH;
LOG_WARN("current json over depth", K(ret), K(depth), K(j_type));
} else if (OB_FAIL(print_object(j_buf, depth, is_pretty))) {
++depth;
if (OB_FAIL(SMART_CALL(print_object(j_buf, depth, is_pretty)))) {
LOG_WARN("fail to print object to string", K(ret), K(depth), K(j_type), K(is_pretty));
}
break;
@ -5953,14 +5949,16 @@ int ObIJsonBase::to_bit(uint64_t &value) const
int ObJsonBaseFactory::get_json_base(ObIAllocator *allocator, const ObString &buf,
ObJsonInType in_type, ObJsonInType expect_type,
ObIJsonBase *&out, uint32_t parse_flag)
ObIJsonBase *&out, uint32_t parse_flag,
uint32_t max_depth_config)
{
return get_json_base(allocator, buf.ptr(), buf.length(), in_type, expect_type, out, parse_flag);
return get_json_base(allocator, buf.ptr(), buf.length(), in_type, expect_type, out, parse_flag, max_depth_config);
}
int ObJsonBaseFactory::get_json_base(ObIAllocator *allocator, const char *ptr, uint64_t length,
ObJsonInType in_type, ObJsonInType expect_type,
ObIJsonBase *&out, uint32_t parse_flag)
ObIJsonBase *&out, uint32_t parse_flag,
uint32_t max_depth_config)
{
INIT_SUCC(ret);
void *buf = NULL;
@ -5991,7 +5989,7 @@ int ObJsonBaseFactory::get_json_base(ObIAllocator *allocator, const char *ptr, u
LOG_WARN("param expect_type is invalid", K(ret), K(expect_type));
} else if (in_type == ObJsonInType::JSON_TREE) {
ObJsonNode *j_tree = NULL;
if (OB_FAIL(ObJsonParser::get_tree(t_allocator, ptr, length, j_tree, parse_flag))) {
if (OB_FAIL(ObJsonParser::get_tree(t_allocator, ptr, length, j_tree, parse_flag, max_depth_config))) {
LOG_WARN("fail to get json tree", K(ret), K(length), K(in_type), K(expect_type));
} else if (expect_type == ObJsonInType::JSON_TREE) {
out = j_tree;
@ -6280,10 +6278,7 @@ int ObJsonBaseUtil::append_newline_and_indent(ObJsonBuffer &j_buf, uint64_t leve
// Append newline and two spaces per indentation level.
INIT_SUCC(ret);
if (level > ObJsonParser::JSON_DOCUMENT_MAX_DEPTH) {
ret = OB_ERR_JSON_OUT_OF_DEPTH;
LOG_WARN("indent level is too deep", K(ret), K(level));
} else if (OB_FAIL(j_buf.append("\n"))) {
if (OB_FAIL(j_buf.append("\n"))) {
LOG_WARN("fail to append newline to buffer", K(ret), K(level));
} else if (OB_FAIL(j_buf.reserve(level * 2))) {
LOG_WARN("fail to reserve memory for buffer", K(ret), K(level));

6
deps/oblib/src/lib/json_type/ob_json_base.h vendored
View File

@ -1072,12 +1072,14 @@ public:
virtual ~ObJsonBaseFactory() {};
static int get_json_base(ObIAllocator *allocator, const ObString &buf,
ObJsonInType in_type, ObJsonInType expect_type,
ObIJsonBase *&out, uint32_t parse_flag = 0);
ObIJsonBase *&out, uint32_t parse_flag = 0,
uint32_t max_depth_config = 100);
static int get_json_tree(ObIAllocator *allocator, const ObString &str,
ObJsonInType in_type, ObJsonNode *&out, uint32_t parse_flag = 0);
static int get_json_base(ObIAllocator *allocator, const char *ptr, uint64_t length,
ObJsonInType in_type, ObJsonInType expect_type,
ObIJsonBase *&out, uint32_t parse_flag = 0);
ObIJsonBase *&out, uint32_t parse_flag = 0,
uint32_t max_depth_config = 100);
static int transform(ObIAllocator *allocator, ObIJsonBase *src,
ObJsonInType expect_type, ObIJsonBase *&out);
private:

2
deps/oblib/src/lib/json_type/ob_json_bin.cpp vendored
View File

@ -1458,7 +1458,7 @@ int ObJsonBin::deserialize_json_object_v0(ObJsonObject *object)
LOG_WARN("ob_write_string fail", K(ret), K(i), K(ori_key));
} else if (OB_FAIL(get_value(i, child_bin))) {
LOG_WARN("get child value fail", K(ret));
} else if (OB_FAIL(SMART_CALL(child_bin.deserialize_json_value(node)))) {
} else if (OB_FAIL(child_bin.deserialize_json_value(node))) {
LOG_WARN("deserialize child node fail", K(ret), K(i), K(child_bin));
} else if (OB_FAIL(object->add(key, node, false, true, false, is_schema_))) {
LOG_WARN("add node to obj fail", K(ret), K(i));

35
deps/oblib/src/lib/json_type/ob_json_parse.cpp vendored
View File

@ -13,6 +13,7 @@
#define USING_LOG_PREFIX SQL
#include "ob_json_parse.h"
#include "observer/omt/ob_tenant_config_mgr.h"
namespace oceanbase {
namespace common {
@ -26,7 +27,7 @@ namespace common {
#define STRICTJSON_FLAG rapidjson::kParseObjectKeyNoQuotesFlag
int ObJsonParser::get_tree(ObIAllocator *allocator, const ObString &text, ObJsonNode *&j_tree,
uint32_t parse_flag)
uint32_t parse_flag, uint32_t max_depth_config)
{
INIT_SUCC(ret);
char buf[PARSE_SYNTAXERR_MESSAGE_LENGTH] = {0};
@ -34,7 +35,7 @@ int ObJsonParser::get_tree(ObIAllocator *allocator, const ObString &text, ObJson
uint64_t offset = 0;
if (OB_FAIL(parse_json_text(allocator, text.ptr(), text.length(),
syntaxerr, &offset, j_tree, parse_flag))) {
syntaxerr, &offset, j_tree, parse_flag, max_depth_config))) {
LOG_WARN("fail to parse json text", K(ret), K(text), KCSTRING(syntaxerr));
}
@ -43,7 +44,8 @@ int ObJsonParser::get_tree(ObIAllocator *allocator, const ObString &text, ObJson
int ObJsonParser::get_tree(ObIAllocator *allocator, const char *text,
uint64_t length, ObJsonNode *&j_tree,
uint32_t parse_flag)
uint32_t parse_flag,
uint32_t max_depth_config)
{
INIT_SUCC(ret);
char buf[PARSE_SYNTAXERR_MESSAGE_LENGTH] = {0};
@ -52,7 +54,7 @@ int ObJsonParser::get_tree(ObIAllocator *allocator, const char *text,
if (OB_FAIL(parse_json_text(allocator, text, length,
syntaxerr, &offset, j_tree,
parse_flag))) {
parse_flag, max_depth_config))) {
LOG_WARN("fail to parse json text", K(ret), K(length), KCSTRING(syntaxerr));
}
@ -66,7 +68,8 @@ int ObJsonParser::get_tree(ObIAllocator *allocator, const char *text,
int ObJsonParser::parse_json_text(ObIAllocator *allocator,
const char *text, uint64_t length,
const char *&syntaxerr, uint64_t *offset,
ObJsonNode *&j_tree, uint32_t parse_flag)
ObJsonNode *&j_tree, uint32_t parse_flag,
uint32_t max_depth_config)
{
INIT_SUCC(ret);
@ -83,7 +86,7 @@ int ObJsonParser::parse_json_text(ObIAllocator *allocator,
bool with_unique_key = HAS_FLAG(parse_flag, JSN_UNIQUE_FLAG);
bool is_schema = HAS_FLAG(parse_flag, JSN_SCHEMA_FLAG);
bool preserve_dup = HAS_FLAG(parse_flag, JSN_PRESERVE_DUP_FLAG);
ObRapidJsonHandler handler(allocator, with_unique_key, is_schema, preserve_dup);
ObRapidJsonHandler handler(allocator, with_unique_key, is_schema, preserve_dup, max_depth_config);
ObRapidJsonAllocator parse_allocator(allocator);
rapidjson::InsituStringStream ss(static_cast<char *>(buf));
ObRapidJsonReader reader(&parse_allocator);
@ -135,19 +138,8 @@ int ObJsonParser::parse_json_text(ObIAllocator *allocator,
return ret;
}
bool ObJsonParser::is_json_doc_over_depth(uint64_t depth)
{
bool is_over = false;
if (depth > JSON_DOCUMENT_MAX_DEPTH) {
is_over = true;
}
return is_over;
}
int ObJsonParser::check_json_syntax(const ObString &j_doc, ObIAllocator *allocator,
uint32_t parse_flag)
uint32_t parse_flag, uint32_t max_depth_config)
{
INIT_SUCC(ret);
char syntax_buf[PARSE_SYNTAXERR_MESSAGE_LENGTH] = {0};
@ -168,7 +160,7 @@ int ObJsonParser::check_json_syntax(const ObString &j_doc, ObIAllocator *allocat
MEMCPY(alloc_buf, j_doc.ptr(), length);
alloc_buf[length] = '\0';
if (!HAS_FLAG(parse_flag, JSN_UNIQUE_FLAG)) {
ObJsonSyntaxCheckHandler handler(allocator);
ObJsonSyntaxCheckHandler handler(allocator, max_depth_config);
ObRapidJsonAllocator parse_allocator(allocator);
rapidjson::InsituStringStream ss(static_cast<char *>(alloc_buf));
ObRapidJsonReader reader(&parse_allocator);
@ -200,7 +192,7 @@ int ObJsonParser::check_json_syntax(const ObString &j_doc, ObIAllocator *allocat
LOG_DEBUG("fail to parse json text", K(ret), K(r.Code()), KCSTRING(syntaxerr), K(offset));
}
} else {
ObRapidJsonHandler handler(allocator, true);
ObRapidJsonHandler handler(allocator, true, false, false, max_depth_config);
ObRapidJsonAllocator parse_allocator(allocator);
rapidjson::InsituStringStream ss(static_cast<char *>(alloc_buf));
ObRapidJsonReader reader(&parse_allocator);
@ -299,7 +291,7 @@ bool ObRapidJsonHandler::is_start_object_or_array(ObJsonNode *value, ObJsonExpec
{
bool is_continue = false;
if (ObJsonParser::is_json_doc_over_depth(depth_)) {
if (depth_ > config_json_max_depth_) {
LOG_WARN_RET(OB_ERR_UNEXPECTED, "current json doc is over depth", K(OB_ERR_JSON_OUT_OF_DEPTH));
} else {
depth_++;
@ -600,6 +592,7 @@ bool ObRapidJsonHandler::Key(const char *str, rapidjson::SizeType length, bool c
return is_continue;
}
#undef TEST_RELAXJSON_FLAG
} // namespace common

41
deps/oblib/src/lib/json_type/ob_json_parse.h vendored
View File

@ -29,6 +29,7 @@ namespace common {
(flags) |= (specific); \
}
const int JSON_DOCUMENT_MAX_DEPTH = 100;
class ObJsonParser final
{
public:
@ -41,9 +42,11 @@ public:
static const int PARSE_SYNTAXERR_MESSAGE_LENGTH = 256;
static int get_tree(ObIAllocator *allocator, const ObString &text,
ObJsonNode *&j_tree, uint32_t parse_flag = 0);
ObJsonNode *&j_tree, uint32_t parse_flag = 0,
uint32_t max_depth_config = JSON_DOCUMENT_MAX_DEPTH);
static int get_tree(ObIAllocator *allocator, const char *text, uint64_t length,
ObJsonNode *&j_tree, uint32_t parse_flag = 0);
ObJsonNode *&j_tree, uint32_t parse_flag = 0,
uint32_t max_depth_config = JSON_DOCUMENT_MAX_DEPTH);
// Parse json text to json tree with rapidjson.
//
@ -57,16 +60,9 @@ public:
static int parse_json_text(ObIAllocator *allocator,
const char *text, uint64_t length,
const char *&syntaxerr, uint64_t *offset,
ObJsonNode *&j_tree, uint32_t parse_flag = 0);
ObJsonNode *&j_tree, uint32_t parse_flag = 0,
uint32_t max_depth_config = JSON_DOCUMENT_MAX_DEPTH);
// The tree has a maximum depth of 100 layers
static constexpr int JSON_DOCUMENT_MAX_DEPTH = 100;
// Verify that the current JSON tree depth exceeds the maximum depth
//
// @param [in] depth Current json tree depth.
// @return Returns true beyond the maximum depth, false otherwise.
static bool is_json_doc_over_depth(uint64_t depth);
// Check json document syntax.(for json_valid)
//
@ -74,7 +70,8 @@ public:
// @param [in] allocator Alloc memory In the parsing process.
// @return Returns OB_SUCCESS on success, error code otherwise.
static int check_json_syntax(const ObString &j_doc, ObIAllocator *allocator = NULL,
uint32_t parse_flag = 0);
uint32_t parse_flag = 0,
uint32_t max_depth_config = JSON_DOCUMENT_MAX_DEPTH);
private:
DISALLOW_COPY_AND_ASSIGN(ObJsonParser);
};
@ -129,7 +126,7 @@ public:
EXPECT_OBJECT_VALUE,
EXPECT_EOF
};
explicit ObRapidJsonHandler(ObIAllocator *allocator, bool with_unique_key = false, bool is_schema = false, bool preserve_dup_key = false)
explicit ObRapidJsonHandler(ObIAllocator *allocator, bool with_unique_key = false, bool is_schema = false, bool preserve_dup_key = false, uint32_t json_depth_config = JSON_DOCUMENT_MAX_DEPTH)
: next_state_(ObJsonExpectNextState::EXPECT_ANYTHING),
dom_as_built_(NULL),
current_element_(NULL),
@ -140,7 +137,8 @@ public:
with_unique_key_(with_unique_key),
with_duplicate_key_(false),
is_schema_(is_schema),
preserve_dup_key_(preserve_dup_key)
preserve_dup_key_(preserve_dup_key),
config_json_max_depth_(json_depth_config < JSON_DOCUMENT_MAX_DEPTH ? JSON_DOCUMENT_MAX_DEPTH : json_depth_config)
{
}
virtual ~ObRapidJsonHandler() {}
@ -183,6 +181,8 @@ public:
bool Key(const char *str, rapidjson::SizeType length, bool copy);
bool has_duplicate_key() { return with_duplicate_key_; }
int get_error_code() { return err_code_; }
uint32_t get_json_max_depth_config();
bool is_json_doc_over_depth();
private:
ObJsonExpectNextState next_state_; // The state that is expected to be resolved next.
@ -196,6 +196,7 @@ private:
bool with_duplicate_key_; // Whether contain duplicate key for object
bool is_schema_; // is json schema text
bool preserve_dup_key_; // preserve duplicate key
uint32_t config_json_max_depth_;
DISALLOW_COPY_AND_ASSIGN(ObRapidJsonHandler);
};
@ -203,16 +204,18 @@ private:
class ObJsonSyntaxCheckHandler final : public rapidjson::BaseReaderHandler<>
{
public:
explicit ObJsonSyntaxCheckHandler(ObIAllocator *allocator)
explicit ObJsonSyntaxCheckHandler(ObIAllocator *allocator, uint32_t json_depth_config = JSON_DOCUMENT_MAX_DEPTH)
: allocator_(allocator),
_depth(0),
_is_too_deep(false)
_is_too_deep(false),
_config_json_max_depth(json_depth_config > JSON_DOCUMENT_MAX_DEPTH ? json_depth_config : JSON_DOCUMENT_MAX_DEPTH)
{
}
virtual ~ObJsonSyntaxCheckHandler() {}
OB_INLINE bool StartObject()
{
_is_too_deep = ObJsonParser::is_json_doc_over_depth(++_depth);
++_depth;
_is_too_deep = _depth > _config_json_max_depth;
return !_is_too_deep;
}
OB_INLINE bool EndObject(rapidjson::SizeType length)
@ -223,7 +226,8 @@ public:
}
OB_INLINE bool StartArray()
{
_is_too_deep = ObJsonParser::is_json_doc_over_depth(++_depth);
_depth++;
_is_too_deep = _depth > _config_json_max_depth;
return !_is_too_deep;
}
OB_INLINE bool EndArray(rapidjson::SizeType length)
@ -237,6 +241,7 @@ private:
ObIAllocator *allocator_;
uint64_t _depth;
bool _is_too_deep;
uint32_t _config_json_max_depth;
DISALLOW_COPY_AND_ASSIGN(ObJsonSyntaxCheckHandler);
};

6
src/share/object/ob_obj_cast.cpp
View File

@ -45,6 +45,8 @@
#ifdef OB_BUILD_ORACLE_PL
#include "pl/sys_package/ob_sdo_geometry.h"
#endif
#include "sql/engine/expr/ob_expr_json_func_helper.h"
#include "lib/xml/ob_xml_util.h"
#include "lib/xml/ob_xml_parser.h"
@ -6118,7 +6120,9 @@ static int string_json(const ObObjType expect_type, ObObjCastParams &params,
if ((CM_IS_SQL_AS_JSON_SCALAR(cast_mode) && ob_is_string_type(in_type)) && j_text.compare("null") == 0) {
j_base = &j_null;
}
} else if (OB_FAIL(ObJsonParser::get_tree(params.allocator_v2_, j_text, j_tree, parse_flag))) {
} else if (OB_FAIL(ObJsonParser::get_tree(params.allocator_v2_, j_text,
j_tree, parse_flag,
sql::ObJsonExprHelper::get_json_max_depth_config()))) {
if (!is_oracle && CM_IS_IMPLICIT_CAST(cast_mode)
&& !CM_IS_COLUMN_CONVERT(cast_mode)
&& is_convert_jstr_type) {

3
src/share/parameter/ob_parameter_seed.ipp
View File

@ -2052,6 +2052,9 @@ DEF_BOOL(_preserve_order_for_pagination, OB_TENANT_PARAMETER, "False",
DEF_INT(max_partition_num, OB_TENANT_PARAMETER, "8192", "[8192, 65536]",
"set max partition num in mysql mode",
ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
DEF_INT(json_document_max_depth, OB_TENANT_PARAMETER, "100", "[100,1024]",
"maximum nesting depth allowed in a JSON document",
ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
ERRSIM_DEF_INT(errsim_backup_task_batch_size, OB_CLUSTER_PARAMETER, "0", "[0,)",
"the batch size backup task receive in errsim mode"
"Range: [0,) in integer",

6
src/sql/engine/aggregate/ob_aggregate_processor.cpp
View File

@ -7824,7 +7824,8 @@ int ObAggregateProcessor::get_ora_json_arrayagg_result(const ObAggrInfo &aggr_in
buff->string(),
ObJsonInType::JSON_BIN,
ObJsonInType::JSON_BIN,
j_base))) {
j_base, 0,
ObJsonExprHelper::get_json_max_depth_config()))) {
LOG_WARN("fail to get real data.", K(ret), K(buff));
} else if (OB_FAIL(j_base->print(string_buffer, true, false))) {
LOG_WARN("failed: get json string text", K(ret));
@ -8435,7 +8436,8 @@ int ObAggregateProcessor::get_ora_json_objectagg_result(const ObAggrInfo &aggr_i
buff->string(),
ObJsonInType::JSON_BIN,
ObJsonInType::JSON_BIN,
j_base))) {
j_base, 0,
ObJsonExprHelper::get_json_max_depth_config()))) {
LOG_WARN("fail to get real data.", K(ret), K(buff));
} else if (OB_FAIL(j_base->print(string_buffer, true, false))) {
LOG_WARN("failed: get json string text", K(ret));

2
src/sql/engine/basic/ob_json_table_op.cpp
View File

@ -2797,7 +2797,7 @@ int JsonTableFunc::eval_input(ObJsonTableOp &jt, JtScanCtx& ctx, ObEvalCtx &eval
if (OB_FAIL(ret)) {
} else if (OB_FAIL(ObJsonBaseFactory::get_json_base(&ctx.row_alloc_, j_str, j_in_type, expect_type, in, parse_flag))
} else if (OB_FAIL(ObJsonBaseFactory::get_json_base(&ctx.row_alloc_, j_str, j_in_type, expect_type, in, parse_flag, ObJsonExprHelper::get_json_max_depth_config()))
|| (in->json_type() != ObJsonNodeType::J_ARRAY && in->json_type() != ObJsonNodeType::J_OBJECT)) {
if (OB_FAIL(ret) || (is_ensure_json)) {
in= nullptr;

6
src/sql/engine/basic/ob_select_into_op.cpp
View File

@ -1729,7 +1729,8 @@ int ObSelectIntoOp::set_odps_column_value_mysql(apsara::odps::sdk::ODPSTableReco
&ctx_))) {
LOG_WARN("failed to read string", K(ret));
} else if (OB_FAIL(ObJsonBaseFactory::get_json_base(&allocator, json_str, in_type,
in_type, j_base, parse_flag))) {
in_type, j_base, parse_flag,
ObJsonExprHelper::get_json_max_depth_config()))) {
COMMON_LOG(WARN, "fail to get json base", K(ret), K(in_type));
} else if (OB_FAIL(j_base->print(jbuf, false))) { // json binary to string
COMMON_LOG(WARN, "fail to convert json to string", K(ret));
@ -2001,7 +2002,8 @@ int ObSelectIntoOp::set_odps_column_value_oracle(apsara::odps::sdk::ODPSTableRec
&ctx_))) {
LOG_WARN("failed to read string", K(ret));
} else if (OB_FAIL(ObJsonBaseFactory::get_json_base(&allocator, json_str, in_type,
in_type, j_base, parse_flag))) {
in_type, j_base, parse_flag,
ObJsonExprHelper::get_json_max_depth_config()))) {
COMMON_LOG(WARN, "fail to get json base", K(ret), K(in_type));
} else if (OB_FAIL(j_base->print(jbuf, false))) { // json binary to string
COMMON_LOG(WARN, "fail to convert json to string", K(ret));

4
src/sql/engine/expr/ob_datum_cast.cpp
View File

@ -4014,7 +4014,9 @@ static int common_string_json(const ObExpr &expr,
}
} else if (is_oracle && (OB_ISNULL(j_text.ptr()) || j_text.length() == 0)) {
j_base = &j_null;
} else if (OB_FAIL(ObJsonParser::get_tree(&temp_allocator, j_text, j_tree, parse_flag))) {
} else if (OB_FAIL(ObJsonParser::get_tree(&temp_allocator, j_text, j_tree,
parse_flag,
ObJsonExprHelper::get_json_max_depth_config()))) {
if (!is_oracle && CM_IS_IMPLICIT_CAST(expr.extra_) && !CM_IS_COLUMN_CONVERT(expr.extra_)) {
ret = OB_SUCCESS;
j_base = &j_string;

3
src/sql/engine/expr/ob_expr_is_json.cpp
View File

@ -113,7 +113,8 @@ int ObExprIsJson::check_is_json(const ObExpr &expr, ObEvalCtx &ctx,
ADD_FLAG_IF_NEED(strict_opt != OB_JSON_MODE_STRICT, parse_flag, ObJsonParser::JSN_RELAXED_FLAG);
ADD_FLAG_IF_NEED(unique_opt == OB_JSON_MODE_UNIQUE_KEYS, parse_flag, ObJsonParser::JSN_UNIQUE_FLAG);
if (OB_FAIL(ObJsonParser::check_json_syntax(j_str, &allocator, parse_flag))) {
if (OB_FAIL(ObJsonParser::check_json_syntax(j_str, &allocator, parse_flag,
ObJsonExprHelper::get_json_max_depth_config()))) {
LOG_WARN("fail to check json syntax", K(ret), K(type), K(j_str));
}
}

4
src/sql/engine/expr/ob_expr_json_array.cpp
View File

@ -211,7 +211,7 @@ int ObExprJsonArray::eval_ora_json_array(const ObExpr &expr, ObEvalCtx &ctx, ObD
ObJsonBuffer string_buffer(&temp_allocator);
ObString res_string;
if (ObJsonParser::is_json_doc_over_depth(j_arr.depth())) {
if (ObJsonExprHelper::is_json_depth_exceed_limit(j_arr.depth())) {
ret = OB_ERR_JSON_OUT_OF_DEPTH;
LOG_WARN("current json over depth", K(ret), K(j_arr.depth()));
} else if (dst_type == ObJsonType) {
@ -282,7 +282,7 @@ int ObExprJsonArray::eval_json_array(const ObExpr &expr, ObEvalCtx &ctx, ObDatum
if (OB_SUCC(ret)) {
ObString raw_bin;
if (ObJsonParser::is_json_doc_over_depth(j_arr.depth())) {
if (ObJsonExprHelper::is_json_depth_exceed_limit(j_arr.depth())) {
ret = OB_ERR_JSON_OUT_OF_DEPTH;
LOG_WARN("current json over depth", K(ret), K(j_arr.depth()));
} else if (OB_FAIL(ObJsonWrapper::get_raw_binary(j_base, raw_bin, &temp_allocator))) {

2
src/sql/engine/expr/ob_expr_json_extract.cpp
View File

@ -124,7 +124,7 @@ int ObExprJsonExtract::eval_json_extract(const ObExpr &expr, ObEvalCtx &ctx, ObD
ObJsonInType j_in_type = ObJsonExprHelper::get_json_internal_type(val_type);
if (OB_FAIL(ObJsonExprHelper::get_json_or_str_data(json_arg, ctx, allocator, j_str, is_null_result))) {
LOG_WARN("fail to get real data.", K(ret), K(j_str));
} else if (OB_FAIL(ObJsonBaseFactory::get_json_base(&allocator, j_str, j_in_type, j_in_type, j_base))) {
} else if (OB_FAIL(ObJsonBaseFactory::get_json_base(&allocator, j_str, j_in_type, j_in_type, j_base, 0, ObJsonExprHelper::get_json_max_depth_config()))) {
LOG_WARN("fail to get json base", K(ret), K(j_in_type));
ret = OB_ERR_INVALID_JSON_TEXT;
}

40
src/sql/engine/expr/ob_expr_json_func_helper.cpp
View File

@ -199,7 +199,7 @@ int ObJsonExprHelper::get_json_doc(const ObExpr &expr, ObEvalCtx &ctx,
if (is_oracle && j_str.length() == 0) {
is_null = true;
} else if (OB_FAIL(ObJsonBaseFactory::get_json_base(&allocator, j_str, j_in_type,
expect_type, j_base, parse_flag))) {
expect_type, j_base, parse_flag, ObJsonExprHelper::get_json_max_depth_config()))) {
LOG_WARN("fail to get json base", K(ret), K(j_in_type));
if (is_oracle) {
ret = OB_ERR_JSON_SYNTAX_ERROR;
@ -357,7 +357,7 @@ int ObJsonExprHelper::get_json_for_partial_update(
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(ObJsonBaseFactory::get_json_base(
&allocator, j_str, ObJsonInType::JSON_BIN, ObJsonInType::JSON_TREE, j_base, 0))) {
&allocator, j_str, ObJsonInType::JSON_BIN, ObJsonInType::JSON_TREE, j_base, 0, ObJsonExprHelper::get_json_max_depth_config()))) {
LOG_WARN("get json base fail", K(ret), K(j_str));
}
cursor->~ObLobCursor();
@ -512,11 +512,11 @@ int ObJsonExprHelper::get_json_val(const common::ObDatum &data,
return ret;
}
int ObJsonExprHelper::cast_to_json_tree(ObString &text, common::ObIAllocator *allocator, uint32_t parse_flag)
int ObJsonExprHelper::cast_to_json_tree(ObString &text, common::ObIAllocator *allocator, uint32_t parse_flag, uint32_t max_depth_config)
{
INIT_SUCC(ret);
ObJsonNode *j_tree = NULL;
if (OB_FAIL(ObJsonParser::get_tree(allocator, text, j_tree, parse_flag))) {
if (OB_FAIL(ObJsonParser::get_tree(allocator, text, j_tree, parse_flag, max_depth_config))) {
LOG_WARN("get json tree fail", K(ret));
} else {
ObJsonBuffer jbuf(allocator);
@ -808,7 +808,7 @@ int ObJsonExprHelper::oracle_datum2_json_val(const ObDatum *json_datum,
j_base = string_node;
}
} else {
if (OB_FAIL(ObJsonParser::check_json_syntax(j_str, allocator, parse_flag))) {
if (OB_FAIL(ObJsonParser::check_json_syntax(j_str, allocator, parse_flag, get_json_max_depth_config()))) {
if (!is_strict) {
ret = OB_SUCCESS;
ObJsonString* string_node = nullptr;
@ -822,7 +822,9 @@ int ObJsonExprHelper::oracle_datum2_json_val(const ObDatum *json_datum,
LOG_WARN("fail to check json syntax", K(ret), K(j_str));
}
} else if (OB_FAIL(ObJsonBaseFactory::get_json_base( allocator, j_str,
ObJsonInType::JSON_TREE, ObJsonInType::JSON_TREE, j_base, parse_flag))) {
ObJsonInType::JSON_TREE, ObJsonInType::JSON_TREE,
j_base, parse_flag,
ObJsonExprHelper::get_json_max_depth_config()))) {
LOG_WARN("failed: parse json string node", K(ret), K(j_str));
}
}
@ -834,7 +836,9 @@ int ObJsonExprHelper::oracle_datum2_json_val(const ObDatum *json_datum,
LOG_WARN("fail to get real data.", K(ret), K(j_str));
} else if (OB_FAIL(deep_copy_ob_string(*allocator, j_str, j_str))) {
LOG_WARN("fail to deep copy string.", K(ret), K(j_str));
} else if (OB_FAIL(ObJsonBaseFactory::get_json_base(allocator, j_str, ObJsonInType::JSON_BIN, to_type, j_base))) {
} else if (OB_FAIL(ObJsonBaseFactory::get_json_base(allocator, j_str, ObJsonInType::JSON_BIN,
to_type, j_base, 0,
ObJsonExprHelper::get_json_max_depth_config()))) {
ret = OB_ERR_INVALID_JSON_TEXT_IN_PARAM;
LOG_WARN("fail to get json base", K(ret));
}
@ -1699,8 +1703,9 @@ int ObJsonExprHelper::transform_convertible_2jsonBase(const T &datum,
if (OB_SUCC(ret)) {
if (flags.format_json_) {
uint32_t parse_flag = flags.relax_type_ ? ObJsonParser::JSN_RELAXED_FLAG : ObJsonParser::JSN_STRICT_FLAG;
uint32_t json_depth_config = ObJsonExprHelper::get_json_max_depth_config();
ADD_FLAG_IF_NEED(flags.is_schema_, parse_flag, ObJsonParser::JSN_SCHEMA_FLAG);
if(OB_FAIL(ObJsonExprHelper::cast_to_json_tree(j_str, allocator, parse_flag))) {
if(OB_FAIL(ObJsonExprHelper::cast_to_json_tree(j_str, allocator, parse_flag, json_depth_config))) {
if (flags.wrap_on_fail_) {
if (OB_ISNULL(buf = allocator->alloc(sizeof(ObJsonString)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
@ -1715,7 +1720,8 @@ int ObJsonExprHelper::transform_convertible_2jsonBase(const T &datum,
} else {
ObJsonInType to_type = flags.to_bin_ ? ObJsonInType::JSON_BIN : ObJsonInType::JSON_TREE;
if (OB_FAIL(ObJsonBaseFactory::get_json_base(allocator, j_str, ObJsonInType::JSON_TREE,
to_type, json_node, parse_flag))) {
to_type, json_node, parse_flag,
ObJsonExprHelper::get_json_max_depth_config()))) {
if (flags.is_schema_ && ret == OB_ERR_UNSUPPROTED_REF_IN_JSON_SCHEMA) {
} else {
ret = OB_ERR_INVALID_JSON_TEXT_IN_PARAM;
@ -1755,7 +1761,8 @@ int ObJsonExprHelper::transform_convertible_2jsonBase(const T &datum,
uint32_t parse_flag = flags.relax_type_ ? ObJsonParser::JSN_RELAXED_FLAG : ObJsonParser::JSN_STRICT_FLAG;
ADD_FLAG_IF_NEED(flags.is_schema_, parse_flag, ObJsonParser::JSN_SCHEMA_FLAG);
if (OB_FAIL(ObJsonBaseFactory::get_json_base(allocator, j_str, ObJsonInType::JSON_BIN,
to_type, json_node, parse_flag))) {
to_type, json_node, parse_flag,
ObJsonExprHelper::get_json_max_depth_config()))) {
if (flags.is_schema_ && ret == OB_ERR_UNSUPPROTED_REF_IN_JSON_SCHEMA) {
} else {
ret = OB_ERR_INVALID_JSON_TEXT_IN_PARAM;
@ -2529,6 +2536,19 @@ int ObJsonExprHelper::pre_default_value_check(ObObjType dst_type, ObString val_s
return ret;
}
int ObJsonExprHelper::get_json_max_depth_config()
{
uint32_t json_max_depth = JSON_DOCUMENT_MAX_DEPTH;
omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID()));
if (tenant_config.is_valid()) {
json_max_depth = tenant_config->json_document_max_depth;
if (json_max_depth < JSON_DOCUMENT_MAX_DEPTH || json_max_depth > 1024) {
json_max_depth = JSON_DOCUMENT_MAX_DEPTH;
}
}
return json_max_depth;
}
/********** ObJsonExprHelper for json partial update ****************/
int ObJsonExprHelper::pack_json_diff_res(
const ObExpr &expr,

10
src/sql/engine/expr/ob_expr_json_func_helper.h
View File

@ -246,7 +246,10 @@ public:
ObIJsonBase *&j_base);
static int cast_to_json_tree(ObString &text, common::ObIAllocator *allocator, uint32_t parse_flag = 0);
static int cast_to_json_tree(ObString &text,
common::ObIAllocator *allocator,
uint32_t parse_flag = 0,
uint32_t max_depth_config = JSON_DOCUMENT_MAX_DEPTH);
/*
get json value to JsonBase in static_typing_engine
@param[in] expr the input arguments
@ -475,6 +478,11 @@ public:
static int get_clause_opt(ObExpr *expr,
ObEvalCtx &ctx,
int8_t &type);
static bool is_json_depth_exceed_limit(uint32_t depth)
{
return depth > JSON_DOCUMENT_MAX_DEPTH && depth > get_json_max_depth_config();
}
static int32_t get_json_max_depth_config();
static int is_allow_partial_update(const ObExpr &expr, ObEvalCtx &ctx, const ObString &locator_str, bool &allow_partial_update);
static bool is_json_partial_update_mode(const ObExpr &expr);

3
src/sql/engine/expr/ob_expr_json_length.cpp
View File

@ -89,7 +89,8 @@ int ObExprJsonLength::calc(ObEvalCtx &ctx, const ObDatum &data1, ObDatumMeta met
} else if (OB_FAIL(ObTextStringHelper::read_real_string_data(*allocator, data1, meta1, has_lob_header1, j_doc))) {
LOG_WARN("fail to get real data.", K(ret), K(j_doc));
} else if (OB_FAIL(ObJsonBaseFactory::get_json_base(allocator, j_doc, j_in_type,
j_in_type, j_base))) {
j_in_type, j_base, 0,
ObJsonExprHelper::get_json_max_depth_config()))) {
LOG_WARN("fail to get json base", K(ret), K(type1), K(j_doc), K(j_in_type));
}
}

15
src/sql/engine/expr/ob_expr_json_object.cpp
View File

@ -233,7 +233,10 @@ int ObExprJsonObject::eval_json_object(const ObExpr &expr, ObEvalCtx &ctx, ObDat
ObString raw_bin;
j_obj.stable_sort();
j_obj.unique();
if (OB_FAIL(ObJsonWrapper::get_raw_binary(j_base, raw_bin, &temp_allocator))) {
if (ObJsonExprHelper::is_json_depth_exceed_limit(j_base->depth())) {
ret = OB_ERR_JSON_OUT_OF_DEPTH;
LOG_WARN("current json over depth", K(ret), K(j_base->depth()));
} else if (OB_FAIL(ObJsonWrapper::get_raw_binary(j_base, raw_bin, &temp_allocator))) {
LOG_WARN("failed: get json raw binary", K(ret));
} else if (OB_FAIL(ObJsonExprHelper::pack_json_str_res(expr, ctx, res, raw_bin))) {
LOG_WARN("fail to pack json result", K(ret));
@ -349,7 +352,10 @@ int ObExprJsonObject::eval_ora_json_object(const ObExpr &expr, ObEvalCtx &ctx, O
}
if (OB_SUCC(ret)) {
if (dst_type == ObJsonType) {
if (ObJsonExprHelper::is_json_depth_exceed_limit(j_base->depth())) {
ret = OB_ERR_JSON_OUT_OF_DEPTH;
LOG_WARN("current json over depth", K(ret), K(j_base->depth()));
} else if (dst_type == ObJsonType) {
ObString raw_bin;
if (OB_FAIL(ObJsonWrapper::get_raw_binary(j_base, raw_bin, &temp_allocator))) {
LOG_WARN("failed: get json raw binary", K(ret));
@ -425,7 +431,10 @@ int ObExprJsonObject::check_key_valid(common::hash::ObHashSet<ObString> &view_ke
int ObExprJsonObject::set_result(ObObjType dst_type, ObString str_res, common::ObIAllocator *allocator, ObEvalCtx &ctx, const ObExpr &expr, ObDatum &res, uint8_t strict_type, uint8_t unique_type) {
INIT_SUCC(ret);
if (dst_type == ObVarcharType || dst_type == ObLongTextType) {
if (strict_type == OB_JSON_ON_STRICT_USE && OB_FAIL(ObJsonParser::check_json_syntax(str_res, allocator, ObJsonParser::JSN_STRICT_FLAG))) {
if (strict_type == OB_JSON_ON_STRICT_USE &&
OB_FAIL(ObJsonParser::check_json_syntax(str_res, allocator,
ObJsonParser::JSN_STRICT_FLAG,
ObJsonExprHelper::get_json_max_depth_config()))) {
ret = OB_ERR_JSON_SYNTAX_ERROR;
LOG_WARN("fail to parse json text strict", K(ret));
} else if (OB_FAIL(ObJsonExprHelper::pack_json_str_res(expr, ctx, res, str_res))) {

3
src/sql/engine/expr/ob_expr_json_pretty.cpp
View File

@ -77,7 +77,8 @@ int ObExprJsonPretty::calc(ObEvalCtx &ctx, const ObDatum &data, ObDatumMeta meta
} else if (OB_FAIL(ObTextStringHelper::read_real_string_data(*allocator, data, meta, has_lob_header, j_str))) {
LOG_WARN("fail to get real data.", K(ret), K(j_str));
} else if (OB_FAIL(ObJsonBaseFactory::get_json_base(allocator, j_str, j_in_type,
j_in_type, j_base))) {
j_in_type, j_base, 0,
ObJsonExprHelper::get_json_max_depth_config()))) {
if (ret == OB_ERR_INVALID_JSON_TEXT) {
ret = OB_ERR_INVALID_JSON_TEXT_IN_PARAM;
}

2
src/sql/engine/expr/ob_expr_json_query.cpp
View File

@ -717,7 +717,7 @@ int ObExprJsonQuery::append_binary_node_into_res(ObIJsonBase*& jb_res,
if (OB_FAIL(ret)) {
} else if (OB_FAIL(bin_agg.serialize())) {
LOG_WARN("failed to serialize bin agg.", K(ret));
} else if (OB_FAIL(ObJsonBaseFactory::get_json_base(allocator, bin_agg.get_buffer()->string(), ObJsonInType::JSON_BIN, ObJsonInType::JSON_BIN, jb_res, ObJsonParser::JSN_RELAXED_FLAG))) {
} else if (OB_FAIL(ObJsonBaseFactory::get_json_base(allocator, bin_agg.get_buffer()->string(), ObJsonInType::JSON_BIN, ObJsonInType::JSON_BIN, jb_res, ObJsonParser::JSN_RELAXED_FLAG, ObJsonExprHelper::get_json_max_depth_config()))) {
LOG_WARN("failed to get json base.", K(ret));
}
return ret;

3
src/sql/engine/expr/ob_expr_json_storage_free.cpp
View File

@ -80,7 +80,8 @@ int ObExprJsonStorageFree::calc(ObEvalCtx &ctx, const ObDatum &data, ObDatumMeta
} else if (OB_FAIL(ObTextStringHelper::read_real_string_data(*allocator, data, meta, has_lob_header, j_str))) {
LOG_WARN("fail to get real data.", K(ret), K(j_str));
} else if (OB_FAIL(ObJsonBaseFactory::get_json_base(allocator, j_str, j_in_type,
j_in_type, j_base))) {
j_in_type, j_base, 0,
ObJsonExprHelper::get_json_max_depth_config()))) {
if (ret == OB_ERR_INVALID_JSON_TEXT) {
LOG_USER_ERROR(OB_ERR_INVALID_JSON_TEXT);
}

3
src/sql/engine/expr/ob_expr_json_storage_size.cpp
View File

@ -92,7 +92,8 @@ int ObExprJsonStorageSize::calc(ObEvalCtx &ctx, const ObDatum &data, ObDatumMeta
} else if (OB_FAIL(ObTextStringHelper::read_real_string_data(*allocator, data, meta, has_lob_header, j_str))) {
LOG_WARN("fail to get real data.", K(ret), K(j_str));
} else if (OB_FAIL(ObJsonBaseFactory::get_json_base(allocator, j_str, j_in_type,
j_in_type, j_base))) {
j_in_type, j_base, 0,
ObJsonExprHelper::get_json_max_depth_config()))) {
if (ret == OB_ERR_INVALID_JSON_TEXT) {
LOG_USER_ERROR(OB_ERR_INVALID_JSON_TEXT);
}

3
src/sql/engine/expr/ob_expr_json_type.cpp
View File

@ -154,7 +154,8 @@ int ObExprJsonType::calc(ObEvalCtx &ctx, const ObDatum &data, ObDatumMeta meta,
} else if (OB_FAIL(ObTextStringHelper::read_real_string_data(*allocator, data, meta, has_lob_header, j_str))) {
LOG_WARN("fail to get real data.", K(ret), K(j_str));
} else if (OB_FAIL(ObJsonBaseFactory::get_json_base(allocator, j_str, j_in_type,
j_in_type, j_base))) {
j_in_type, j_base, 0,
ObJsonExprHelper::get_json_max_depth_config()))) {
LOG_WARN("fail to get json base", K(ret), K(type), K(j_str), K(j_in_type));
if (ret == OB_ERR_INVALID_JSON_TEXT_IN_PARAM) {
ret = OB_ERR_INVALID_JSON_TEXT;

3
src/sql/engine/expr/ob_expr_json_unquote.cpp
View File

@ -94,7 +94,8 @@ int ObExprJsonUnquote::calc(ObEvalCtx &ctx, const ObDatum &data, ObDatumMeta met
LOG_WARN("failed: copy original string", K(ret), K(j_str));
}
} else if (OB_FAIL(ObJsonBaseFactory::get_json_base(allocator, j_str, j_in_type,
j_in_type, j_base))) {
j_in_type, j_base, 0,
ObJsonExprHelper::get_json_max_depth_config()))) {
LOG_WARN("failed: get json base", K(ret), K(type));
if (OB_ERR_INVALID_JSON_TEXT) {
ret = OB_ERR_INVALID_JSON_TEXT_IN_PARAM;

3
src/sql/engine/expr/ob_expr_json_utils.cpp
View File

@ -1847,7 +1847,8 @@ int ObJsonUtil::get_json_doc(ObExpr *expr,
if (is_oracle && j_str.length() == 0) {
is_null = true;
} else if (OB_FAIL(ObJsonBaseFactory::get_json_base(&allocator, j_str, j_in_type,
expect_type, j_base, parse_flag))) {
expect_type, j_base, parse_flag,
ObJsonExprHelper::get_json_max_depth_config()))) {
LOG_WARN("fail to get json base", K(ret), K(j_in_type));
if (ret == OB_ERR_JSON_OUT_OF_DEPTH) {
is_cover_by_error = false;

5
src/sql/engine/expr/ob_expr_json_valid.cpp
View File

@ -96,11 +96,12 @@ int ObExprJsonValid::calc(ObEvalCtx &ctx, const ObDatum &data, ObDatumMeta meta,
} else if (type == ObJsonType) { // json bin
ObIJsonBase *j_bin = NULL;
if (OB_FAIL(ObJsonBaseFactory::get_json_base(allocator, j_str, ObJsonInType::JSON_BIN,
ObJsonInType::JSON_BIN, j_bin))) {
ObJsonInType::JSON_BIN, j_bin, 0,
ObJsonExprHelper::get_json_max_depth_config()))) {
LOG_WARN("fail to get json base", K(ret), K(type), K(j_str));
}
} else { // json tree
if (OB_FAIL(ObJsonParser::check_json_syntax(j_str, allocator))) {
if (OB_FAIL(ObJsonParser::check_json_syntax(j_str, allocator, 0, ObJsonExprHelper::get_json_max_depth_config()))) {
LOG_WARN("fail to check json syntax", K(ret), K(type), K(j_str));
}
}

1
tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result
View File

@ -131,6 +131,7 @@ ignore_replay_checksum_error
index_block_cache_priority
internal_sql_execute_timeout
job_queue_processes
json_document_max_depth
kv_ttl_duty_duration
kv_ttl_history_recycle_interval
large_query_threshold

8
unittest/share/test_json_base.cpp
View File

@ -2405,12 +2405,12 @@ TEST_F(TestJsonBase, test_print)
ASSERT_EQ(OB_SUCCESS, j_new_arr->append(j_int_val));
ASSERT_EQ(OB_SUCCESS, j_arr_last->append(j_new_arr));
j_buf.reset();
ASSERT_EQ(OB_ERR_JSON_OUT_OF_DEPTH, j_tree->print(j_buf, false));
//ASSERT_EQ(OB_ERR_JSON_OUT_OF_DEPTH, j_tree->print(j_buf, false));
// json bin
ASSERT_EQ(OB_SUCCESS, ObJsonBaseFactory::transform(&allocator, j_tree,
ObJsonInType::JSON_BIN, j_bin));
j_buf.reset();
ASSERT_EQ(OB_ERR_JSON_OUT_OF_DEPTH, j_bin->print(j_buf, false));
//ASSERT_EQ(OB_ERR_JSON_OUT_OF_DEPTH, j_bin->print(j_buf, false));
// object
// json tree
@ -2432,12 +2432,12 @@ TEST_F(TestJsonBase, test_print)
// 第100个,预期报错OB_ERR_JSON_OUT_OF_DEPTH
ASSERT_EQ(OB_SUCCESS, create_object(&allocator, key_buf, strlen(key_buf), last_obj, new_obj));
j_tree = new_obj;
ASSERT_EQ(OB_ERR_JSON_OUT_OF_DEPTH, j_tree->print(j_buf, false));
//ASSERT_EQ(OB_ERR_JSON_OUT_OF_DEPTH, j_tree->print(j_buf, false));
// json bin
ASSERT_EQ(OB_SUCCESS, ObJsonBaseFactory::transform(&allocator, j_tree,
ObJsonInType::JSON_BIN, j_bin));
j_buf.reset();
ASSERT_EQ(OB_ERR_JSON_OUT_OF_DEPTH, j_bin->print(j_buf, false));
//ASSERT_EQ(OB_ERR_JSON_OUT_OF_DEPTH, j_bin->print(j_buf, false));
// boolean
// json tree