From 5a47b904ed8779a256c7556b39615376b60ec54d Mon Sep 17 00:00:00 2001 From: obdev Date: Fri, 22 Sep 2023 09:44:04 +0000 Subject: [PATCH] fix:is json check constrain bug, json-table path expr not quoted supported, gdb charset result incorrect --- src/sql/engine/expr/ob_expr_json_array.cpp | 12 +++- .../engine/expr/ob_expr_json_func_helper.cpp | 66 +++++++++++++++++++ .../engine/expr/ob_expr_json_func_helper.h | 15 +++-- .../engine/expr/ob_expr_json_merge_patch.cpp | 35 ++++++++-- src/sql/engine/expr/ob_expr_json_object.cpp | 24 ++++++- src/sql/resolver/ddl/ob_ddl_resolver.cpp | 22 ++++--- src/sql/resolver/dml/ob_dml_resolver.cpp | 42 +++++++----- unittest/share/test_json_tree.cpp | 3 +- 8 files changed, 179 insertions(+), 40 deletions(-) diff --git a/src/sql/engine/expr/ob_expr_json_array.cpp b/src/sql/engine/expr/ob_expr_json_array.cpp index c72296d41..b005943b4 100644 --- a/src/sql/engine/expr/ob_expr_json_array.cpp +++ b/src/sql/engine/expr/ob_expr_json_array.cpp @@ -220,7 +220,17 @@ int ObExprJsonArray::eval_ora_json_array(const ObExpr &expr, ObEvalCtx &ctx, ObD } else if (OB_FAIL(j_arr.print(string_buffer, true, false))) { LOG_WARN("failed: get json string text", K(ret)); } else { - res_string.assign_ptr(string_buffer.ptr(), string_buffer.length()); + ObCollationType in_cs_type = CS_TYPE_UTF8MB4_BIN; + ObCollationType dst_cs_type = expr.obj_meta_.get_collation_type(); + ObString temp_str = string_buffer.string(); + + if (OB_FAIL(ObJsonExprHelper::convert_string_collation_type(in_cs_type, + dst_cs_type, + &temp_allocator, + temp_str, + res_string))) { + LOG_WARN("fail to convert string result", K(ret)); + } } } diff --git a/src/sql/engine/expr/ob_expr_json_func_helper.cpp b/src/sql/engine/expr/ob_expr_json_func_helper.cpp index 6c68300ce..63e7c92e4 100644 --- a/src/sql/engine/expr/ob_expr_json_func_helper.cpp +++ b/src/sql/engine/expr/ob_expr_json_func_helper.cpp @@ -535,6 +535,72 @@ int ObJsonExprHelper::oracle_datum2_json_val(const ObDatum *json_datum, return ret; } +int ObJsonExprHelper::convert_string_collation_type(ObCollationType in_cs_type, + ObCollationType dst_cs_type, + ObIAllocator* allocator, + ObString &in_str, + ObString &out_str) +{ + INIT_SUCC(ret); + + bool is_need_convert = ((CS_TYPE_BINARY == dst_cs_type) + || (ObCharset::charset_type_by_coll(in_cs_type) != ObCharset::charset_type_by_coll(dst_cs_type))); + + if (is_need_convert) { + if (CS_TYPE_BINARY != in_cs_type && CS_TYPE_BINARY != dst_cs_type + && (ObCharset::charset_type_by_coll(in_cs_type) != ObCharset::charset_type_by_coll(dst_cs_type))) { + char *buf = nullptr; + int64_t buf_len = (in_str.length() == 0 ? 1 : in_str.length()) * ObCharset::CharConvertFactorNum; + uint32_t result_len = 0; + + if (OB_ISNULL(buf = reinterpret_cast(allocator->alloc(buf_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc memory failed", K(ret), K(buf_len)); + } else if (OB_FAIL(ObCharset::charset_convert(in_cs_type, + in_str.ptr(), + in_str.length(), + dst_cs_type, + buf, + buf_len, + result_len))) { + LOG_WARN("charset convert failed", K(ret)); + } else { + out_str.assign_ptr(buf, result_len); + } + } else { + if (CS_TYPE_BINARY == in_cs_type || CS_TYPE_BINARY == dst_cs_type) { + // just copy string when in_cs_type or out_cs_type is binary + const ObCharsetInfo *cs = NULL; + int64_t align_offset = 0; + if (CS_TYPE_BINARY == in_cs_type && (NULL != (cs = ObCharset::get_charset(dst_cs_type)))) { + if (cs->mbminlen > 0 && in_str.length() % cs->mbminlen != 0) { + align_offset = cs->mbminlen - in_str.length() % cs->mbminlen; + } + } + int64_t len = align_offset + in_str.length(); + char *buf = nullptr; + + if (OB_ISNULL(buf = static_cast(allocator->alloc(len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate memory failed", K(ret), K(len)); + } else { + MEMMOVE(buf + align_offset, in_str.ptr(), len - align_offset); + MEMSET(buf, 0, align_offset); + out_str.assign_ptr(buf, len); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("same charset should not be here, just use cast_eval_arg", K(ret), + K(in_cs_type), K(dst_cs_type), K(in_cs_type), K(dst_cs_type)); + } + } + } else { + out_str = in_str; + } + + return ret; +} + int ObJsonExprHelper::json_base_replace(ObIJsonBase *json_old, ObIJsonBase *json_new, ObIJsonBase *&json_doc) { diff --git a/src/sql/engine/expr/ob_expr_json_func_helper.h b/src/sql/engine/expr/ob_expr_json_func_helper.h index eeffbb680..af3d09962 100644 --- a/src/sql/engine/expr/ob_expr_json_func_helper.h +++ b/src/sql/engine/expr/ob_expr_json_func_helper.h @@ -163,11 +163,11 @@ public: static bool is_cs_type_bin(ObCollationType &cs_type); static int get_timestamp_str_in_oracle_mode(ObEvalCtx &ctx, - const ObDatum &datum, - ObObjType type, - ObScale scale, - const ObTimeZoneInfo *tz_info, - ObJsonBuffer &j_buf); + const ObDatum &datum, + ObObjType type, + ObScale scale, + const ObTimeZoneInfo *tz_info, + ObJsonBuffer &j_buf); static bool is_convertible_to_json(ObObjType &type); static int is_valid_for_json(ObExprResType* types_stack, uint32_t index, const char* func_name); @@ -176,6 +176,11 @@ public: static void set_type_for_value(ObExprResType* types_stack, uint32_t index); static int ensure_collation(ObObjType type, ObCollationType cs_type); static ObJsonInType get_json_internal_type(ObObjType type); + static int convert_string_collation_type(ObCollationType in, + ObCollationType dst, + ObIAllocator* allocator, + ObString& in_str, + ObString &out_str); template static int pack_json_str_res(const ObExpr &expr, ObEvalCtx &ctx, diff --git a/src/sql/engine/expr/ob_expr_json_merge_patch.cpp b/src/sql/engine/expr/ob_expr_json_merge_patch.cpp index 52e4d3751..350e55a26 100644 --- a/src/sql/engine/expr/ob_expr_json_merge_patch.cpp +++ b/src/sql/engine/expr/ob_expr_json_merge_patch.cpp @@ -294,24 +294,45 @@ int ObExprJsonMergePatch::eval_ora_json_merge_patch(const ObExpr &expr, ObEvalCt if (has_null) { res.set_null(); } else { - ObJsonBuffer jbuf(&temp_allocator); + ObJsonBuffer* jbuf = nullptr; ObString res_string; bool is_res_blob = expr.datum_meta_.cs_type_ == CS_TYPE_BINARY && dst_type == ObLongTextType; - if (dst_type == ObJsonType) { + if (OB_ISNULL( jbuf = OB_NEWx(ObJsonBuffer, &temp_allocator, &temp_allocator))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to construct jbuf", K(ret)); + } else if (dst_type == ObJsonType) { if (OB_FAIL(j_base->get_raw_binary(res_string, &temp_allocator))) { LOG_WARN("failed: get json raw binary", K(ret)); } } else { + ObString tmp_val; + ObString result_str; bool is_quote = j_base->json_type() == ObJsonNodeType::J_STRING; - if (OB_FAIL(j_base->print(jbuf, is_quote, is_pretty > 0))) { + + if (OB_FAIL(j_base->print(*jbuf, is_quote, is_pretty > 0))) { LOG_WARN("json binary to string failed", K(ret)); - } else if (jbuf.empty()) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("allocate memory for result failed", K(ret)); + } else if (jbuf->empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("jbuf should not empty", K(ret)); + } else { + tmp_val = jbuf->string(); + ObCollationType in_cs_type = CS_TYPE_UTF8MB4_BIN; + ObCollationType dst_cs_type = expr.obj_meta_.get_collation_type(); + result_str = tmp_val; + + if (OB_FAIL(ObJsonExprHelper::convert_string_collation_type(in_cs_type, + dst_cs_type, + &temp_allocator, + tmp_val, + result_str))) { + LOG_WARN("fail to convert string result", K(ret)); + } else { + tmp_val = result_str; + } } - ObString tmp_val(jbuf.length(), jbuf.ptr()); + if (OB_SUCC(ret) && is_asc && !is_res_blob /* clob varchar */ ) { if (OB_FAIL(ObJsonExprHelper::character2_ascii_string(&temp_allocator, expr, ctx, tmp_val, 1))) { LOG_WARN("fail to transform string 2 ascii character", K(ret)); diff --git a/src/sql/engine/expr/ob_expr_json_object.cpp b/src/sql/engine/expr/ob_expr_json_object.cpp index 9a789bd45..770440040 100644 --- a/src/sql/engine/expr/ob_expr_json_object.cpp +++ b/src/sql/engine/expr/ob_expr_json_object.cpp @@ -357,18 +357,38 @@ int ObExprJsonObject::eval_ora_json_object(const ObExpr &expr, ObEvalCtx &ctx, O LOG_WARN("fail to pack json result", K(ret), K(raw_bin)); } } else { + + ObString temp_str; + ObString result_str; + if (OB_FAIL(string_buffer.reserve(j_obj.get_serialize_size()))) { LOG_WARN("fail to reserve string.", K(ret), K(j_obj.get_serialize_size())); } else if (OB_FAIL(j_base->print(string_buffer, false, false))) { LOG_WARN("fail to transform to string.", K(ret), K(string_buffer.length())); - } else if (dst_type == ObVarcharType && string_buffer.length() > dst_len) { + } else { + ObCollationType in_cs_type = CS_TYPE_UTF8MB4_BIN; + ObCollationType dst_cs_type = expr.obj_meta_.get_collation_type(); + temp_str = string_buffer.string(); + result_str = temp_str; + + if (OB_FAIL(ObJsonExprHelper::convert_string_collation_type(in_cs_type, + dst_cs_type, + &temp_allocator, + temp_str, + result_str))) { + LOG_WARN("fail to convert string result", K(ret)); + } + } + + + if (dst_type == ObVarcharType && result_str.length() > dst_len) { char res_ptr[OB_MAX_DECIMAL_PRECISION] = {0}; if (OB_ISNULL(ObCharset::lltostr(dst_len, res_ptr, 10, 1))) { LOG_WARN("dst_len fail to string.", K(ret)); } ret = OB_OPERATE_OVERFLOW; LOG_USER_ERROR(OB_OPERATE_OVERFLOW, res_ptr, "json_object"); - } else if (OB_FAIL(ObJsonExprHelper::pack_json_str_res(expr, ctx, res, string_buffer.string()))) { + } else if (OB_FAIL(ObJsonExprHelper::pack_json_str_res(expr, ctx, res, result_str))) { LOG_WARN("fail to pack json result", K(ret)); } } diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.cpp b/src/sql/resolver/ddl/ob_ddl_resolver.cpp index 490d76268..290d73266 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.cpp +++ b/src/sql/resolver/ddl/ob_ddl_resolver.cpp @@ -7190,14 +7190,20 @@ int ObDDLResolver::check_is_json_contraint(ObTableSchema &t_table_schema, ObIArr } for (ObTableSchema::const_constraint_iterator iter = t_table_schema.constraint_begin(); OB_SUCC(ret) && iter != t_table_schema.constraint_end(); iter ++) { - if (OB_ISNULL((*iter)->get_check_expr_str().ptr())) { - } else if (OB_FAIL(ObRawExprUtils::parse_bool_expr_node_from_str( - (*iter)->get_check_expr_str(), *allocator_, node))) { - LOG_WARN("parse expr node from string failed", K(ret)); - } else { - if (node->type_ == T_FUN_SYS_IS_JSON) { - ret = OB_ERR_ADDITIONAL_IS_JSON; - LOG_WARN("cannot add additional is json check constraint", K(ret)); + ObConstraint *cst = *iter; + for (ObConstraint::const_cst_col_iterator cst_col_iter = cst->cst_col_begin(); + OB_SUCC(ret) && (cst_col_iter != cst->cst_col_end()); ++cst_col_iter) { + if (*cst_col_iter == col_id) { + if (OB_ISNULL(cst->get_check_expr_str().ptr())) { + } else if (OB_FAIL(ObRawExprUtils::parse_bool_expr_node_from_str( + cst->get_check_expr_str(), *allocator_, node))) { + LOG_WARN("parse expr node from string failed", K(ret)); + } else { + if (node->type_ == T_FUN_SYS_IS_JSON) { + ret = OB_ERR_ADDITIONAL_IS_JSON; + LOG_WARN("cannot add additional is json check constraint", K(ret)); + } + } } } } diff --git a/src/sql/resolver/dml/ob_dml_resolver.cpp b/src/sql/resolver/dml/ob_dml_resolver.cpp index 4e9073aec..3d3b6c1c7 100755 --- a/src/sql/resolver/dml/ob_dml_resolver.cpp +++ b/src/sql/resolver/dml/ob_dml_resolver.cpp @@ -8572,8 +8572,11 @@ int ObDMLResolver::json_table_make_json_path(const ParseNode &parse_tree, ret = OB_ERR_UNEXPECTED; LOG_WARN("allocator is NULL", K(ret)); } else if (parse_tree.type_ == T_OBJ_ACCESS_REF) { - ObJsonBuffer path_buffer(allocator); - if (OB_FAIL(path_buffer.append("$."))) { + ObJsonBuffer* path_buffer = nullptr; + if (OB_ISNULL(path_buffer = static_cast(allocator->alloc(sizeof(ObJsonBuffer))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + } else if (FALSE_IT(path_buffer = new (path_buffer) ObJsonBuffer(allocator))) { + } else if (OB_FAIL(path_buffer->append("$."))) { LOG_WARN("failed to append path start", K(ret)); } else if (parse_tree.num_child_ != 2 || OB_ISNULL(parse_tree.children_) @@ -8582,8 +8585,8 @@ int ObDMLResolver::json_table_make_json_path(const ParseNode &parse_tree, ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to make path, param not expected", K(ret), K(parse_tree.num_child_), KP(parse_tree.children_)); - } else if (OB_FAIL(path_buffer.append(parse_tree.children_[0]->str_value_, - parse_tree.children_[0]->str_len_))) { + } else if (OB_FAIL(path_buffer->append(parse_tree.children_[0]->raw_text_, + parse_tree.children_[0]->text_len_))) { LOG_WARN("failed to append raw text", K(ret)); } else { ParseNode *tmp_path = parse_tree.children_[1]; @@ -8592,7 +8595,7 @@ int ObDMLResolver::json_table_make_json_path(const ParseNode &parse_tree, tmp_path = NULL; // do nothing } else { - if (OB_FAIL(print_json_path(tmp_path, path_buffer))) { + if (OB_FAIL(print_json_path(tmp_path, *path_buffer))) { LOG_WARN("failed to print path", K(ret)); } } @@ -8600,12 +8603,12 @@ int ObDMLResolver::json_table_make_json_path(const ParseNode &parse_tree, char* path_buf = NULL; if (OB_FAIL(ret)) { - } else if (OB_ISNULL(path_buf = static_cast(allocator->alloc(path_buffer.length() + 1)))) { + } else if (OB_ISNULL(path_buf = static_cast(allocator->alloc(path_buffer->length() + 1)))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to allocate path buffer", K(ret), K(path_buffer.length())); + LOG_WARN("failed to allocate path buffer", K(ret), K(path_buffer->length())); } else { - MEMCPY(path_buf, path_buffer.ptr(), path_buffer.length()); - path_buf[path_buffer.length()] = 0; + MEMCPY(path_buf, path_buffer->ptr(), path_buffer->length()); + path_buf[path_buffer->length()] = 0; path_str.assign_ptr(path_buf, strlen(path_buf)); } } @@ -9202,8 +9205,15 @@ int ObDMLResolver::resolve_json_table_nested_column(const ParseNode &parse_tree, col_def = new (buf) ObDmlJtColDef(); col_def->col_base_info_.col_type_ = NESTED_COL_TYPE; - const ParseNode* path_node = parse_tree.children_[0]; - if (OB_ISNULL(path_node->str_value_) || path_node->str_len_ == 0) { + ParseNode* path_node = const_cast(parse_tree.children_[0]); + + // json table nested path syntax, not quoted: + // nested path employees[*] columns ( name, job ) + if (path_node->value_ == 2) { + if (OB_FAIL(json_table_make_json_path(*path_node, allocator_, col_def->col_base_info_.path_))) { + LOG_WARN("failed to make json path.", K(ret)); + } + } else if (OB_ISNULL(path_node->str_value_) || path_node->str_len_ == 0) { ret = OB_INVALID_ARGUMENT; LOG_WARN("allocate memory failed", K(ret), K(alloc_size)); } else { @@ -9212,12 +9222,12 @@ int ObDMLResolver::resolve_json_table_nested_column(const ParseNode &parse_tree, } else if (OB_FAIL(json_table_make_json_path(*path_node, allocator_, col_def->col_base_info_.path_))) { LOG_WARN("failed to make json path.", K(ret)); } + } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(resolve_json_table_column_item(*parse_tree.children_[1], - table_item, col_def, parent, id, cur_column_id))) { - LOG_WARN("failed to resolve nested column defination.", K(ret)); - } + if (OB_SUCC(ret) + && OB_FAIL(resolve_json_table_column_item(*parse_tree.children_[1], + table_item, col_def, parent, id, cur_column_id))) { + LOG_WARN("failed to resolve nested column defination.", K(ret)); } } return ret; diff --git a/unittest/share/test_json_tree.cpp b/unittest/share/test_json_tree.cpp index ec787b78e..cc06d55a3 100644 --- a/unittest/share/test_json_tree.cpp +++ b/unittest/share/test_json_tree.cpp @@ -13,6 +13,7 @@ #include #define private public #include "lib/json_type/ob_json_tree.h" +#include "lib/json_type/ob_json_bin.h" #include "lib/json_type/ob_json_parse.h" #include "lib/timezone/ob_timezone_info.h" #undef private @@ -2264,7 +2265,7 @@ TEST_F(TestJsonTree, test_big_json) char value_buffer[16] = {0}; int idx = 0; - for (int64_t pos = 0; pos < 20000; ++pos) { + for (int64_t pos = 0; pos < 50000; ++pos) { for (int i = 0; i < 32; ++i) { idx = ObRandom::rand(0, 15); key_buffer[i] = origin[idx];