From aa46ea1fbd319d4838dcbc5b61c7ecc3b1bea8ad Mon Sep 17 00:00:00 2001 From: obdev Date: Tue, 7 Mar 2023 03:11:01 +0000 Subject: [PATCH] [BUGFIX]fix lob locator with charset convert --- src/observer/mysql/ob_query_driver.cpp | 150 ++++++++++++++------ src/observer/mysql/ob_query_driver.h | 2 +- src/sql/engine/expr/ob_expr_output_pack.cpp | 136 +++++++++++++----- 3 files changed, 204 insertions(+), 84 deletions(-) diff --git a/src/observer/mysql/ob_query_driver.cpp b/src/observer/mysql/ob_query_driver.cpp index 411f016f9..273999f06 100644 --- a/src/observer/mysql/ob_query_driver.cpp +++ b/src/observer/mysql/ob_query_driver.cpp @@ -511,7 +511,7 @@ int ObQueryDriver::process_lob_locator_results(ObObj& value, bool is_use_lob_locator, bool is_support_outrow_locator_v2, ObIAllocator *allocator, - sql::ObSQLSessionInfo *session_info) + const sql::ObSQLSessionInfo *session_info) { int ret = OB_SUCCESS; // 1. if client is_use_lob_locator, return lob locator @@ -534,7 +534,7 @@ int ObQueryDriver::process_lob_locator_results(ObObj& value, ret = OB_INVALID_ARGUMENT; LOG_WARN("Lob: oracle lob locator v2 with out extern segment", K(value), K(GET_MIN_CLUSTER_VERSION())); } else { - if (!is_support_outrow_locator_v2 && !loc.is_inrow()) { + if (!is_support_outrow_locator_v2 && !loc.has_inrow_data()) { if (OB_FAIL(ObTextStringIter::append_outrow_lob_fulldata(value, session_info, *allocator))) { LOG_WARN("Lob: convert lob to outrow failed", K(value), K(GET_MIN_CLUSTER_VERSION())); } @@ -688,49 +688,111 @@ int ObQueryDriver::convert_text_value_charset(ObObj& value, LOG_WARN("Lob: invalid collation", K(from_collation_type), K(to_collation_type), K(ret)); } else if (CS_TYPE_BINARY != from_collation_type && CS_TYPE_BINARY != to_collation_type && strcmp(from_charset_info->csname, to_charset_info->csname) != 0) { - - // get full data, buffer size is full byte length * 4 - ObString data_str = value.get_string(); - int64_t lob_data_byte_len = data_str.length(); - if (!value.has_lob_header()) { - } else { - ObLobLocatorV2 loc(raw_str, value.has_lob_header()); - ObTextStringIter str_iter(value); - if (OB_FAIL(str_iter.init(0, session, &allocator))) { - LOG_WARN("Lob: init lob str iter failed ", K(ret), K(value)); - } else if (OB_FAIL(str_iter.get_full_data(data_str))) { - LOG_WARN("Lob: get full data failed ", K(ret), K(value)); - } else if (OB_FAIL(loc.get_lob_data_byte_len(lob_data_byte_len))) { - LOG_WARN("Lob: get lob data byte len failed", K(ret), K(loc)); - } - } - if (OB_SUCC(ret)) { - // mock result buffer and reserve data length - // could do streaming charset convert - ObTextStringResult new_tmp_lob(type, value.has_lob_header(), &allocator); - char *buf = NULL; - int64_t buf_len = 0; - uint32_t result_len = 0; - - if (OB_FAIL(new_tmp_lob.init(lob_data_byte_len * 4))) { - LOG_WARN("Lob: init tmp lob failed", K(ret), K(lob_data_byte_len * 4)); - } else if (OB_FAIL(new_tmp_lob.get_reserved_buffer(buf, buf_len))) { - LOG_WARN("Lob: get empty buffer failed", K(ret), K(lob_data_byte_len * 4)); - } else if (OB_FAIL(convert_string_charset(data_str, from_collation_type, to_collation_type, - buf, buf_len, result_len))) { - LOG_WARN("Lob: convert string charset failed", K(ret)); - } else if (OB_FAIL(new_tmp_lob.lseek(result_len, 0))) { - LOG_WARN("Lob: temp lob lseek failed", K(ret)); + if (value.is_lob_storage() && value.has_lob_header() && OB_NOT_NULL(session) && + session->is_client_use_lob_locator() && lib::is_oracle_mode()) { + ObLobLocatorV2 lob; + ObString inrow_data; + if (OB_FAIL(process_lob_locator_results(value, + session->is_client_use_lob_locator(), + session->is_client_support_lob_locatorv2(), + &allocator, + session))) { + LOG_WARN("fail to process lob locator", K(ret), K(value)); + } else if (OB_FAIL(value.get_lob_locatorv2(lob))) { + LOG_WARN("fail to lob locator v2", K(ret), K(value)); + } else if (!lob.has_inrow_data()) { + // do nothing + } else if (OB_FAIL(lob.get_inrow_data(inrow_data))) { + LOG_WARN("fail to get inrow data", K(ret), K(lob)); } else { - ObString lob_loc_str; - new_tmp_lob.get_result_buffer(lob_loc_str); - LOG_INFO("Lob: new temp convert_text_value_charset in convert_text_value", - K(ret), K(raw_str), K(type), K(to_collation_type), K(from_collation_type), - K(from_charset_info->csname), K(to_charset_info->csname)); - value.set_lob_value(type, lob_loc_str.ptr(), lob_loc_str.length()); - value.set_collation_type(to_collation_type); - if (new_tmp_lob.has_lob_header()) { - value.set_has_lob_header(); + int64_t lob_data_byte_len = inrow_data.length(); + int64_t offset_len = reinterpret_cast(inrow_data.ptr()) - reinterpret_cast(lob.ptr_); + int64_t res_len = offset_len + lob_data_byte_len * ObCharset::CharConvertFactorNum; + char *buf = static_cast(allocator.alloc(res_len)); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc memory", K(ret), K(res_len)); + } else { + MEMCPY(buf, lob.ptr_, offset_len); + uint32_t result_len = 0; + if (OB_FAIL(convert_string_charset(inrow_data, from_collation_type, to_collation_type, + buf + offset_len, res_len - offset_len, result_len))) { + LOG_WARN("Lob: convert string charset failed", K(ret)); + } else { + // refresh payload size + if (lob.has_extern()) { + ObMemLobExternHeader *ex_header = nullptr; + if (OB_FAIL(lob.get_extern_header(ex_header))) { + LOG_WARN("fail to get extern header", K(ret), K(lob)); + } else { + ex_header->payload_size_ = result_len; + } + } + // refresh lob data size + if (OB_SUCC(ret)) { + ObLobCommon *lob_common = nullptr; + if (OB_FAIL(lob.get_disk_locator(lob_common))) { + LOG_WARN("fail to get lob common", K(ret), K(lob)); + } else if (lob_common->is_init_) { + ObLobData *lob_data = reinterpret_cast(lob_common->buffer_); + lob_data->byte_size_ = result_len; + } + } + if (OB_SUCC(ret)) { + LOG_INFO("Lob: new temp convert_text_value_charset in convert_text_value", + K(ret), K(raw_str), K(type), K(to_collation_type), K(from_collation_type), + K(from_charset_info->csname), K(to_charset_info->csname)); + value.set_lob_value(type, buf, offset_len + result_len); + value.set_collation_type(to_collation_type); + value.set_has_lob_header(); + } + } + } + } + } else { + // get full data, buffer size is full byte length * ObCharset::CharConvertFactorNum + ObString data_str = value.get_string(); + int64_t lob_data_byte_len = data_str.length(); + if (!value.has_lob_header()) { + } else { + ObLobLocatorV2 loc(raw_str, value.has_lob_header()); + ObTextStringIter str_iter(value); + if (OB_FAIL(str_iter.init(0, session, &allocator))) { + LOG_WARN("Lob: init lob str iter failed ", K(ret), K(value)); + } else if (OB_FAIL(str_iter.get_full_data(data_str))) { + LOG_WARN("Lob: get full data failed ", K(ret), K(value)); + } else if (OB_FAIL(loc.get_lob_data_byte_len(lob_data_byte_len))) { + LOG_WARN("Lob: get lob data byte len failed", K(ret), K(loc)); + } + } + if (OB_SUCC(ret)) { + // mock result buffer and reserve data length + // could do streaming charset convert + ObTextStringResult new_tmp_lob(type, value.has_lob_header(), &allocator); + char *buf = NULL; + int64_t buf_len = 0; + uint32_t result_len = 0; + int64_t converted_len = lob_data_byte_len * ObCharset::CharConvertFactorNum; + if (OB_FAIL(new_tmp_lob.init(converted_len))) { + LOG_WARN("Lob: init tmp lob failed", K(ret), K(converted_len)); + } else if (OB_FAIL(new_tmp_lob.get_reserved_buffer(buf, buf_len))) { + LOG_WARN("Lob: get empty buffer failed", K(ret), K(converted_len)); + } else if (OB_FAIL(convert_string_charset(data_str, from_collation_type, to_collation_type, + buf, buf_len, result_len))) { + LOG_WARN("Lob: convert string charset failed", K(ret)); + } else if (OB_FAIL(new_tmp_lob.lseek(result_len, 0))) { + LOG_WARN("Lob: temp lob lseek failed", K(ret)); + } else { + ObString lob_loc_str; + new_tmp_lob.get_result_buffer(lob_loc_str); + LOG_INFO("Lob: new temp convert_text_value_charset in convert_text_value", + K(ret), K(raw_str), K(type), K(to_collation_type), K(from_collation_type), + K(from_charset_info->csname), K(to_charset_info->csname)); + value.set_lob_value(type, lob_loc_str.ptr(), lob_loc_str.length()); + value.set_collation_type(to_collation_type); + if (new_tmp_lob.has_lob_header()) { + value.set_has_lob_header(); + } } } } diff --git a/src/observer/mysql/ob_query_driver.h b/src/observer/mysql/ob_query_driver.h index 4be533462..0231af605 100644 --- a/src/observer/mysql/ob_query_driver.h +++ b/src/observer/mysql/ob_query_driver.h @@ -81,7 +81,7 @@ public: bool is_use_lob_locator, bool is_support_outrow_locator_v2, common::ObIAllocator *allocator, - sql::ObSQLSessionInfo *session_info); + const sql::ObSQLSessionInfo *session_info); static int convert_string_charset(const common::ObString &in_str, const common::ObCollationType in_cs_type, const common::ObCollationType out_cs_type, diff --git a/src/sql/engine/expr/ob_expr_output_pack.cpp b/src/sql/engine/expr/ob_expr_output_pack.cpp index a2974c83e..1b954ca3b 100644 --- a/src/sql/engine/expr/ob_expr_output_pack.cpp +++ b/src/sql/engine/expr/ob_expr_output_pack.cpp @@ -328,45 +328,103 @@ int ObExprOutputPack::convert_text_value_charset(common::ObObj& value, LOG_WARN("invalid collation", K(from_collation_type), K(to_collation_type), K(ret)); } else if (CS_TYPE_BINARY != from_collation_type && CS_TYPE_BINARY != to_collation_type && strcmp(from_charset_info->csname, to_charset_info->csname) != 0) { - - // get full data, buffer size is full byte length * 4 - ObString data_str; - ObLobLocatorV2 loc(raw_str, value.has_lob_header()); - ObTextStringIter str_iter(value); - if (OB_FAIL(str_iter.init(0, &my_session, &alloc))) { - LOG_WARN("Lob: init lob str iter failed ", K(ret), K(value)); - } else if (OB_FAIL(str_iter.get_full_data(data_str))) { - LOG_WARN("Lob: init lob str iter failed ", K(ret), K(value)); - } else { - // mock result buffer and reserve data length - // could do streaming charset convert - ObTextStringResult new_tmp_lob(type, res_has_lob_header, &alloc); - char *buf = NULL; - int64_t buf_len = 0; - int64_t lob_data_byte_len = 0; - uint32_t result_len = 0; - - if (OB_FAIL(loc.get_lob_data_byte_len(lob_data_byte_len))) { - LOG_WARN("Lob: get lob data byte len failed", K(ret), K(loc)); - } else if (OB_FAIL(new_tmp_lob.init(lob_data_byte_len * 4))) { - LOG_WARN("Lob: init tmp lob failed", K(ret), K(lob_data_byte_len * 4)); - } else if (OB_FAIL(new_tmp_lob.get_reserved_buffer(buf, buf_len))) { - LOG_WARN("Lob: get empty buffer failed", K(ret), K(lob_data_byte_len * 4)); - } else if (OB_FAIL(convert_string_charset(data_str, from_collation_type, to_collation_type, - buf, buf_len, result_len))) { - LOG_WARN("convert string charset failed", K(ret)); - } else if (OB_FAIL(new_tmp_lob.lseek(result_len, 0))) { - LOG_WARN("temp lob lseek failed", K(ret)); + if (value.is_lob_storage() && value.has_lob_header() && res_has_lob_header && + my_session.is_client_use_lob_locator() && lib::is_oracle_mode()) { + ObLobLocatorV2 lob; + ObString inrow_data; + if (OB_FAIL(process_lob_locator_results(value, alloc, my_session))) { + LOG_WARN("fail to process lob locator", K(ret), K(value)); + } else if (OB_FAIL(value.get_lob_locatorv2(lob))) { + LOG_WARN("fail to lob locator v2", K(ret), K(value)); + } else if (!lob.has_inrow_data()) { + // do nothing + } else if (OB_FAIL(lob.get_inrow_data(inrow_data))) { + LOG_WARN("fail to get inrow data", K(ret), K(lob)); } else { - ObString lob_loc_str; - new_tmp_lob.get_result_buffer(lob_loc_str); - LOG_INFO("Lob: new temp convert_text_value_charset in convert_text_value", - K(ret), K(raw_str), K(type), K(to_collation_type), K(from_collation_type), - K(from_charset_info->csname), K(to_charset_info->csname)); - value.set_lob_value(type, lob_loc_str.ptr(), lob_loc_str.length()); - value.set_collation_type(to_collation_type); - if (new_tmp_lob.has_lob_header()) { - value.set_has_lob_header(); + int64_t lob_data_byte_len = inrow_data.length(); + int64_t offset_len = reinterpret_cast(inrow_data.ptr()) - reinterpret_cast(lob.ptr_); + int64_t res_len = offset_len + lob_data_byte_len * ObCharset::CharConvertFactorNum; + char *buf = static_cast(alloc.alloc(res_len)); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc memory", K(ret), K(res_len)); + } else { + MEMCPY(buf, lob.ptr_, offset_len); + uint32_t result_len = 0; + if (OB_FAIL(convert_string_charset(inrow_data, from_collation_type, to_collation_type, + buf + offset_len, res_len - offset_len, result_len))) { + LOG_WARN("Lob: convert string charset failed", K(ret)); + } else { + // refresh payload size + if (lob.has_extern()) { + ObMemLobExternHeader *ex_header = nullptr; + if (OB_FAIL(lob.get_extern_header(ex_header))) { + LOG_WARN("fail to get extern header", K(ret), K(lob)); + } else { + ex_header->payload_size_ = result_len; + } + } + // refresh lob data size + if (OB_SUCC(ret)) { + ObLobCommon *lob_common = nullptr; + if (OB_FAIL(lob.get_disk_locator(lob_common))) { + LOG_WARN("fail to get lob common", K(ret), K(lob)); + } else if (lob_common->is_init_) { + ObLobData *lob_data = reinterpret_cast(lob_common->buffer_); + lob_data->byte_size_ = result_len; + } + } + if (OB_SUCC(ret)) { + LOG_INFO("Lob: new temp convert_text_value_charset in convert_text_value", + K(ret), K(raw_str), K(type), K(to_collation_type), K(from_collation_type), + K(from_charset_info->csname), K(to_charset_info->csname)); + value.set_lob_value(type, buf, offset_len + result_len); + value.set_collation_type(to_collation_type); + value.set_has_lob_header(); + } + } + } + } + } else { + // get full data, buffer size is full byte length * 4 + ObString data_str; + ObLobLocatorV2 loc(raw_str, value.has_lob_header()); + ObTextStringIter str_iter(value); + if (OB_FAIL(str_iter.init(0, &my_session, &alloc))) { + LOG_WARN("Lob: init lob str iter failed ", K(ret), K(value)); + } else if (OB_FAIL(str_iter.get_full_data(data_str))) { + LOG_WARN("Lob: init lob str iter failed ", K(ret), K(value)); + } else { + // mock result buffer and reserve data length + // could do streaming charset convert + ObTextStringResult new_tmp_lob(type, res_has_lob_header, &alloc); + char *buf = NULL; + int64_t buf_len = 0; + int64_t lob_data_byte_len = 0; + uint32_t result_len = 0; + + if (OB_FAIL(loc.get_lob_data_byte_len(lob_data_byte_len))) { + LOG_WARN("Lob: get lob data byte len failed", K(ret), K(loc)); + } else if (OB_FAIL(new_tmp_lob.init(lob_data_byte_len * 4))) { + LOG_WARN("Lob: init tmp lob failed", K(ret), K(lob_data_byte_len * 4)); + } else if (OB_FAIL(new_tmp_lob.get_reserved_buffer(buf, buf_len))) { + LOG_WARN("Lob: get empty buffer failed", K(ret), K(lob_data_byte_len * 4)); + } else if (OB_FAIL(convert_string_charset(data_str, from_collation_type, to_collation_type, + buf, buf_len, result_len))) { + LOG_WARN("convert string charset failed", K(ret)); + } else if (OB_FAIL(new_tmp_lob.lseek(result_len, 0))) { + LOG_WARN("temp lob lseek failed", K(ret)); + } else { + ObString lob_loc_str; + new_tmp_lob.get_result_buffer(lob_loc_str); + LOG_INFO("Lob: new temp convert_text_value_charset in convert_text_value", + K(ret), K(raw_str), K(type), K(to_collation_type), K(from_collation_type), + K(from_charset_info->csname), K(to_charset_info->csname)); + value.set_lob_value(type, lob_loc_str.ptr(), lob_loc_str.length()); + value.set_collation_type(to_collation_type); + if (new_tmp_lob.has_lob_header()) { + value.set_has_lob_header(); + } } } } @@ -441,7 +499,7 @@ int ObExprOutputPack::process_lob_locator_results(common::ObObj& value, // should not come here OB_ASSERT(0); } - if (!is_support_outrow_locator_v2 && !loc.is_inrow()) { + if (!is_support_outrow_locator_v2 && !loc.has_inrow_data()) { if (OB_FAIL(ObTextStringIter::append_outrow_lob_fulldata(value, &my_session, alloc))) { LOG_WARN("Lob: convert lob to outrow failed", K(value), K(GET_MIN_CLUSTER_VERSION())); }