diff --git a/deps/oblib/src/common/object/ob_object.h b/deps/oblib/src/common/object/ob_object.h index f88a230ba3..e57df27667 100644 --- a/deps/oblib/src/common/object/ob_object.h +++ b/deps/oblib/src/common/object/ob_object.h @@ -976,6 +976,14 @@ public: (loc->is_valid()) && (loc->is_mem_loc_ == 0); } + OB_INLINE bool is_inrow_disk_lob_locator() const + { + // Notice: should be called only when ptr_ is not null + ObLobCommon *loc = reinterpret_cast(ptr_); + return has_lob_header_ && (loc != nullptr) && (size_ >= sizeof(ObLobCommon)) && + (loc->is_valid()) && (loc->is_mem_loc_ == 0) && (loc->in_row_); + } + OB_INLINE bool is_valid(bool is_assert = true) const { bool bret = true; diff --git a/deps/oblib/src/lib/ob_define.h b/deps/oblib/src/lib/ob_define.h index 08522bcf00..5de4f46c60 100644 --- a/deps/oblib/src/lib/ob_define.h +++ b/deps/oblib/src/lib/ob_define.h @@ -1804,6 +1804,10 @@ const int64_t OB_MIN_LOB_INROW_THRESHOLD = 0; // 0 means disable inrow lob const int64_t OB_MAX_LOB_INROW_THRESHOLD = OB_MAX_USER_ROW_LENGTH / 2; // 1.5M/2 const int64_t OB_DEFAULT_LOB_INROW_THRESHOLD = 4096; // 4K +// this's used for ob_default_lob_inrow_threshold system variable +// used to set the default inrow threshold for newly created tables +const int64_t OB_SYS_VAR_DEFAULT_LOB_INROW_THRESHOLD = 8192; // 8K + const int64_t OB_MAX_CAST_CHAR_VARCHAR_LENGTH = 512; const int64_t OB_MAX_CAST_CHAR_TEXT_LENGTH = 16383; const int64_t OB_MAX_CAST_CHAR_MEDIUMTEXT_LENGTH = 4194303; diff --git a/src/observer/mysql/ob_query_driver.cpp b/src/observer/mysql/ob_query_driver.cpp index a2861423c8..37b0faa2bd 100644 --- a/src/observer/mysql/ob_query_driver.cpp +++ b/src/observer/mysql/ob_query_driver.cpp @@ -479,7 +479,7 @@ int ObQueryDriver::convert_text_value_charset(common::ObObj& value, sql::ObResul charset_type = nchar; } if (OB_FAIL(ret)) { - } else if (OB_FAIL(convert_text_value_charset(value, charset_type, *allocator, &my_session))) { + } else if (OB_FAIL(convert_text_value_charset(value, charset_type, *allocator, &my_session, &result.get_exec_context()))) { LOG_WARN("convert lob value fail.", K(ret), K(value)); } return ret; @@ -615,7 +615,8 @@ int ObQueryDriver::process_lob_locator_results(ObObj& value, sql::ObResultSet &r session_.is_client_use_lob_locator(), session_.is_client_support_lob_locatorv2(), allocator, - &result.get_session()))) { + &result.get_session(), + &result.get_exec_context()))) { LOG_WARN("convert lob to longtext fail.", K(ret), K(value)); } return ret; @@ -625,7 +626,8 @@ int ObQueryDriver::process_lob_locator_results(ObObj& value, bool is_use_lob_locator, bool is_support_outrow_locator_v2, ObIAllocator *allocator, - const sql::ObSQLSessionInfo *session_info) + const sql::ObSQLSessionInfo *session_info, + sql::ObExecContext *exec_ctx) { int ret = OB_SUCCESS; // 1. if client is_use_lob_locator, return lob locator @@ -699,7 +701,7 @@ int ObQueryDriver::process_lob_locator_results(ObObj& value, } else { // lob locator v2 ObArenaAllocator tmp_alloc("ObLobRead", OB_MALLOC_NORMAL_BLOCK_SIZE, session_info->get_effective_tenant_id()); ObTextStringIter instr_iter(value); - if (OB_FAIL(instr_iter.init(0, session_info, allocator, &tmp_alloc))) { + if (OB_FAIL(ObTextStringHelper::build_text_iter(instr_iter, exec_ctx, session_info, allocator, &tmp_alloc))) { LOG_WARN("init lob str inter failed", K(ret), K(value)); } else if (OB_FAIL(instr_iter.get_full_data(data))) { LOG_WARN("Lob: init lob str iter failed ", K(value)); @@ -809,7 +811,8 @@ int ObQueryDriver::convert_lob_value_charset(ObObj& value, int ObQueryDriver::convert_text_value_charset(ObObj& value, ObCharsetType charset_type, ObIAllocator &allocator, - const sql::ObSQLSessionInfo *session) + const sql::ObSQLSessionInfo *session, + sql::ObExecContext *exec_ctx) { int ret = OB_SUCCESS; ObString raw_str = value.get_string(); @@ -845,7 +848,8 @@ int ObQueryDriver::convert_text_value_charset(ObObj& value, session->is_client_use_lob_locator(), session->is_client_support_lob_locatorv2(), &allocator, - session))) { + session, + exec_ctx))) { LOG_WARN("fail to process lob locator", K(ret), K(value)); } else if (OB_FAIL(value.get_lob_locatorv2(lob))) { LOG_WARN("fail to lob locator v2", K(ret), K(value)); @@ -909,7 +913,7 @@ int ObQueryDriver::convert_text_value_charset(ObObj& value, } else { ObLobLocatorV2 loc(raw_str, value.has_lob_header()); ObTextStringIter str_iter(value); - if (OB_FAIL(str_iter.init(0, session, &allocator))) { + if (OB_FAIL(ObTextStringHelper::build_text_iter(str_iter, exec_ctx, session, &allocator))) { LOG_WARN("Lob: init lob str iter failed ", K(ret), K(value)); } else if (OB_FAIL(str_iter.get_full_data(data_str))) { LOG_WARN("Lob: get full data failed ", K(ret), K(value)); diff --git a/src/observer/mysql/ob_query_driver.h b/src/observer/mysql/ob_query_driver.h index 9ab5f2a0fb..e2672a7690 100644 --- a/src/observer/mysql/ob_query_driver.h +++ b/src/observer/mysql/ob_query_driver.h @@ -91,7 +91,8 @@ public: bool is_use_lob_locator, bool is_support_outrow_locator_v2, common::ObIAllocator *allocator, - const sql::ObSQLSessionInfo *session_info); + const sql::ObSQLSessionInfo *session_info, + sql::ObExecContext *exec_ctx = nullptr); static int convert_string_charset(const common::ObString &in_str, const common::ObCollationType in_cs_type, const common::ObCollationType out_cs_type, @@ -102,7 +103,8 @@ public: static int convert_text_value_charset(ObObj& value, ObCharsetType charset_type, ObIAllocator &allocator, - const sql::ObSQLSessionInfo *session_info); + const sql::ObSQLSessionInfo *session_info, + sql::ObExecContext *exec_ctx = nullptr); private: int convert_field_charset(common::ObIAllocator& allocator, const common::ObCollationType& from_collation, diff --git a/src/observer/mysql/obmp_base.cpp b/src/observer/mysql/obmp_base.cpp index 3ac0e7a035..be39373a0e 100644 --- a/src/observer/mysql/obmp_base.cpp +++ b/src/observer/mysql/obmp_base.cpp @@ -573,7 +573,7 @@ int ObMPBase::response_row(ObSQLSessionInfo &session, && OB_FAIL(ObQueryDriver::convert_lob_value_charset(value, charset_type, allocator))) { LOG_WARN("convert lob value charset failed", K(ret)); } else if (ob_is_text_tc(value.get_type()) - && OB_FAIL(ObQueryDriver::convert_text_value_charset(value, charset_type, allocator, &session))) { + && OB_FAIL(ObQueryDriver::convert_text_value_charset(value, charset_type, allocator, &session, exec_ctx))) { LOG_WARN("convert text value charset failed", K(ret)); } if (OB_FAIL(ret)) { @@ -581,7 +581,8 @@ int ObMPBase::response_row(ObSQLSessionInfo &session, session.is_client_use_lob_locator(), session.is_client_support_lob_locatorv2(), &allocator, - &session))) { + &session, + exec_ctx))) { LOG_WARN("convert lob locator to longtext failed", K(ret)); } else if ((value.is_user_defined_sql_type() || value.is_collection_sql_type() || value.is_geometry()) && OB_FAIL(ObXMLExprHelper::process_sql_udt_results(value, diff --git a/src/observer/table/ob_table_context.cpp b/src/observer/table/ob_table_context.cpp index 516f589f2c..eed32230b4 100644 --- a/src/observer/table/ob_table_context.cpp +++ b/src/observer/table/ob_table_context.cpp @@ -480,13 +480,13 @@ int ObTableCtx::cons_column_info(const ObColumnSchemaV2 &column_schema, int ObTableCtx::convert_lob(ObIAllocator &allocator, ObObj &obj) { int ret = OB_SUCCESS; - - if (obj.is_persist_lob()) { + ObLobLocatorV2 locator(obj.get_string(), obj.has_lob_header()); + if (obj.is_persist_lob() || locator.is_inrow_disk_lob_locator()) { // do nothing } else if (obj.has_lob_header()) { // we add lob header in write_datum ret = OB_ERR_UNEXPECTED; LOG_USER_ERROR(OB_ERR_UNEXPECTED, "lob object should not have lob header"); - LOG_WARN("object should not have lob header", K(ret), K(obj)); + LOG_WARN("object should not have lob header", K(ret), K(obj), K(locator)); } return ret; diff --git a/src/share/ob_lob_access_utils.cpp b/src/share/ob_lob_access_utils.cpp index 45c44b0fff..4ee178c603 100644 --- a/src/share/ob_lob_access_utils.cpp +++ b/src/share/ob_lob_access_utils.cpp @@ -67,7 +67,8 @@ ObTextStringIter::~ObTextStringIter() int ObTextStringIter::init(uint32_t buffer_len, const sql::ObBasicSessionInfo *session, ObIAllocator *res_allocator, - ObIAllocator *tmp_allocator) + ObIAllocator *tmp_allocator, + ObLobAccessCtx *lob_access_ctx) { int ret = OB_SUCCESS; if (is_init_) { @@ -102,6 +103,7 @@ int ObTextStringIter::init(uint32_t buffer_len, } else { ctx_ = new (ctx_buffer) ObLobTextIterCtx(locator, session, res_allocator, buffer_len); ctx_ ->init(); + ctx_->lob_access_ctx_ = lob_access_ctx; } } } @@ -119,68 +121,46 @@ static int init_lob_access_param(storage::ObLobAccessParam ¶m, { int ret = OB_SUCCESS; int64_t query_timeout = 0; - ObMemLobTxInfo *tx_info = NULL; - ObMemLobLocationInfo *location_info = NULL; - ObLobCommon *disk_loc = NULL; - ObString disk_loc_str; + int64_t timeout_ts = 0; + storage::ObLobManager* lob_mngr = MTL(storage::ObLobManager*); if (OB_ISNULL(lob_iter_ctx)) { ret = OB_INVALID_ARGUMENT; COMMON_LOG(WARN, "Lob: invalid lob iter ctx.", K(ret)); + } else if (OB_ISNULL(allocator = (allocator == nullptr ? lob_iter_ctx->alloc_: allocator))) { + ret = OB_INVALID_ARGUMENT; + COMMON_LOG(WARN, "Lob: allocator is null", K(ret), KP(allocator), KP(lob_iter_ctx->alloc_)); } else if (!lob_iter_ctx->locator_.is_persist_lob()) { ret = OB_NOT_IMPLEMENT; COMMON_LOG(WARN, "Lob: outrow temp lob is not supported", K(ret), K(lob_iter_ctx->locator_)); } else if (lob_iter_ctx->locator_.is_delta_temp_lob()) { ret = OB_INVALID_ARGUMENT; COMMON_LOG(WARN, "Lob: is delta lob", K(ret), K(lob_iter_ctx->locator_)); - } else if (OB_FAIL(lob_iter_ctx->locator_.get_disk_locator(disk_loc_str))) { - COMMON_LOG(WARN, "Lob: get disk locator failed.", K(ret)); - } else if (FALSE_IT(disk_loc = reinterpret_cast(disk_loc_str.ptr()))){ - } else if (OB_ISNULL(disk_loc) || disk_loc->is_init_ == false) { - ret = OB_INVALID_ARGUMENT; - COMMON_LOG(WARN, "Lob: invalid disk_locator.", K(ret), KP(disk_loc)); - } else { - if (OB_ISNULL(lob_iter_ctx->session_)) { - query_timeout = ObTimeUtility::current_time() + 60 * USECS_PER_SEC; - } else if (OB_FAIL(lob_iter_ctx->session_->get_query_timeout(query_timeout))) { - COMMON_LOG(WARN, "Lob: get_query_timeout failed.", K(ret), K(*lob_iter_ctx)); - } + } else if (OB_ISNULL(lob_iter_ctx->session_)) { + query_timeout = ObTimeUtility::current_time() + 60 * USECS_PER_SEC; + } else if (OB_FAIL(lob_iter_ctx->session_->get_query_timeout(query_timeout))) { + COMMON_LOG(WARN, "Lob: get_query_timeout failed.", K(ret), K(*lob_iter_ctx)); + } + + if (OB_SUCC(ret)) { + timeout_ts = (lob_iter_ctx->session_ == NULL) + ? query_timeout + : (lob_iter_ctx->session_->get_query_start_time() + query_timeout); } if (OB_FAIL(ret)) { - } else if (OB_FAIL(lob_iter_ctx->locator_.get_tx_info(tx_info))) { - } else if (OB_FAIL(lob_iter_ctx->locator_.get_location_info(location_info))) { - } else { - param.tx_desc_ = NULL; - param.snapshot_.core_.tx_id_ = tx_info->snapshot_tx_id_; - param.snapshot_.core_.version_.convert_for_tx(tx_info->snapshot_version_); - param.snapshot_.core_.scn_ = transaction::ObTxSEQ::cast_from_int(tx_info->snapshot_seq_); - param.snapshot_.valid_= true; - param.snapshot_.source_ = transaction::ObTxReadSnapshot::SRC::LS; - param.snapshot_.snapshot_lsid_ = share::ObLSID(location_info->ls_id_); - - param.ls_id_ = share::ObLSID(location_info->ls_id_); - param.sql_mode_ = (lob_iter_ctx->session_ == NULL) ? SMO_DEFAULT : lob_iter_ctx->session_->get_sql_mode(); - - param.tablet_id_ = ObTabletID(location_info->tablet_id_); - if (allocator == nullptr) { - allocator = lob_iter_ctx->alloc_; - } - param.allocator_ = allocator; - int64_t disk_loc_handle_size = disk_loc_str.length(); - param.lob_common_ = disk_loc; - param.handle_size_ = disk_loc_handle_size; - param.byte_size_ = param.lob_common_->get_byte_size(param.handle_size_); - param.coll_type_ = cs_type; - if (!lob_iter_ctx->locator_.is_lob_disk_locator()) { - param.lob_locator_ = &lob_iter_ctx->locator_; - } - param.timeout_ = (lob_iter_ctx->session_ == NULL) - ? query_timeout - : (lob_iter_ctx->session_->get_query_start_time() + query_timeout); - param.scan_backward_ = false; - param.offset_ = 0; // use 0 offset when reading full lob data - param.len_ = 0; + } else if (OB_FAIL(lob_mngr->build_lob_param(param, *allocator, cs_type, + 0, UINT64_MAX, timeout_ts, lob_iter_ctx->locator_))) { + LOG_WARN("build_lob_param fail", K(ret), K(*lob_iter_ctx)); + } else if (! param.snapshot_.core_.tx_id_.is_valid()) { + // if tx_id is valid, means read may be in a tx + // lob can not set read_latest flag + // so reuse lob aux table iterator only if tx_id is invalid + // for exmaple + // insert into t values (1,'v0'); + // insert ignore into t values (1,'v11'), (1,'v222') on duplicate key update c1 = md5(c1); + // second read shoud get "v11" not "v0" + param.access_ctx_ = lob_iter_ctx->lob_access_ctx_; } return ret; diff --git a/src/share/ob_lob_access_utils.h b/src/share/ob_lob_access_utils.h index 5c4848c9ab..c53f113fb3 100644 --- a/src/share/ob_lob_access_utils.h +++ b/src/share/ob_lob_access_utils.h @@ -30,6 +30,7 @@ namespace storage { class ObLobQueryIter; class ObLobDiffHeader; + class ObLobAccessCtx; } // namespace storage namespace common { @@ -68,7 +69,7 @@ struct ObLobTextIterCtx total_access_len_(0), total_byte_len_(0), content_byte_len_(0), content_len_(0), reserved_byte_len_(0), reserved_len_(0), accessed_byte_len_(0), accessed_len_(0), last_accessed_byte_len_(0), last_accessed_len_(0), iter_count_(0), is_cloned_temporary_(false), - is_backward_(false), locator_(locator), lob_query_iter_(NULL) + is_backward_(false), locator_(locator), lob_query_iter_(NULL), lob_access_ctx_(nullptr) {} TO_STRING_KV(KP_(alloc), KP_(session), KP_(buff), K_(buff_byte_len), K_(start_offset), K_(total_access_len), @@ -113,6 +114,7 @@ struct ObLobTextIterCtx ObLobLocatorV2 locator_; storage::ObLobQueryIter *lob_query_iter_; + storage::ObLobAccessCtx *lob_access_ctx_; }; // wrapper class to handle string/text type input @@ -151,7 +153,8 @@ public: int init(uint32_t buffer_len, const sql::ObBasicSessionInfo *session = NULL, ObIAllocator *res_allocator = NULL, - ObIAllocator *tmp_allocator = NULL); + ObIAllocator *tmp_allocator = NULL, + storage::ObLobAccessCtx *lob_access_ctx = NULL); ObTextStringIterState get_next_block(ObString &str); diff --git a/src/share/parameter/ob_parameter_seed.ipp b/src/share/parameter/ob_parameter_seed.ipp index d046e91713..48789fdb0b 100644 --- a/src/share/parameter/ob_parameter_seed.ipp +++ b/src/share/parameter/ob_parameter_seed.ipp @@ -1968,7 +1968,10 @@ DEF_BOOL(_enable_dbms_job_package, OB_CLUSTER_PARAMETER, "True", DEF_BOOL(_enable_kv_feature, OB_CLUSTER_PARAMETER, "True", "Enable or disable OBKV feature.", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); - +DEF_CAP(lob_enable_block_cache_threshold, OB_TENANT_PARAMETER, "256K", "[0B, 512M]", + "For outrow-stored LOBs, if the length is less than or equal to that threshold, " + "they can be admitted into the block cache to speed up the next query.", + ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); DEF_INT(_ob_pl_compile_max_concurrency, OB_CLUSTER_PARAMETER, "4", "[0,)", "The maximum number of threads that an observer node can compile PL concurrently.", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); diff --git a/src/share/schema/ob_schema_printer.cpp b/src/share/schema/ob_schema_printer.cpp index 9ccb56ecd1..0663e7a34c 100644 --- a/src/share/schema/ob_schema_printer.cpp +++ b/src/share/schema/ob_schema_printer.cpp @@ -5658,9 +5658,18 @@ int ObSchemaPrinter::print_table_definition_lob_params(const ObTableSchema &tabl int64_t& pos) const { int ret = OB_SUCCESS; - if (table_schema.get_lob_inrow_threshold() == OB_DEFAULT_LOB_INROW_THRESHOLD) { + uint64_t tenant_id = table_schema.get_tenant_id(); + uint64_t data_version = 0; + if (table_schema.is_sys_table() || table_schema.is_vir_table()) { + // skip for sys/vir table + } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) { + LOG_WARN("failed to get data version", K(ret), K(tenant_id)); + } else if (data_version < DATA_VERSION_4_3_2_0 && table_schema.get_lob_inrow_threshold() == OB_DEFAULT_LOB_INROW_THRESHOLD) { // if is default not display SHARE_SCHEMA_LOG(INFO, "default inrow threashold not display", K(ret), "lob inrow threshold", table_schema.get_lob_inrow_threshold()); + } else if (data_version >= DATA_VERSION_4_3_2_0 && table_schema.get_lob_inrow_threshold() == OB_SYS_VAR_DEFAULT_LOB_INROW_THRESHOLD) { + // if lob_inrow_threshold is OB_SYS_VAR_DEFAULT_LOB_INROW_THRESHOLD in new version, not display + SHARE_SCHEMA_LOG(INFO, "new default inrow threashold not display", K(ret), "lob inrow threshold", table_schema.get_lob_inrow_threshold()); } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "LOB_INROW_THRESHOLD=%ld ", table_schema.get_lob_inrow_threshold()))) { SHARE_SCHEMA_LOG(WARN, "fail to print lob inrow threshold", K(ret), K(table_schema)); } diff --git a/src/share/system_variable/ob_system_variable_init.cpp b/src/share/system_variable/ob_system_variable_init.cpp index 8e9be7435f..f2eab5729f 100644 --- a/src/share/system_variable/ob_system_variable_init.cpp +++ b/src/share/system_variable/ob_system_variable_init.cpp @@ -3416,7 +3416,7 @@ static struct VarsInit{ }(); [&] (){ - ObSysVars[242].default_value_ = "4096" ; + ObSysVars[242].default_value_ = "8192" ; ObSysVars[242].info_ = "default lob inrow threshold config" ; ObSysVars[242].name_ = "ob_default_lob_inrow_threshold" ; ObSysVars[242].data_type_ = ObIntType ; diff --git a/src/share/system_variable/ob_system_variable_init.json b/src/share/system_variable/ob_system_variable_init.json index 4ce09c198c..7744377985 100644 --- a/src/share/system_variable/ob_system_variable_init.json +++ b/src/share/system_variable/ob_system_variable_init.json @@ -3452,7 +3452,7 @@ "ob_default_lob_inrow_threshold": { "id": 10158, "name": "ob_default_lob_inrow_threshold", - "default_value": "4096", + "default_value": "8192", "base_value": "4096", "data_type": "int", "info": "default lob inrow threshold config", diff --git a/src/sql/engine/expr/ob_datum_cast.cpp b/src/sql/engine/expr/ob_datum_cast.cpp index af963ccc12..29deaac8e2 100644 --- a/src/sql/engine/expr/ob_datum_cast.cpp +++ b/src/sql/engine/expr/ob_datum_cast.cpp @@ -489,8 +489,7 @@ int ObDatumHexUtils::rawtohex(const ObExpr &expr, const ObString &in_str, case ObGeometryType: { ObString lob_data = in_str; if (OB_FAIL(ObTextStringHelper::read_real_string_data(&tmp_alloc, in_type, - expr.args_[0]->datum_meta_.cs_type_, expr.args_[0]->obj_meta_.has_lob_header(), - lob_data))) { + expr.args_[0]->obj_meta_.has_lob_header(), lob_data, &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(lob_data)); } else { out_str = lob_data; @@ -1790,7 +1789,7 @@ static int get_text_full_data(const sql::ObExpr &expr, if (ob_is_text_tc(in_type)) { bool has_lob_header = expr.args_[0]->obj_meta_.has_lob_header(); ObTextStringIter instr_iter(in_type, in_cs_type, in_datum->get_string(), has_lob_header); - if (OB_FAIL(instr_iter.init(0, ctx.exec_ctx_.get_my_session(), allocator))) { + if (OB_FAIL(ObTextStringHelper::build_text_iter(instr_iter, &ctx.exec_ctx_, ctx.exec_ctx_.get_my_session(), allocator))) { LOG_WARN("Lob: init lob str iter failed", K(ret), K(in_type), K(*in_datum)); } else if (OB_FAIL(instr_iter.get_full_data(data))) { LOG_WARN("Lob: get lob str iter full data failed ", K(ret), K(in_type), K(*in_datum)); @@ -2322,7 +2321,8 @@ static int common_json_string(const ObExpr &expr, ObCastMode cast_mode = expr.extra_; ObString j_bin_str = in.get_string(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(allocator, in, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), j_bin_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), + j_bin_str, &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(j_bin_str)); } else { ObJsonBin j_bin(j_bin_str.ptr(), j_bin_str.length(), &allocator); @@ -3766,7 +3766,8 @@ CAST_FUNC_NAME(string, json) ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(in_str)); } else { ret = common_string_json(expr, in_str, ctx, res_datum); @@ -3949,7 +3950,7 @@ CAST_FUNC_NAME(text, string) common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); ObExprStrResAlloc res_alloc(expr, ctx); ObTextStringIter instr_iter(in_type, in_cs_type, child_res->get_string(), has_lob_header); - if (OB_FAIL(instr_iter.init(0, ctx.exec_ctx_.get_my_session(), + if (OB_FAIL(ObTextStringHelper::build_text_iter(instr_iter, &ctx.exec_ctx_, ctx.exec_ctx_.get_my_session(), is_same_charset ? reinterpret_cast(&res_alloc) : &temp_allocator, &temp_allocator))) { LOG_WARN("init lob str iter failed ", K(ret), K(in_type)); @@ -4004,7 +4005,7 @@ CAST_FUNC_NAME(text, text) res_datum.set_string(in_str.ptr(), in_str.length()); } else { ObTextStringIter instr_iter(in_type, in_cs_type, in_str, has_lob_header); - if (OB_FAIL(instr_iter.init(0, ctx.exec_ctx_.get_my_session(), &temp_allocator))) { + if (OB_FAIL(ObTextStringHelper::build_text_iter(instr_iter, &ctx.exec_ctx_, ctx.exec_ctx_.get_my_session(), &temp_allocator))) { LOG_WARN("init lob str iter failed ", K(ret), K(in_type)); } else if (OB_FAIL(instr_iter.get_full_data(data))) { LOG_WARN("init lob str iter failed ", K(ret), K(in_type)); @@ -4093,7 +4094,7 @@ CAST_FUNC_NAME(text, lob) common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); ObTextStringIter instr_iter(in_type, in_cs_type, child_res->get_string(), has_lob_header); ObString data_str; - if (OB_FAIL(instr_iter.init(0, ctx.exec_ctx_.get_my_session(), &temp_allocator))) { + if (OB_FAIL(ObTextStringHelper::build_text_iter(instr_iter, &ctx.exec_ctx_, ctx.exec_ctx_.get_my_session(), &temp_allocator))) { LOG_WARN("init lob str iter failed ", K(ret), K(in_type)); } else if (OB_FAIL(instr_iter.get_full_data(data_str))) { LOG_WARN("init lob str iter failed ", K(ret), K(in_type)); @@ -4118,7 +4119,8 @@ CAST_FUNC_NAME(string, geometry) omt::ObSrsCacheGuard srs_guard; const ObSrsItem *srs = NULL; if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(in_str)); } else if (OB_FAIL(ObGeoExprUtils::get_srs_item(ctx, srs_guard, in_str, srs, true, cast_name))) { LOG_WARN("fail to get srs item", K(ret), K(in_str)); @@ -7946,7 +7948,8 @@ CAST_FUNC_NAME(json, int) ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), j_bin_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), j_bin_str, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(j_bin_str)); } else { ObJsonBin j_bin(j_bin_str.ptr(), j_bin_str.length(), &temp_allocator); @@ -7980,7 +7983,8 @@ CAST_FUNC_NAME(json, uint) ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), j_bin_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), j_bin_str, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(j_bin_str)); } else { ObJsonBin j_bin(j_bin_str.ptr(), j_bin_str.length(), &temp_allocator); @@ -8015,7 +8019,8 @@ CAST_FUNC_NAME(json, double) ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), j_bin_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), j_bin_str, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(j_bin_str)); } else { ObJsonBin j_bin(j_bin_str.ptr(), j_bin_str.length(), &temp_allocator); @@ -8049,7 +8054,8 @@ CAST_FUNC_NAME(json, float) ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), j_bin_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), j_bin_str, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(j_bin_str)); } else { ObJsonBin j_bin(j_bin_str.ptr(), j_bin_str.length(), &temp_allocator); @@ -8078,7 +8084,7 @@ static int common_json_number(common::ObDatum &child_res, const ObExpr &expr, ObString j_bin_str = child_res.get_string(); if (OB_FAIL(ObTextStringHelper::read_real_string_data( alloc, child_res, expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), - j_bin_str))) { + j_bin_str, &ctx.exec_ctx_))) { LOG_WARN("fail to get real data", K(ret), K(j_bin_str)); } else { ObJsonBin j_bin(j_bin_str.ptr(), j_bin_str.length(), &alloc); @@ -8108,7 +8114,7 @@ CAST_FUNC_NAME(json, number) number::ObNumber out_nmb; if (OB_FAIL(common_json_number(*child_res, expr, expr.datum_meta_.type_, ctx, temp_allocator, out_nmb))) { - LOG_WARN("json_number failed", K(ret)); + LOG_WARN("fail to get real data.", K(ret)); } else { res_datum.set_number(out_nmb); } @@ -8130,7 +8136,8 @@ CAST_FUNC_NAME(json, datetime) common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); ObTimeConvertCtx cvrt_ctx(session->get_timezone_info(), ObTimestampType == out_type); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), j_bin_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), j_bin_str, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(j_bin_str)); } else { ObJsonBin j_bin(j_bin_str.ptr(), j_bin_str.length(), &temp_allocator); @@ -8162,7 +8169,8 @@ CAST_FUNC_NAME(json, date) ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), j_bin_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), j_bin_str, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(j_bin_str)); } else { ObJsonBin j_bin(j_bin_str.ptr(), j_bin_str.length(), &temp_allocator); @@ -8193,7 +8201,8 @@ CAST_FUNC_NAME(json, time) ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), j_bin_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), j_bin_str, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(j_bin_str)); } else { ObJsonBin j_bin(j_bin_str.ptr(), j_bin_str.length(), &temp_allocator); @@ -8225,7 +8234,8 @@ CAST_FUNC_NAME(json, year) ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), j_bin_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), j_bin_str, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(j_bin_str)); } else { ObJsonBin j_bin(j_bin_str.ptr(), j_bin_str.length(), &temp_allocator); @@ -8308,7 +8318,8 @@ CAST_FUNC_NAME(json, string) common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); ObString j_bin_str = child_res->get_string(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), j_bin_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), j_bin_str, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(j_bin_str)); } else { ObJsonBin j_bin(j_bin_str.ptr(), j_bin_str.length(), &temp_allocator); @@ -8363,7 +8374,8 @@ CAST_FUNC_NAME(json, bit) ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), j_bin_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), j_bin_str, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(j_bin_str)); } else { ObJsonBin j_bin(j_bin_str.ptr(), j_bin_str.length(), &temp_allocator); @@ -8396,7 +8408,8 @@ CAST_FUNC_NAME(json, otimestamp) ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), j_bin_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), j_bin_str, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(j_bin_str)); } else { ObJsonBin j_bin(j_bin_str.ptr(), j_bin_str.length(), &temp_allocator); @@ -8551,7 +8564,8 @@ CAST_FUNC_NAME(geometry, int) ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(in_str)); } else if (OB_FAIL(common_string_int(expr, expr.extra_, in_str, true, res_datum))) { LOG_WARN("fail to cast string to int", K(ret)); @@ -8573,7 +8587,8 @@ CAST_FUNC_NAME(geometry, uint) ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(in_str)); } else if (OB_FAIL(common_string_uint(expr, in_str, true, res_datum))) { LOG_WARN("fail to cast string to uint", K(ret)); @@ -8597,7 +8612,8 @@ CAST_FUNC_NAME(geometry, double) common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); DEF_IN_OUT_TYPE(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(in_str)); } else if (OB_FAIL(common_string_double(expr, in_type, expr.args_[0]->datum_meta_.cs_type_, out_type, in_str, res_datum))) { LOG_WARN("fail to cast string to double", K(ret)); @@ -8620,7 +8636,8 @@ CAST_FUNC_NAME(geometry, float) ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(in_str)); } else if (OB_FAIL(common_string_float(expr, in_str, out_val))) { LOG_WARN("fail to cast string to float", K(ret)); @@ -8646,7 +8663,8 @@ CAST_FUNC_NAME(geometry, number) ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(in_str)); } else if (OB_FAIL(common_string_number(expr, in_str, tmp_alloc, nmb))) { LOG_WARN("fail to cast string to number", K(ret)); @@ -8672,7 +8690,8 @@ CAST_FUNC_NAME(geometry, datetime) ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(in_str)); } else if (OB_FAIL(common_string_datetime(expr, in_str, ctx, res_datum))) { LOG_WARN("fail to cast string to datetime", K(ret)); @@ -8693,7 +8712,8 @@ CAST_FUNC_NAME(geometry, date) ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(in_str)); } else if (OB_FAIL(common_string_date(expr, in_str, res_datum))) { LOG_WARN("fail to cast string to date", K(ret)); @@ -8717,7 +8737,8 @@ CAST_FUNC_NAME(geometry, time) ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(in_str)); } else if (OB_FAIL(common_string_time(expr, in_str, res_datum))) { LOG_WARN("fail to cast string to time", K(ret)); @@ -8738,7 +8759,8 @@ CAST_FUNC_NAME(geometry, year) ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(in_str)); } else if (OB_FAIL(common_string_year(expr, in_str, res_datum))) { LOG_WARN("fail to cast string to year", K(ret)); @@ -8760,7 +8782,8 @@ CAST_FUNC_NAME(geometry, bit) ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(in_str)); } else if (OB_FAIL(common_string_bit(expr, in_str, ctx, res_datum))) { LOG_WARN("fail to cast string to bit", K(ret)); @@ -8786,7 +8809,8 @@ CAST_FUNC_NAME(geometry, otimestamp) ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(in_str)); } else if (OB_FAIL(common_string_otimestamp(expr, in_str, ctx, res_datum))) { LOG_WARN("fail to cast string to otimestamp", K(ret)); @@ -8812,7 +8836,8 @@ CAST_FUNC_NAME(geometry, decimalint) ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); ObDecimalIntBuilder res_val; - if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str))) { + if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), in_str, &ctx.exec_ctx_))) { LOG_WARN("failed to get real data", K(ret), K(in_str)); } else if (OB_FAIL(common_string_decimalint(expr, in_str, ctx.exec_ctx_.get_user_logging_ctx(), res_val))) { @@ -8883,7 +8908,8 @@ CAST_FUNC_NAME(geometry, string) ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), wkb))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), wkb, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(wkb)); } else if (OB_FAIL(geom_copy_string(expr, wkb, ctx, res_datum))){ LOG_WARN("fail to copy string", K(ret)); @@ -8924,7 +8950,8 @@ CAST_FUNC_NAME(geometry, geometry) res_datum.set_string(wkb); } else { if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), wkb))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), wkb, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(wkb)); } else if (OB_FAIL(ObGeoExprUtils::get_srs_item(ctx, srs_guard, wkb, srs, true, cast_name))) { LOG_WARN("fail to get srs item", K(ret), K(wkb)); @@ -9140,7 +9167,8 @@ CAST_FUNC_NAME(udt, string) ObLongTextType, CS_TYPE_BINARY, true, - blob_data))) { + blob_data, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(blob_data)); } else if (OB_FAIL(ObXmlUtil::cast_to_string(blob_data, temp_allocator, xml_plain_text, session_cs_type))) { LOG_WARN("failed to convert xml to string", K(ret), KP(blob_data.ptr()), K(blob_data.length())); @@ -9239,7 +9267,8 @@ CAST_FUNC_NAME(pl_extend, string) ObLongTextType, CS_TYPE_BINARY, true, - blob_data))) { + blob_data, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(blob_data)); } else if (OB_FAIL(ObXmlUtil::cast_to_string(blob_data, temp_allocator, xml_plain_text, session_cs_type))) { LOG_WARN("failed to convert xml to string", K(ret), KP(blob_data.ptr()), K(blob_data.length())); @@ -9557,7 +9586,8 @@ CAST_FUNC_NAME(geometry, pl_extend) ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); if (OB_FAIL(ObTextStringHelper::read_real_string_data(temp_allocator, *child_res, - expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), wkb))) { + expr.args_[0]->datum_meta_, expr.args_[0]->obj_meta_.has_lob_header(), wkb, + &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(wkb)); } else if (OB_FAIL(pl::ObSdoGeometry::wkb_to_pl_extend(allocator, &ctx.exec_ctx_, wkb, result))) { LOG_WARN("failed to get geometry wkb from pl extend", K(ret)); diff --git a/src/sql/engine/expr/ob_expr_json_func_helper.cpp b/src/sql/engine/expr/ob_expr_json_func_helper.cpp index 0693b9928e..1364b3af1f 100644 --- a/src/sql/engine/expr/ob_expr_json_func_helper.cpp +++ b/src/sql/engine/expr/ob_expr_json_func_helper.cpp @@ -108,7 +108,7 @@ int ObJsonExprHelper::get_json_or_str_data(ObExpr *expr, ObEvalCtx &ctx, ret = OB_ERR_INVALID_TYPE_FOR_OP; LOG_WARN("input type error", K(val_type)); } else if (OB_FAIL(ObTextStringHelper::read_real_string_data(allocator, *json_datum, - expr->datum_meta_, expr->obj_meta_.has_lob_header(), str))) { + expr->datum_meta_, expr->obj_meta_.has_lob_header(), str, &ctx.exec_ctx_))) { LOG_WARN("fail to get real data.", K(ret), K(str)); } return ret; @@ -351,7 +351,7 @@ int ObJsonExprHelper::get_json_for_partial_update( LOG_WARN("get real data fail", KR(ret), K(json_datum), K(json_expr)); } } else if (OB_FAIL(ObTextStringHelper::read_real_string_data(allocator, json_datum, - json_expr.datum_meta_, json_expr.obj_meta_.has_lob_header(), j_str))) { + json_expr.datum_meta_, json_expr.obj_meta_.has_lob_header(), j_str, &ctx.exec_ctx_))) { LOG_WARN("get real data fail", KR(ret), K(json_datum), K(json_expr)); } if (OB_FAIL(ret)) { diff --git a/src/sql/engine/expr/ob_expr_lob_utils.cpp b/src/sql/engine/expr/ob_expr_lob_utils.cpp index 5e2e1f3993..2d05f8d30b 100644 --- a/src/sql/engine/expr/ob_expr_lob_utils.cpp +++ b/src/sql/engine/expr/ob_expr_lob_utils.cpp @@ -64,6 +64,66 @@ void ObTextStringObObjResult::set_result() } } +int ObTextStringHelper::build_text_iter( + ObTextStringIter &text_iter, + ObExecContext *exec_ctx, + const sql::ObBasicSessionInfo *session, + ObIAllocator *res_allocator, + ObIAllocator *tmp_allocator) +{ + int ret = OB_SUCCESS; + ObLobAccessCtx *lob_access_ctx = nullptr; + if (OB_NOT_NULL(exec_ctx) && OB_FAIL(exec_ctx->get_lob_access_ctx(lob_access_ctx))) { + LOG_WARN("get_lob_access_ctx fail", K(ret)); + } else if (OB_FAIL(text_iter.init(0/*buffer_len*/, session, res_allocator, tmp_allocator, lob_access_ctx))) { + LOG_WARN("init lob str iter fail", K(ret), K(text_iter)); + } + return ret; +} + +int ObTextStringHelper::read_real_string_data( + ObIAllocator *allocator, + ObObjType type, + ObCollationType cs_type, + bool has_lob_header, + ObString &str, + sql::ObExecContext *exec_ctx) +{ + int ret = OB_SUCCESS; + if (is_lob_storage(type)) { + ObTextStringIter str_iter(type, cs_type, str, has_lob_header); + if (OB_FAIL(build_text_iter(str_iter, exec_ctx, nullptr/*session*/, allocator))) { + LOG_WARN("Lob: init lob str iter failed ", K(ret), K(str_iter)); + } else if (OB_FAIL(str_iter.get_full_data(str))) { + COMMON_LOG(WARN, "Lob: str iter get full data failed ", K(ret), K(str_iter)); + } + } + return ret; +} + +int ObTextStringHelper::read_real_string_data( + ObIAllocator *allocator, + const common::ObObj &obj, + ObString &str, + sql::ObExecContext *exec_ctx) +{ + int ret = OB_SUCCESS; + const ObObjMeta& meta = obj.get_meta(); + str = obj.get_string(); + if (meta.is_null()) { + str.reset(); + } else if (OB_FAIL(read_real_string_data( + allocator, + meta.get_type(), + meta.get_collation_type(), + obj.has_lob_header(), + str, + exec_ctx))) { + COMMON_LOG(WARN, "read_real_string_data fail", K(ret)); + } + return ret; +} + int ob_adjust_lob_datum(const ObObj &origin_obj, const common::ObObjMeta &obj_meta, const ObObjDatumMapType &obj_datum_map_, diff --git a/src/sql/engine/expr/ob_expr_lob_utils.h b/src/sql/engine/expr/ob_expr_lob_utils.h index bc8bea3251..82088217ca 100644 --- a/src/sql/engine/expr/ob_expr_lob_utils.h +++ b/src/sql/engine/expr/ob_expr_lob_utils.h @@ -195,7 +195,7 @@ public: }; static int read_real_string_data(ObIAllocator &allocator, const ObDatum &datum, const ObDatumMeta &meta, - bool has_lob_header, ObString &str) + bool has_lob_header, ObString &str, sql::ObExecContext *exec_ctx = nullptr) { int ret = OB_SUCCESS; str = datum.get_string(); @@ -222,11 +222,15 @@ public: } } } else { - ObTextStringIter str_iter(meta.type_, meta.cs_type_, str, has_lob_header); - if (OB_FAIL(str_iter.init(0, NULL, &allocator))) { - COMMON_LOG(WARN, "Lob: str iter init failed ", K(ret), K(str_iter)); - } else if (OB_FAIL(str_iter.get_full_data(str))) { - COMMON_LOG(WARN, "Lob: str iter get full data failed ", K(ret), K(str_iter)); + // read outrow lob data + if (OB_FAIL(read_real_string_data( + &allocator, + meta.type_, + meta.cs_type_, + has_lob_header, + str, + exec_ctx))) { + COMMON_LOG(WARN, "read_real_string_data fail", K(ret), K(meta), K(has_lob_header), K(datum)); } } } @@ -237,7 +241,7 @@ public: template static int read_real_string_data(ObIAllocator &allocator, const VectorType &vector, const ObDatumMeta &meta, bool has_lob_header, ObString &str, - const int64_t idx) + const int64_t idx, sql::ObExecContext *exec_ctx = nullptr) { int ret = OB_SUCCESS; str = vector->get_string(idx); @@ -269,11 +273,15 @@ public: } } } else { - ObTextStringIter str_iter(meta.type_, meta.cs_type_, str, has_lob_header); - if (OB_FAIL(str_iter.init(0, NULL, &allocator))) { - COMMON_LOG(WARN, "Lob: str iter init failed ", K(ret), K(str_iter)); - } else if (OB_FAIL(str_iter.get_full_data(str))) { - COMMON_LOG(WARN, "Lob: str iter get full data failed ", K(ret), K(str_iter)); + // read outrow lob data + if (OB_FAIL(read_real_string_data( + &allocator, + meta.type_, + meta.cs_type_, + has_lob_header, + str, + exec_ctx))) { + COMMON_LOG(WARN, "read_real_string_data fail", K(ret), K(meta), K(has_lob_header), K(vector)); } } } @@ -281,38 +289,30 @@ public: return ret; }; - static int read_real_string_data(ObIAllocator *allocator, const common::ObObj &obj, ObString &str) + static int read_real_string_data( + ObIAllocator *allocator, + const common::ObObj &obj, + ObString &str, + sql::ObExecContext *exec_ctx = nullptr); + + static int read_real_string_data( + ObIAllocator *allocator, + ObObjType type, + ObCollationType cs_type, + bool has_lob_header, + ObString &str, + sql::ObExecContext *exec_ctx = nullptr); + + static int read_real_string_data( + ObIAllocator *allocator, + ObObjType type, + bool has_lob_header, + ObString &str, + sql::ObExecContext *exec_ctx) { - int ret = OB_SUCCESS; - const ObObjMeta& meta = obj.get_meta(); - str = obj.get_string(); - if (meta.is_null()) { - str.reset(); - } else if (is_lob_storage(meta.get_type())) { - ObTextStringIter str_iter(meta.get_type(), meta.get_collation_type(), str, obj.has_lob_header()); - if (OB_FAIL(str_iter.init(0, NULL, allocator))) { - COMMON_LOG(WARN, "Lob: init lob str iter failed ", K(ret), K(str_iter)); - } else if (OB_FAIL(str_iter.get_full_data(str))) { - COMMON_LOG(WARN, "Lob: str iter get full data failed ", K(ret), K(str_iter)); - } - } - return ret; + return read_real_string_data(allocator, type, CS_TYPE_BINARY, has_lob_header, str, exec_ctx); } - static int read_real_string_data(ObIAllocator *allocator, ObObjType type, ObCollationType cs_type, - bool has_lob_header, ObString &str) - { - int ret = OB_SUCCESS; - if (is_lob_storage(type)) { - ObTextStringIter str_iter(type, cs_type, str, has_lob_header); - if (OB_FAIL(str_iter.init(0, NULL, allocator))) { - COMMON_LOG(WARN, "Lob: init lob str iter failed ", K(ret), K(str_iter)); - } else if (OB_FAIL(str_iter.get_full_data(str))) { - COMMON_LOG(WARN, "Lob: str iter get full data failed ", K(ret), K(str_iter)); - } - } - return ret; - } // get outrow lob prefix or inrow/string tc full data static int read_prefix_string_data(ObEvalCtx &ctx, @@ -410,7 +410,12 @@ public: return ret; } - + static int build_text_iter( + ObTextStringIter &text_iter, + sql::ObExecContext *exec_ctx, + const sql::ObBasicSessionInfo *session = NULL, + ObIAllocator *res_allocator = NULL, + ObIAllocator *tmp_allocator = NULL); }; diff --git a/src/sql/engine/expr/ob_expr_output_pack.cpp b/src/sql/engine/expr/ob_expr_output_pack.cpp index b597b3ba95..d3dcdfff14 100644 --- a/src/sql/engine/expr/ob_expr_output_pack.cpp +++ b/src/sql/engine/expr/ob_expr_output_pack.cpp @@ -325,7 +325,8 @@ int ObExprOutputPack::convert_lob_value_charset(common::ObObj &value, int ObExprOutputPack::convert_text_value_charset(common::ObObj& value, bool res_has_lob_header, common::ObIAllocator &alloc, - const ObSQLSessionInfo &my_session) + const ObSQLSessionInfo &my_session, + sql::ObExecContext &exec_ctx) { int ret = OB_SUCCESS; ObCharsetType charset_type = CHARSET_INVALID; @@ -366,7 +367,7 @@ int ObExprOutputPack::convert_text_value_charset(common::ObObj& value, my_session.is_client_use_lob_locator() && lib::is_oracle_mode()) { ObLobLocatorV2 lob; ObString inrow_data; - if (OB_FAIL(process_lob_locator_results(value, alloc, my_session))) { + if (OB_FAIL(process_lob_locator_results(value, alloc, my_session, exec_ctx))) { LOG_WARN("fail to process lob locator", K(ret), K(value)); } else if (OB_FAIL(value.get_lob_locatorv2(lob))) { LOG_WARN("fail to lob locator v2", K(ret), K(value)); @@ -426,7 +427,7 @@ int ObExprOutputPack::convert_text_value_charset(common::ObObj& value, ObString data_str; ObLobLocatorV2 loc(raw_str, value.has_lob_header()); ObTextStringIter str_iter(value); - if (OB_FAIL(str_iter.init(0, &my_session, &alloc))) { + if (OB_FAIL(ObTextStringHelper::build_text_iter(str_iter, &exec_ctx, &my_session, &alloc))) { LOG_WARN("Lob: init lob str iter failed ", K(ret), K(value)); } else if (OB_FAIL(str_iter.get_full_data(data_str))) { LOG_WARN("Lob: init lob str iter failed ", K(ret), K(value)); @@ -514,7 +515,8 @@ int ObExprOutputPack::convert_lob_locator_to_longtext(common::ObObj &value, int ObExprOutputPack::process_lob_locator_results(common::ObObj& value, common::ObIAllocator &alloc, - const ObSQLSessionInfo &my_session) + const ObSQLSessionInfo &my_session, + sql::ObExecContext &exec_ctx) { int ret = OB_SUCCESS; // 1. if client is_use_lob_locator, return lob locator @@ -581,7 +583,7 @@ int ObExprOutputPack::process_lob_locator_results(common::ObObj& value, } } else { // lob locator v2 ObTextStringIter instr_iter(value); - if (OB_FAIL(instr_iter.init(0, &my_session, &alloc))) { + if (OB_FAIL(ObTextStringHelper::build_text_iter(instr_iter, &exec_ctx, &my_session, &alloc))) { LOG_WARN("init lob str inter failed", K(ret), K(value)); } else if (OB_FAIL(instr_iter.get_full_data(data))) { LOG_WARN("Lob: init lob str iter failed ", K(value)); @@ -677,12 +679,12 @@ int ObExprOutputPack::try_encode_row(const ObExpr &expr, ObEvalCtx &ctx, && OB_FAIL(convert_lob_value_charset(obj, alloc, *session))) { LOG_WARN("convert lob obj charset failed", K(ret)); } else if (ob_is_text_tc(obj.get_type()) - && OB_FAIL(convert_text_value_charset(obj, expr.obj_meta_.has_lob_header(), alloc, *session))) { + && OB_FAIL(convert_text_value_charset(obj, expr.obj_meta_.has_lob_header(), alloc, *session, ctx.exec_ctx_))) { LOG_WARN("convert text obj charset failed", K(ret)); } if (OB_FAIL(ret)) { } else if ((obj.is_lob() || obj.is_lob_locator() || obj.is_json() || obj.is_geometry()) - && OB_FAIL(process_lob_locator_results(obj, alloc, *session))) { + && OB_FAIL(process_lob_locator_results(obj, alloc, *session, ctx.exec_ctx_))) { LOG_WARN("convert lob locator to longtext failed", K(ret)); } else if ((obj.is_user_defined_sql_type() || obj.is_collection_sql_type() || obj.is_geometry()) && OB_FAIL(ObXMLExprHelper::process_sql_udt_results(obj, &alloc, session, diff --git a/src/sql/engine/expr/ob_expr_output_pack.h b/src/sql/engine/expr/ob_expr_output_pack.h index 4e106bbe97..833c38292a 100644 --- a/src/sql/engine/expr/ob_expr_output_pack.h +++ b/src/sql/engine/expr/ob_expr_output_pack.h @@ -70,10 +70,12 @@ private: static int convert_text_value_charset(common::ObObj& value, bool res_has_lob_header, common::ObIAllocator &alloc, - const ObSQLSessionInfo &my_session); + const ObSQLSessionInfo &my_session, + sql::ObExecContext &exec_ctx); static int process_lob_locator_results(common::ObObj& value, common::ObIAllocator &alloc, - const ObSQLSessionInfo &my_session); + const ObSQLSessionInfo &my_session, + sql::ObExecContext &exec_ctx); static int convert_string_charset(const common::ObString &in_str, const ObCollationType in_cs_type, diff --git a/src/sql/engine/expr/ob_expr_substr.cpp b/src/sql/engine/expr/ob_expr_substr.cpp index 2135a4d5fe..43a6539dcb 100644 --- a/src/sql/engine/expr/ob_expr_substr.cpp +++ b/src/sql/engine/expr/ob_expr_substr.cpp @@ -691,7 +691,7 @@ int ObExprSubstr::eval_substr(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &expr_ ObTextStringIter input_iter(input_meta.type_, input_meta.cs_type_, str_datum->get_string(), has_lob_header); ObTextStringDatumResult output_result(expr.datum_meta_.type_, &expr, &ctx, &expr_datum); int64_t total_byte_len = 0; - if (OB_FAIL(input_iter.init(0, NULL, &calc_alloc))) { + if (OB_FAIL(ObTextStringHelper::build_text_iter(input_iter, &ctx.exec_ctx_, NULL/*session*/, &calc_alloc))) { LOG_WARN("init input_iter failed ", K(ret), K(input_iter)); } else if (OB_FAIL(input_iter.get_byte_len(total_byte_len))) { LOG_WARN("get input byte len failed", K(ret)); @@ -811,7 +811,7 @@ int ObExprSubstr::eval_substr_batch(const ObExpr &expr, ObEvalCtx &ctx, ObTextStringIter input_iter(input_meta.type_, input_meta.cs_type_, datum_array[j].get_string(), has_lob_header); ObTextStringDatumResult output_result(expr.datum_meta_.type_, &expr, &ctx, &results[j]); int64_t total_byte_len = 0; - if (OB_FAIL(input_iter.init(0, NULL, &calc_alloc))) { + if (OB_FAIL(ObTextStringHelper::build_text_iter(input_iter, &ctx.exec_ctx_, NULL/*session*/, &calc_alloc))) { LOG_WARN("init input_iter failed ", K(ret), K(input_iter)); } else if (OB_FAIL(input_iter.get_byte_len(total_byte_len))) { LOG_WARN("get input byte len failed", K(ret), K(j)); diff --git a/src/sql/engine/ob_exec_context.cpp b/src/sql/engine/ob_exec_context.cpp index 8098537a1a..7f50991227 100644 --- a/src/sql/engine/ob_exec_context.cpp +++ b/src/sql/engine/ob_exec_context.cpp @@ -25,6 +25,7 @@ #include "share/interrupt/ob_global_interrupt_call.h" #include "ob_operator.h" #include "observer/ob_server.h" +#include "storage/lob/ob_lob_persistent_reader.h" #ifdef OB_BUILD_SPM #include "sql/spm/ob_spm_controller.h" #endif @@ -130,7 +131,8 @@ ObExecContext::ObExecContext(ObIAllocator &allocator) errcode_(OB_SUCCESS), dblink_snapshot_map_(), user_logging_ctx_(), - is_online_stats_gathering_(false) + is_online_stats_gathering_(false), + lob_access_ctx_(nullptr) { } @@ -194,6 +196,11 @@ ObExecContext::~ObExecContext() } update_columns_ = nullptr; errcode_ = OB_SUCCESS; + + if (OB_NOT_NULL(lob_access_ctx_)) { + lob_access_ctx_->~ObLobAccessCtx(); + lob_access_ctx_ = nullptr; + } } void ObExecContext::clean_resolve_ctx() @@ -1064,5 +1071,21 @@ int ObExecContext::get_subschema_id_by_udt_id(uint64_t udt_type_id, return ret; } + +int ObExecContext::get_lob_access_ctx(ObLobAccessCtx *&lob_access_ctx) +{ + int ret = OB_SUCCESS; + ObIAllocator &allocator = get_allocator(); + if (OB_NOT_NULL(lob_access_ctx_)) { + lob_access_ctx = lob_access_ctx_; + } else if (OB_ISNULL(lob_access_ctx_ = OB_NEWx(ObLobAccessCtx, &allocator))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc", K(ret), "size", sizeof(ObLobAccessCtx)); + } else { + lob_access_ctx = lob_access_ctx_; + } + return ret; +} + } // namespace sql } // namespace oceanbase diff --git a/src/sql/engine/ob_exec_context.h b/src/sql/engine/ob_exec_context.h index 5a2d003414..cdf862ea62 100644 --- a/src/sql/engine/ob_exec_context.h +++ b/src/sql/engine/ob_exec_context.h @@ -64,6 +64,11 @@ namespace common class ObMySQLProxy; } +namespace storage +{ +class ObLobAccessCtx; +} + namespace pl { class ObPL; @@ -514,6 +519,9 @@ public: int get_local_var_array(int64_t local_var_array_id, const ObSolidifiedVarsContext *&var_array); void set_is_online_stats_gathering(bool v) { is_online_stats_gathering_ = v; } bool is_online_stats_gathering() const { return is_online_stats_gathering_; } + + int get_lob_access_ctx(ObLobAccessCtx *&lob_access_ctx); + private: int build_temp_expr_ctx(const ObTempExpr &temp_expr, ObTempExprCtx *&temp_expr_ctx); int set_phy_op_ctx_ptr(uint64_t index, void *phy_op); @@ -693,6 +701,8 @@ protected: // for online stats gathering bool is_online_stats_gathering_; //--------------- + + ObLobAccessCtx *lob_access_ctx_; private: DISALLOW_COPY_AND_ASSIGN(ObExecContext); }; diff --git a/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp b/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp index c59d9d8688..a9d6ec1c9a 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp +++ b/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp @@ -374,11 +374,19 @@ int ObCreateTableResolverBase::set_table_option_to_schema(ObTableSchema &table_s if (OB_SUCC(ret)) { // if lob_inrow_threshold not set, used config default_lob_inrow_threshold + uint64_t tenant_data_version = 0; if (is_set_lob_inrow_threshold_) { table_schema.set_lob_inrow_threshold(lob_inrow_threshold_); } else if (OB_ISNULL(session_info_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("session if NULL", K(ret)); + } else if (OB_FAIL(GET_MIN_DATA_VERSION(session_info_->get_effective_tenant_id(), tenant_data_version))) { + LOG_WARN("get tenant data version failed", K(ret)); + } else if (tenant_data_version < DATA_VERSION_4_2_1_2){ + // lob_inrow_threshold is added in 421 bp2 + // so need ensure lob_inrow_threshold is 4096 before 421 bp2 for compat + lob_inrow_threshold_ = OB_DEFAULT_LOB_INROW_THRESHOLD; + table_schema.set_lob_inrow_threshold(lob_inrow_threshold_); } else if (OB_FALSE_IT((lob_inrow_threshold_ = session_info_->get_default_lob_inrow_threshold()))) { } else if (lob_inrow_threshold_ < OB_MIN_LOB_INROW_THRESHOLD || lob_inrow_threshold_ > OB_MAX_LOB_INROW_THRESHOLD) { ret = OB_INVALID_ARGUMENT; diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.cpp b/src/sql/resolver/ddl/ob_ddl_resolver.cpp index cfe0b1284c..bc9233c21e 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.cpp +++ b/src/sql/resolver/ddl/ob_ddl_resolver.cpp @@ -4973,6 +4973,7 @@ void ObDDLResolver::reset() { hash_subpart_num_ = -1; ttl_definition_.reset(); kv_attributes_.reset(); + is_set_lob_inrow_threshold_ = false; lob_inrow_threshold_ = OB_DEFAULT_LOB_INROW_THRESHOLD; } diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index c01f315105..d1b639bd8f 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -804,6 +804,10 @@ ob_set_subtarget(ob_storage lob lob/ob_lob_util.cpp lob/ob_lob_rpc_struct.cpp lob/ob_ext_info_callback.cpp + lob/ob_lob_access_param.cpp + lob/ob_lob_persistent_reader.cpp + lob/ob_lob_persistent_iterator.cpp + lob/ob_lob_write_buffer.cpp ) ob_set_subtarget(ob_storage mview diff --git a/src/storage/lob/ob_lob_access_param.cpp b/src/storage/lob/ob_lob_access_param.cpp new file mode 100644 index 0000000000..ba720a46c8 --- /dev/null +++ b/src/storage/lob/ob_lob_access_param.cpp @@ -0,0 +1,135 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "ob_lob_util.h" +#include "storage/lob/ob_lob_meta.h" + +namespace oceanbase +{ +namespace storage +{ + +bool ObLobAccessParam::has_single_chunk() const +{ + int ret = OB_SUCCESS; + bool res = false; + int64_t chunk_size = 0; + if (! lib::is_mysql_mode() || byte_size_ <= 0) { + // skip + } else if (OB_FAIL(get_store_chunk_size(chunk_size))) { + LOG_WARN("get_store_chunk_size fail", K(ret), KPC(this)); + } else if (byte_size_ <= chunk_size) { + res = true; + } + return res; +} + +bool ObLobAccessParam::enable_block_cache() const +{ + bool res = false; + omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id_)); + if (!tenant_config.is_valid()) { + res = false; + } else { + res = byte_size_ <= tenant_config->lob_enable_block_cache_threshold; + } + return res; +} + +int build_rowkey_range(ObLobAccessParam ¶m, ObRowkey &min_row_key, ObRowkey &max_row_key, ObNewRange &range) +{ + int ret = OB_SUCCESS; + range.table_id_ = 0; // make sure this is correct + range.start_key_ = min_row_key; + range.end_key_ = max_row_key; + range.border_flag_.set_inclusive_start(); + range.border_flag_.set_inclusive_end(); + return ret; +} + +int build_rowkey_range(ObLobAccessParam ¶m, ObObj key_objs[4], ObNewRange &range) +{ + const char *lob_id_ptr = reinterpret_cast(¶m.lob_data_->id_); + key_objs[0].reset(); + key_objs[0].set_varchar(lob_id_ptr, sizeof(ObLobId)); // lob_id + key_objs[0].set_collation_type(common::ObCollationType::CS_TYPE_BINARY); + key_objs[1] = ObObj::make_min_obj(); // seq_id set min + ObRowkey min_row_key(key_objs, 2); + + key_objs[2].reset(); + key_objs[2].set_varchar(lob_id_ptr, sizeof(ObLobId)); // lob_id + key_objs[2].set_collation_type(common::ObCollationType::CS_TYPE_BINARY); + key_objs[3] = ObObj::make_max_obj(); // seq_id set max + ObRowkey max_row_key(key_objs + 2, 2); + return build_rowkey_range(param, min_row_key, max_row_key, range); +} + +int build_rowkey(ObLobAccessParam ¶m, ObObj key_objs[4], ObString &seq_id, ObNewRange &range) +{ + int ret = OB_SUCCESS; + const char *lob_id_ptr = reinterpret_cast(¶m.lob_data_->id_); + key_objs[0].reset(); + key_objs[0].set_varchar(lob_id_ptr, sizeof(ObLobId)); // lob_id + key_objs[0].set_collation_type(common::ObCollationType::CS_TYPE_BINARY); + key_objs[1].set_varchar(seq_id); // seq_id set min + key_objs[1].set_collation_type(common::ObCollationType::CS_TYPE_BINARY); + ObRowkey min_row_key(key_objs, 2); + + key_objs[2].reset(); + key_objs[2].set_varchar(lob_id_ptr, sizeof(ObLobId)); // lob_id + key_objs[2].set_collation_type(common::ObCollationType::CS_TYPE_BINARY); + key_objs[3].set_varchar(seq_id); // seq_id set max + key_objs[3].set_collation_type(common::ObCollationType::CS_TYPE_BINARY); + ObRowkey max_row_key(key_objs + 2, 2); + + return build_rowkey_range(param, min_row_key, max_row_key, range); +} + +int build_rowkey(ObLobAccessParam ¶m, ObObj key_objs[4], ObNewRange &range) +{ + int ret = OB_SUCCESS; + void *seq_id_buf = nullptr; + ObString seq_id; + if (OB_ISNULL(seq_id_buf = param.allocator_->alloc(sizeof(uint32_t)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc range obj failed.", K(ret)); + } else if (OB_FALSE_IT(seq_id.assign_ptr((char*)seq_id_buf, sizeof(uint32_t)))) { + } else if (OB_FAIL(ObLobSeqId::get_seq_id(0, seq_id))) { + LOG_WARN("get_seq_id failed.", K(ret)); + } else if (OB_FAIL(build_rowkey(param, key_objs, seq_id, range))) { + LOG_WARN("build_rowkey_range fail", K(ret)); + } + return ret; +} + +int build_range(ObLobAccessParam ¶m, ObObj key_objs[4], ObNewRange &range) +{ + int ret = OB_SUCCESS; + if (param.has_single_chunk()) { + if (OB_FAIL(build_rowkey(param, key_objs, range))) { + LOG_WARN("build_rowkey fail", K(ret)); + } + } else if (OB_FAIL(build_rowkey_range(param, key_objs, range))) { + LOG_WARN("build_rowkey_range fail", K(ret)); + } + return ret; +} + +int ObLobAccessParam::get_rowkey_range(ObObj rowkey_objs[4], ObNewRange &range) +{ + return build_range(*this, rowkey_objs, range); +} + +} // storage +} // oceanbase diff --git a/src/storage/lob/ob_lob_locator.cpp b/src/storage/lob/ob_lob_locator.cpp index 940f125416..9c7a9a5458 100644 --- a/src/storage/lob/ob_lob_locator.cpp +++ b/src/storage/lob/ob_lob_locator.cpp @@ -212,6 +212,19 @@ int ObLobLocatorHelper::fill_lob_locator(ObDatumRow &row, return ret; } +bool ObLobLocatorHelper::can_skip_build_mem_lob_locator(const common::ObString &payload) +{ + int bret = false; + const ObLobCommon *lob_common = reinterpret_cast(payload.ptr()); + if (payload.length() == 0) { + // do nothing + } else if (lob_common->in_row_ && lib::is_mysql_mode()) { + // mysql mode inrow lob can skip build mem lob locator + bret = true; + } + return bret; +} + int ObLobLocatorHelper::fill_lob_locator_v2(ObDatumRow &row, const ObTableAccessContext &access_ctx, const ObTableAccessParam &access_param) @@ -240,6 +253,11 @@ int ObLobLocatorHelper::fill_lob_locator_v2(ObDatumRow &row, ObLobLocatorV2 locator; if (datum_meta.is_lob_storage()) { if (datum.is_null() || datum.is_nop()) { + // read sys table is changed to mysql mode for normal oracle tenant + // and that may return disk lob lob locator to jdbc + // and cause jdbc error because jdbc can not handle disk lob locator + // so sys table can not skip build mem lob locator + } else if (! is_sys_table(access_param.iter_param_.table_id_) && can_skip_build_mem_lob_locator(datum.get_string())) { } else if (OB_FAIL(build_lob_locatorv2(locator, datum.get_string(), out_cols_param->at(i)->get_column_id(), diff --git a/src/storage/lob/ob_lob_locator.h b/src/storage/lob/ob_lob_locator.h index e802a8f4d8..6c3260a67b 100644 --- a/src/storage/lob/ob_lob_locator.h +++ b/src/storage/lob/ob_lob_locator.h @@ -79,6 +79,7 @@ private: ObCollationType cs_type, bool is_simple, bool is_systable); + bool can_skip_build_mem_lob_locator(const common::ObString &payload); private: uint64_t table_id_; uint64_t tablet_id_; diff --git a/src/storage/lob/ob_lob_manager.cpp b/src/storage/lob/ob_lob_manager.cpp index 411768cd01..06e946c302 100644 --- a/src/storage/lob/ob_lob_manager.cpp +++ b/src/storage/lob/ob_lob_manager.cpp @@ -19,6 +19,7 @@ #include "observer/ob_server.h" #include "storage/tx_storage/ob_ls_service.h" #include "sql/engine/expr/ob_expr_util.h" +#include "storage/lob/ob_lob_persistent_iterator.h" namespace oceanbase { @@ -39,6 +40,52 @@ static int check_write_length(ObLobAccessParam& param, int64_t expected_len) const ObLobCommon ObLobManager::ZERO_LOB = ObLobCommon(); +static int prepare_data_buffer(ObLobAccessParam& param, ObString &buffer, int64_t buffer_size) +{ + int ret = OB_SUCCESS; + char *ptr = nullptr; + if (OB_ISNULL(param.get_tmp_allocator())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("allocator is null", K(ret), K(param)); + } else if (OB_ISNULL(ptr = static_cast(param.get_tmp_allocator()->alloc(buffer_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc buffer failed.", K(ret)); + } else { + buffer.assign_buffer(ptr, buffer_size); + } + return ret; +} + +// for only one lob meta in mysql mode, we can have no char len here +static int is_store_char_len(ObLobAccessParam& param, int64_t store_chunk_size, int64_t add_len) +{ + int ret = OB_SUCCESS; + if (! lib::is_mysql_mode()) { + LOG_DEBUG("not mysql mode", K(add_len), K(store_chunk_size), K(param)); + } else if (! param.is_char()) { + LOG_DEBUG("not text", K(add_len), K(store_chunk_size), K(param)); + } else if (store_chunk_size <= (param.byte_size_ + add_len)) { + LOG_DEBUG("not single", K(add_len), K(store_chunk_size), K(param)); + } else if (param.tablet_id_.is_inner_tablet()) { + LOG_DEBUG("inner table skip", K(add_len), K(store_chunk_size), K(param)); + } else { + uint64_t tenant_id = param.tenant_id_; + uint64_t data_version = 0; + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) { + LOG_WARN("failed to get data version", K(ret), K(store_chunk_size), K(tenant_id)); + } else if (data_version < MOCK_DATA_VERSION_4_2_3_0) { + LOG_DEBUG("before 4.2.3", K(add_len), K(store_chunk_size), K(param)); + } else if (data_version >= DATA_VERSION_4_3_0_0 && data_version < DATA_VERSION_4_3_2_0) { + LOG_DEBUG("before 4.3.2", K(add_len), K(store_chunk_size), K(param)); + } else { + // sinlge not need char_len after >= 4.3.2 or (>= 4.2.3 and < 4.3.0 + param.is_store_char_len_ = false; + LOG_DEBUG("not store char_len for single piece", K(add_len), K(store_chunk_size), K(param)); + } + } + return ret; +} + static bool lob_handle_has_char_len_field(ObLobAccessParam& param) { bool bret = false; @@ -625,6 +672,11 @@ int ObLobManager::query_remote(ObLobAccessParam& param, ObString& data) if (ret == OB_ITER_END) { ret = OB_SUCCESS; } + + if (OB_SUCC(ret) && param.is_full_read() && param.byte_size_ != data.length()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("read size not macth", K(ret), "current_length", data.length(), "data_length", param.byte_size_, K(param), K(data)); + } } return ret; } @@ -664,6 +716,11 @@ static int read_all( if (ret == OB_ITER_END) { ret = OB_SUCCESS; } + + if (OB_SUCC(ret) && param.byte_size_ != output_data.length()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("read size not macth", K(ret), "read_length", output_data.length(), "read_length", param.byte_size_, K(param), K(output_data)); + } return ret; } @@ -1046,7 +1103,7 @@ int ObLobManager::compare(ObLobLocatorV2& lob_left, ObLobCompareParams& cmp_params, int64_t& result) { INIT_SUCC(ret); - ObArenaAllocator tmp_allocator("ObLobCmp", OB_MALLOC_MIDDLE_BLOCK_SIZE, MTL_ID()); + ObArenaAllocator tmp_allocator("LobCmp", OB_MALLOC_MIDDLE_BLOCK_SIZE, MTL_ID()); ObLobManager *lob_mngr = MTL(ObLobManager*); if (OB_ISNULL(lob_mngr)) { ret = OB_ERR_UNEXPECTED; @@ -1217,9 +1274,12 @@ int ObLobManager::write_one_piece(ObLobAccessParam& param, int ret = OB_SUCCESS; ObLobMetaInfo meta_row = meta_info; meta_row.lob_data_.assign_ptr(data.ptr(), data.length()); - if (meta_info.char_len_ > meta_info.byte_len_) { + if (param.is_store_char_len_ && meta_info.char_len_ > meta_info.byte_len_) { ret = OB_ERR_UNEXPECTED; LOG_WARN("char len should not bigger than byte len", K(ret), K(meta_info)); + } else if (0 == meta_row.byte_len_ || meta_row.byte_len_ != meta_row.lob_data_.length()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("byte length invalid", K(ret), K(meta_row)); } else if (OB_FAIL(lob_ctx.lob_meta_mngr_->write(param, meta_row))) { LOG_WARN("write lob meta row failed.", K(ret)); } else if (OB_FAIL(update_out_ctx(param, nullptr, meta_row))) { // new row @@ -1245,9 +1305,18 @@ int ObLobManager::update_one_piece(ObLobAccessParam& param, { int ret = OB_SUCCESS; new_meta_info.lob_data_.assign_ptr(data.ptr(), data.length()); - if (new_meta_info.char_len_ > new_meta_info.byte_len_) { + if (!param.is_store_char_len_ && lob_handle_has_char_len_field(param)) { + old_meta_info.char_len_ = UINT32_MAX; + } + if (param.is_store_char_len_ && new_meta_info.char_len_ > new_meta_info.byte_len_) { ret = OB_ERR_UNEXPECTED; LOG_WARN("char len should not bigger than byte len", K(ret), K(new_meta_info)); + } else if (! param.is_store_char_len_ && new_meta_info.char_len_ != UINT32_MAX) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("char length invalid", K(ret), K(new_meta_info), K(param)); + } else if (0 == new_meta_info.byte_len_ || new_meta_info.byte_len_ != new_meta_info.lob_data_.length()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("byte length invalid", K(ret), K(new_meta_info)); } else if (OB_FAIL(lob_ctx.lob_meta_mngr_->update(param, old_meta_info, new_meta_info))) { LOG_WARN("write lob meta row failed.", K(ret)); } else if (OB_FAIL(update_out_ctx(param, &old_meta_info, new_meta_info))) { @@ -1273,7 +1342,9 @@ int ObLobManager::erase_one_piece(ObLobAccessParam& param, ObLobPieceInfo& piece_info) { int ret = OB_SUCCESS; - + if (!lob_handle_has_char_len(param) && lob_handle_has_char_len_field(param)) { + meta_info.char_len_ = UINT32_MAX; + } if (OB_FAIL(lob_ctx.lob_meta_mngr_->erase(param, meta_info))) { LOG_WARN("write lob meta row failed.", K(ret)); } else if (OB_FAIL(update_out_ctx(param, nullptr, meta_info))) { // old row @@ -1365,6 +1436,8 @@ int ObLobManager::check_need_out_row( if (OB_ISNULL(buf)) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc buf failed.", K(ret)); + } else if (OB_FAIL(is_store_char_len(param, param.get_schema_chunk_size(), add_len))) { + LOG_WARN("cacl is_store_char_len failed.", K(ret), K(add_len), K(param)); } else { MEMCPY(buf, param.lob_common_, sizeof(ObLobCommon)); ObLobCommon *new_lob_common = reinterpret_cast(buf); @@ -1397,39 +1470,78 @@ int ObLobManager::check_need_out_row( ctx->chunk_size_ = param.get_schema_chunk_size() / ObLobDataOutRowCtx::OUTROW_LOB_CHUNK_SIZE_UNIT; // init char len uint64_t *char_len = reinterpret_cast(ctx + 1); - *char_len = 0; + *char_len = (param.is_store_char_len_) ? 0 : UINT64_MAX; param.handle_size_ = LOB_OUTROW_FULL_SIZE; } } } } else if (! param.lob_common_->in_row_ && need_out_row) { // outrow -> outrow : keep outrow + int64_t store_chunk_size = 0; bool has_char_len = lob_handle_has_char_len(param); - if (has_char_len) { // skip - } else if (! lob_handle_has_char_len_field(param)) { // skip + if (add_len < 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("add_len is negative", K(ret), K(param)); + } else if (add_len == 0) { + // no data add, keep char_len state + param.is_store_char_len_ = has_char_len; + } else if (OB_FAIL(param.get_store_chunk_size(store_chunk_size))) { + LOG_WARN("get_store_chunk_size fail", K(ret), K(param)); + } else if (OB_FAIL(is_store_char_len(param, store_chunk_size, add_len))) { + LOG_WARN("cacl is_store_char_len failed.", K(ret), K(store_chunk_size), K(add_len), K(param)); } else if (param.op_type_ != ObLobDataOutRowCtx::OpType::SQL) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected case", K(ret), K(param), K(has_char_len)); + if (! param.is_store_char_len_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected case", K(ret), K(param), K(has_char_len)); + } } else if (0 != param.offset_ || 0 != param.byte_size_) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected case", K(ret), K(param), K(has_char_len)); - } else if (param.is_full_insert()) { - // reset char_len to 0 from UINT64_MAX + if (! param.is_store_char_len_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected case", K(ret), K(param), K(has_char_len)); + } + } else if (has_char_len && param.is_store_char_len_) { + // keep char_len + } else if (! has_char_len && ! param.is_store_char_len_) { + // keep no char_len + } else if (has_char_len && ! param.is_store_char_len_) { + // old data has char , but new data no char_len + // reset char_len to UINT64_MAX from 0 int64_t *char_len = ObLobManager::get_char_len_ptr(param); if (OB_ISNULL(char_len)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("char_len ptr is null", K(ret), K(param), K(has_char_len)); - } else if (*char_len != UINT64_MAX) { + } else if (*char_len != 0) { ret = OB_ERR_UNEXPECTED; LOG_WARN("char_len should be zero", K(ret), K(param), K(has_char_len), K(*char_len)); } else { - *char_len = 0; - LOG_INFO("no_char_len to has_char_len", K(param)); + *char_len = UINT64_MAX; + LOG_DEBUG("has_char_len to no_char_len", K(param)); + } + } else if (! has_char_len && param.is_store_char_len_) { + if (param.handle_size_ < LOB_OUTROW_FULL_SIZE) { + LOG_INFO("old old data", K(param)); + param.is_store_char_len_ = true; + } else if (param.is_full_insert()) { + // reset char_len to 0 from UINT64_MAX + int64_t *char_len = ObLobManager::get_char_len_ptr(param); + if (OB_ISNULL(char_len)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("char_len ptr is null", K(ret), K(param), K(has_char_len)); + } else if (*char_len != UINT64_MAX) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("char_len should be zero", K(ret), K(param), K(has_char_len), K(*char_len)); + } else { + *char_len = 0; + LOG_DEBUG("no_char_len to has_char_len", K(param)); + } + } else { + // partial update aloways store char_len beacaure is only support in oracle mode + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unsupport situation", K(ret), K(param), K(has_char_len)); } } else { - // partial update aloways store char_len beacaure is only support in oracle mode ret = OB_ERR_UNEXPECTED; - LOG_WARN("unsupport situation", K(ret), K(param), K(has_char_len)); + LOG_WARN("unkown situation", K(ret), K(param), K(has_char_len)); } } return ret; @@ -1535,6 +1647,9 @@ int ObLobManager::append( ObLobLocatorV2 &lob) { int ret = OB_SUCCESS; + ObArenaAllocator tmp_allocator("LobCmp", OB_MALLOC_MIDDLE_BLOCK_SIZE, MTL_ID()); + param.set_tmp_allocator(&tmp_allocator); + bool is_char = param.coll_type_ != common::ObCollationType::CS_TYPE_BINARY; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; @@ -1611,7 +1726,7 @@ int ObLobManager::append( SMART_VAR(ObLobAccessParam, read_param) { read_param.tx_desc_ = param.tx_desc_; read_param.tenant_id_ = param.src_tenant_id_; - if (OB_FAIL(build_lob_param(read_param, *param.allocator_, param.coll_type_, + if (OB_FAIL(build_lob_param(read_param, *param.get_tmp_allocator(), param.coll_type_, 0, UINT64_MAX, param.timeout_, lob))) { LOG_WARN("fail to build read param", K(ret), K(lob)); } else if (OB_FAIL(query(read_param, data))) { @@ -1639,270 +1754,13 @@ int ObLobManager::append( } } } - } else { - // prepare out row ctx - ObLobCtx lob_ctx = lob_ctx_; - int64_t store_chunk_size = 0; - if (OB_FAIL(init_out_row_ctx(param, append_lob_len, param.op_type_))) { - LOG_WARN("init lob data out row ctx failed", K(ret)); - } else if (OB_FAIL(param.get_store_chunk_size(store_chunk_size))) { - LOG_WARN("get_store_chunk_size fail", KR(ret), K(param)); - } - // prepare write buffer - ObString write_buffer; - int64_t buf_len = store_chunk_size; - char *buf = nullptr; - if (OB_SUCC(ret)) { - buf = reinterpret_cast(param.allocator_->alloc(buf_len)); - if (OB_ISNULL(buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("alloc buffer failed.", K(buf_len)); - } else { - write_buffer.assign_buffer(buf, buf_len); - } - } - - // scan get last info - ObLobMetaScanIter meta_iter; - bool use_buffer = false; - bool do_update = false; - ObLobMetaInfo scan_last_info; - ObLobMetaInfo last_info; - ObLobSeqId seq_id(param.allocator_); - if (OB_SUCC(ret)) { - bool need_init_info = false; - if (ori_inrow_data.length() > 0) { - // inrow to outrow, do not need scan, fill ori data - need_init_info = true; - } else { - // do reverse scan - param.scan_backward_ = true; - if (OB_FAIL(lob_ctx.lob_meta_mngr_->scan(param, meta_iter))) { - LOG_WARN("do lob meta scan failed.", K(ret), K(param)); - } else { - ObLobMetaScanResult scan_res; - ret = meta_iter.get_next_row(scan_res); - if (OB_FAIL(ret)) { - if (ret == OB_ITER_END) { - // empty table - ret = OB_SUCCESS; - need_init_info = true; - } else { - LOG_WARN("failed to get next row.", K(ret)); - } - } else if (scan_res.info_.byte_len_ > ObLobMetaUtil::LOB_OPER_PIECE_DATA_SIZE / 2) { - need_init_info = true; - seq_id.set_seq_id(scan_res.info_.seq_id_); - } else { - use_buffer = true; - do_update = true; - scan_last_info = scan_res.info_; - last_info = scan_res.info_; - MEMCPY(write_buffer.ptr(), scan_res.info_.lob_data_.ptr(), last_info.byte_len_); - write_buffer.set_length(last_info.byte_len_); - last_info.lob_data_ = write_buffer; - seq_id.set_seq_id(scan_res.info_.seq_id_); - } - } - } - if (OB_SUCC(ret) && need_init_info) { - use_buffer = (ori_inrow_data.length() > 0); - last_info.lob_id_ = param.lob_data_->id_; - last_info.piece_id_ = ObLobMetaUtil::LOB_META_INLINE_PIECE_ID; - if (ori_inrow_data.length() > 0) { - last_info.byte_len_ = ori_inrow_data.length(); - last_info.char_len_ = ObCharset::strlen_char(param.coll_type_, ori_inrow_data.ptr(), ori_inrow_data.length()); - last_info.lob_data_ = write_buffer; - MEMCPY(last_info.lob_data_.ptr(), ori_inrow_data.ptr(), ori_inrow_data.length()); - last_info.lob_data_.set_length(ori_inrow_data.length()); - } else { - last_info.byte_len_ = 0; - last_info.char_len_ = 0; - last_info.lob_data_.reset(); - } - if (OB_FAIL(seq_id.get_next_seq_id(last_info.seq_id_))) { - LOG_WARN("failed to next seq id.", K(ret)); - } - } - } - - // refresh outrow ctx - if (OB_SUCC(ret)) { - if (last_info.byte_len_ > 0) { - ObLobDataOutRowCtx *out_row_ctx = reinterpret_cast(param.lob_data_->buffer_); - out_row_ctx->first_meta_offset_ = last_info.byte_len_; - } - } - - // prepare read full lob - if (OB_SUCC(ret)) { - SMART_VAR(ObLobAccessParam, read_param) { - read_param.tx_desc_ = param.tx_desc_; - read_param.tenant_id_ = param.src_tenant_id_; - if (OB_FAIL(build_lob_param(read_param, *param.allocator_, param.coll_type_, - 0, UINT64_MAX, param.timeout_, lob))) { - LOG_WARN("fail to build read param", K(ret), K(lob)); - } else { - ObLobQueryIter *iter = nullptr; - if (OB_FAIL(query(read_param, iter))) { - LOG_WARN("do query src by iter failed.", K(ret), K(read_param)); - } else { - // prepare read buffer - ObString read_buffer; - uint64_t read_buff_size = OB_MIN(LOB_READ_BUFFER_LEN, read_param.byte_size_); - char *read_buff = static_cast(param.allocator_->alloc(read_buff_size)); - if (OB_ISNULL(read_buff)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("alloc read buffer failed.", K(ret), K(read_buff_size)); - } - while (OB_SUCC(ret)) { - read_buffer.assign_buffer(read_buff, read_buff_size); - if (OB_FAIL(iter->get_next_row(read_buffer))) { - if (ret != OB_ITER_END) { - LOG_WARN("failed to get next buffer.", K(ret)); - } - } else { - int64_t read_offset = 0; - bool fill_full = false; - while (OB_SUCC(ret) && read_offset < read_buffer.length()) { - // fill data to last info - int64_t left_size = read_buffer.length() - read_offset; - int64_t by_len = 0; - int64_t char_len = 0; - if (use_buffer) { - fill_full = (left_size >= last_info.lob_data_.remain()); - by_len = (last_info.lob_data_.remain() > left_size) ? left_size : last_info.lob_data_.remain(); - by_len = ObCharset::max_bytes_charpos(param.coll_type_, - read_buffer.ptr() + read_offset, - read_buffer.length() - read_offset, - by_len, - char_len); - by_len = ob_lob_writer_length_validation(param.coll_type_, read_buffer.length() - read_offset, by_len, char_len); - if (last_info.lob_data_.write(read_buffer.ptr() + read_offset, by_len) != by_len) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to write data to inner buffer", K(ret), K(last_info.lob_data_), K(by_len)); - } - } else { - // only after reset can access here - OB_ASSERT(last_info.byte_len_ == 0); - if (left_size >= buf_len) { - // read left size is bigger than one lob meta size, do not need set - by_len = ObCharset::max_bytes_charpos(param.coll_type_, - read_buffer.ptr() + read_offset, - left_size, - buf_len, - char_len); - by_len = ob_lob_writer_length_validation(param.coll_type_, left_size, by_len, char_len); - last_info.lob_data_.assign_ptr(read_buffer.ptr() + read_offset, by_len); - fill_full = true; - } else { - use_buffer = true; - last_info.lob_data_.assign_buffer(buf, buf_len); - by_len = ObCharset::max_bytes_charpos(param.coll_type_, - read_buffer.ptr() + read_offset, - left_size, - left_size, - char_len); - by_len = ob_lob_writer_length_validation(param.coll_type_, left_size, by_len, char_len); - if (last_info.lob_data_.write(read_buffer.ptr() + read_offset, by_len) != by_len) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to write data to inner buffer", K(ret), K(last_info.lob_data_), K(by_len)); - } - } - } - // update last info and offset - if (OB_SUCC(ret)) { - last_info.byte_len_ += by_len; - last_info.char_len_ += char_len; - read_offset += by_len; - } - // if fill full, do update or write - if (OB_SUCC(ret) && fill_full) { - if (do_update) { - ObLobPieceInfo piece_info; - if (OB_FAIL(update_one_piece(param, - lob_ctx, - scan_last_info, - last_info, - piece_info, - last_info.lob_data_))) { - LOG_WARN("failed to update.", K(ret), K(last_info), K(scan_last_info)); - } else { - do_update = false; - } - } else { - common::ObTabletID piece_tablet_id; // TODO get piece tablet id - if (OB_FAIL(write_one_piece(param, - piece_tablet_id, - lob_ctx, - last_info, - last_info.lob_data_, - false))) { - LOG_WARN("failed write data.", K(ret), K(last_info)); - } - } - if (OB_SUCC(ret)) { - // reset last_info - last_info.byte_len_ = 0; - last_info.char_len_ = 0; - last_info.lob_data_.reset(); - use_buffer = false; - fill_full = false; - if (OB_FAIL(seq_id.get_next_seq_id(last_info.seq_id_))) { - LOG_WARN("failed to next seq id.", K(ret)); - } - } - } - } - } - } - if (ret == OB_ITER_END) { - ret = OB_SUCCESS; - } - if (OB_SUCC(ret) && last_info.byte_len_ > 0) { - // do update or write - if (do_update) { - ObLobPieceInfo piece_info; - if (OB_FAIL(update_one_piece(param, - lob_ctx, - scan_last_info, - last_info, - piece_info, - last_info.lob_data_))) { - LOG_WARN("failed to update.", K(ret), K(last_info), K(scan_last_info)); - } - } else { - common::ObTabletID piece_tablet_id; - if (OB_FAIL(write_one_piece(param, - piece_tablet_id, - lob_ctx, - last_info, - last_info.lob_data_, - false))) { - LOG_WARN("failed write data.", K(ret), K(last_info)); - } - } - } - if (OB_NOT_NULL(read_buff)) { - param.allocator_->free(read_buff); - } - } - if (OB_NOT_NULL(iter)) { - iter->reset(); - OB_DELETE(ObLobQueryIter, "unused", iter); - } - } - } - } - if (OB_NOT_NULL(buf)) { - param.allocator_->free(buf); - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(check_write_length(param, append_lob_len))) { - LOG_WARN("check_write_length fail", K(ret), K(param), K(lob), K(append_lob_len)); - } + } else if (OB_FAIL(append_outrow(param, lob, append_lob_len, ori_inrow_data))) { + LOG_WARN("failed to process write out row", K(ret), K(param), K(lob), K(append_lob_len), K(ori_inrow_data)); + } else if (OB_FAIL(check_write_length(param, append_lob_len))) { + LOG_WARN("check_write_length fail", K(ret), K(param), K(lob), K(append_lob_len)); } } + param.set_tmp_allocator(nullptr); return ret; } @@ -2090,11 +1948,25 @@ int ObLobManager::prepare_lob_common(ObLobAccessParam& param, bool &alloc_inside return ret; } +int ObLobManager::batch_insert(ObLobAccessParam& param, ObLobMetaWriteIter &iter) +{ + int ret = OB_SUCCESS; + ObLobPersistInsertIter insert_iter; + if (OB_FAIL(insert_iter.init(¶m, &iter))) { + LOG_WARN("init insert iter fail", K(ret)); + } else if (OB_FAIL(lob_ctx_.lob_meta_mngr_->batch_insert(param, insert_iter))) { + LOG_WARN("write lob meta row failed.", K(ret)); + } + return ret; +} + int ObLobManager::append( ObLobAccessParam& param, ObString& data) { int ret = OB_SUCCESS; + ObArenaAllocator tmp_allocator("LobCmp", OB_MALLOC_MIDDLE_BLOCK_SIZE, MTL_ID()); + param.set_tmp_allocator(&tmp_allocator); bool save_is_reverse = param.scan_backward_; uint64_t save_param_len = param.len_; bool is_char = param.coll_type_ != common::ObCollationType::CS_TYPE_BINARY; @@ -2112,6 +1984,7 @@ int ObLobManager::append( ObLobCommon *lob_common = param.lob_common_; ObLobData *lob_data = param.lob_data_; bool is_remote_lob = false; + bool ori_is_inrow = (lob_common == nullptr) ? false : (lob_common->in_row_ == 1); common::ObAddr dst_addr; int64_t store_chunk_size = 0; if (OB_FAIL(ret)) { @@ -2166,70 +2039,17 @@ int ObLobManager::append( } } } - } else if (param.lob_locator_ != nullptr && !param.lob_locator_->is_persist_lob()) { - ret = OB_NOT_IMPLEMENT; - LOG_WARN("Unsupport outrow tmp lob.", K(ret), K(param)); - } else if (OB_FAIL(param.get_store_chunk_size(store_chunk_size))) { - LOG_WARN("get_store_chunk_size fail", KR(ret), K(param)); - } else { // outrow - ObLobMetaWriteIter iter(data, param.allocator_, store_chunk_size); - ObLobCtx lob_ctx = lob_ctx_; - if (OB_FAIL(init_out_row_ctx(param, data.length(), param.op_type_))) { - LOG_WARN("init lob data out row ctx failed", K(ret)); - } else if (OB_FAIL(lob_ctx.lob_meta_mngr_->append(param, iter))) { - LOG_WARN("Failed to open lob meta write iter.", K(ret), K(param)); - } else { - ObLobMetaWriteResult result; - while (OB_SUCC(ret)) { - // split append data into data pieces 250k/piece - ret = iter.get_next_row(result); - if (OB_FAIL(ret)) { - if (ret == OB_ITER_END) { - ret = OB_SUCCESS; - break; - } else { - LOG_WARN("failed to get next row.", K(ret)); - } - } else if (ObTimeUtility::current_time() > param.timeout_) { - ret = OB_TIMEOUT; - int64_t cur_time = ObTimeUtility::current_time(); - LOG_WARN("query timeout", K(cur_time), K(param.timeout_), K(ret)); - } else { - // get len and pos from meta_info - if (result.is_update_) { - ObLobPieceInfo piece_info; - if (OB_FAIL(update_one_piece(param, - lob_ctx, - result.old_info_, - result.info_, - piece_info, - result.data_))) { - LOG_WARN("failed to update.", K(ret), K(result.info_), K(result.old_info_)); - } - } else { - common::ObTabletID piece_tablet_id; // TODO get piece tablet id - if (OB_FAIL(write_one_piece(param, - piece_tablet_id, - lob_ctx, - result.info_, - result.data_, - result.need_alloc_macro_id_))) { - LOG_WARN("failed write data.", K(ret), K(result.info_)); - } - } - } - } - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(check_write_length(param, data.length()))) { - LOG_WARN("check_write_length fail", K(ret), K(param), K(data.length())); - } + } else if (OB_FAIL(append_outrow(param, ori_is_inrow, data))) { + LOG_WARN("append_outrow fail", K(ret), K(param), K(data)); + } else if (OB_FAIL(check_write_length(param, data.length()))) { + LOG_WARN("check_write_length fail", K(ret), K(param), K(data.length())); } } if (OB_SUCC(ret)) { param.len_ = save_param_len; param.scan_backward_ = save_is_reverse; } + param.set_tmp_allocator(nullptr); return ret; } @@ -2494,12 +2314,12 @@ int ObLobManager::process_diff(ObLobAccessParam& param, ObLobLocatorV2& delta_lo int32_t seq_id_int = 0; ObString seq_id_st(sizeof(seq_id_int), reinterpret_cast(&seq_id_int)); ObString seq_id_ed; - ObLobMetaWriteIter write_iter(new_meta_row->lob_data_, param.allocator_, store_chunk_size); + ObLobMetaWriteIter write_iter(param.allocator_, store_chunk_size); ObString post_data; ObString remain_buf; if (OB_FAIL(ObLobSeqId::get_seq_id(offset/store_chunk_size - 1, seq_id_st))) { LOG_WARN("get_seq_id fail", K(ret), K(offset)); - } else if (OB_FAIL(write_iter.open(param, 0, post_data, remain_buf, seq_id_st, seq_id_ed))) { + } else if (OB_FAIL(write_iter.open(param, new_meta_row->lob_data_, 0, post_data, remain_buf, seq_id_st, seq_id_ed))) { LOG_WARN("failed to open meta writer", K(ret), K(write_iter), K(param.byte_size_), K(offset), K(store_chunk_size)); } else if (OB_FAIL(write_outrow_result(param, write_iter))) { LOG_WARN("failed to write outrow result", K(ret), K(write_iter), K(param.byte_size_), K(offset), K(store_chunk_size)); @@ -2531,8 +2351,6 @@ int ObLobManager::getlength_remote(ObLobAccessParam& param, common::ObAddr& dst_ int ret = OB_SUCCESS; ObLobLocatorV2 *lob_locator = param.lob_locator_; ObLobQueryBlock header; - char *buf = nullptr; - int64_t buffer_len = header.get_serialize_size(); if (OB_ISNULL(lob_locator)) { ret = OB_ERR_NULL_VALUE; LOG_WARN("lob locator is null.", K(ret), K(param)); @@ -2588,12 +2406,9 @@ bool ObLobManager::lob_handle_has_char_len(ObLobAccessParam& param) uint64_t *len = reinterpret_cast(ptr + LOB_WITH_OUTROW_CTX_SIZE); if (*len != UINT64_MAX) { bret = true; - } else { - LOG_WARN_RET(OB_SUCCESS, "found no char_len, this only happen in inner QA upgrade test, can not happen in user situation", K(param)); } } return bret; - } int64_t* ObLobManager::get_char_len_ptr(ObLobAccessParam& param) @@ -2863,32 +2678,6 @@ int ObLobManager::write_inrow(ObLobAccessParam& param, ObLobLocatorV2& lob, uint return ret; } -int ObLobManager::prepare_write_buffers(ObLobAccessParam& param, ObString &remain_buf, ObString &tmp_buf) -{ - int ret = OB_SUCCESS; - // prepare remain buf - char *buf = reinterpret_cast(param.allocator_->alloc(ObLobMetaUtil::LOB_OPER_PIECE_DATA_SIZE)); - if (OB_ISNULL(buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("alloc buf failed.", K(ret)); - } else { - remain_buf.assign_buffer(buf, ObLobMetaUtil::LOB_OPER_PIECE_DATA_SIZE); - } - - // prepare tmp buf - if (OB_FAIL(ret)) { - } else { - buf = reinterpret_cast(param.allocator_->alloc(ObLobMetaUtil::LOB_OPER_PIECE_DATA_SIZE)); - if (OB_ISNULL(buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("alloc buf failed.", K(ret)); - } else { - tmp_buf.assign_buffer(buf, ObLobMetaUtil::LOB_OPER_PIECE_DATA_SIZE); - } - } - return ret; -} - int ObLobManager::write_outrow_result(ObLobAccessParam& param, ObLobMetaWriteIter &write_iter) { int ret = OB_SUCCESS; @@ -2902,7 +2691,7 @@ int ObLobManager::write_outrow_result(ObLobAccessParam& param, ObLobMetaWriteIte ret = OB_SUCCESS; break; } else { - LOG_WARN("failed to get next row.", K(ret)); + LOG_WARN("failed to get next row.", K(ret), K(cnt)); } } else if (ObTimeUtility::current_time() > param.timeout_) { ret = OB_TIMEOUT; @@ -2910,15 +2699,26 @@ int ObLobManager::write_outrow_result(ObLobAccessParam& param, ObLobMetaWriteIte LOG_WARN("query timeout", K(cur_time), K(param.timeout_), K(ret)); } else { cnt++; - // get len and pos from meta_info - common::ObTabletID piece_tablet_id; // TODO get piece tablet id - if (OB_FAIL(write_one_piece(param, - piece_tablet_id, - lob_ctx_, - result.info_, - result.data_, - result.need_alloc_macro_id_))) { - LOG_WARN("failed write data.", K(ret), K(cnt), K(result.info_)); + if (result.is_update_) { + ObLobPieceInfo piece_info; + if (OB_FAIL(update_one_piece(param, + lob_ctx_, + result.old_info_, + result.info_, + piece_info, + result.data_))) { + LOG_WARN("failed to update.", K(ret), K(cnt), K(result.info_), K(result.old_info_)); + } + } else { + common::ObTabletID piece_tablet_id; // TODO get piece tablet id + if (OB_FAIL(write_one_piece(param, + piece_tablet_id, + lob_ctx_, + result.info_, + result.data_, + result.need_alloc_macro_id_))) { + LOG_WARN("failed write data.", K(ret), K(cnt), K(result.info_)); + } } } } @@ -2928,6 +2728,7 @@ int ObLobManager::write_outrow_result(ObLobAccessParam& param, ObLobMetaWriteIte int ObLobManager::write_outrow_inner(ObLobAccessParam& param, ObLobQueryIter *iter, ObString& read_buf, ObString& old_data) { int ret = OB_SUCCESS; + int64_t store_chunk_size = 0; SMART_VAR(ObLobMetaScanIter, meta_iter) { uint64_t modified_len = param.len_; int64_t mbmaxlen = 1; @@ -2949,6 +2750,8 @@ int ObLobManager::write_outrow_inner(ObLobAccessParam& param, ObLobQueryIter *it } if (OB_FAIL(ret)) { + } else if (OB_FAIL(param.get_store_chunk_size(store_chunk_size))) { + LOG_WARN("get_store_chunk_size fail", KR(ret), K(param)); } else if (OB_FAIL(init_out_row_ctx(param, modified_len + old_data.length(), param.op_type_))) { LOG_WARN("init lob data out row ctx failed", K(ret)); } else { @@ -2966,8 +2769,10 @@ int ObLobManager::write_outrow_inner(ObLobAccessParam& param, ObLobQueryIter *it LOG_INFO("param scan_backward is true. Make it be false.", K(param)); param.scan_backward_ = false; } - if (OB_FAIL(prepare_write_buffers(param, remain_buf, tmp_buf))) { - LOG_WARN("fail to prepare buffers", K(ret)); + if(OB_FAIL(prepare_data_buffer(param, remain_buf, store_chunk_size))) { + LOG_WARN("fail to prepare buffers", K(ret), K(param)); + } else if(OB_FAIL(prepare_data_buffer(param, tmp_buf, store_chunk_size))) { + LOG_WARN("fail to prepare buffers", K(ret), K(param)); } else if (OB_FAIL(lob_ctx_.lob_meta_mngr_->scan(param, meta_iter))) { LOG_WARN("do lob meta scan failed.", K(ret), K(param)); } else { @@ -2998,7 +2803,7 @@ int ObLobManager::write_outrow_inner(ObLobAccessParam& param, ObLobQueryIter *it found_end = true; } } - if (OB_SUCC(ret) && OB_FAIL(replace_process_meta_info(param, meta_iter, result, iter, read_buf, remain_buf, tmp_buf))) { + if (OB_SUCC(ret) && OB_FAIL(replace_process_meta_info(param, store_chunk_size, meta_iter, result, iter, read_buf, remain_buf, tmp_buf))) { LOG_WARN("process erase meta info failed.", K(ret), K(param), K(result)); } } @@ -3081,7 +2886,7 @@ int ObLobManager::write_outrow_inner(ObLobAccessParam& param, ObLobQueryIter *it LOG_WARN("get_store_chunk_size fail", KR(ret), K(param)); } else { // prepare write iter - ObLobMetaWriteIter write_iter(read_buf, param.allocator_, store_chunk_size); + ObLobMetaWriteIter write_iter(param.allocator_, store_chunk_size); if (OB_FAIL(write_iter.open(param, iter, read_buf, padding_size, post_data, remain_buf, seq_id_st, seq_id_ed))) { LOG_WARN("failed to open meta writer", K(ret), K(write_iter), K(meta_iter), K(found_begin), K(found_end), K(range_begin), K(range_end)); @@ -3258,340 +3063,6 @@ int ObLobManager::write(ObLobAccessParam& param, ObString& data) return ret; } -int ObLobManager::replace_process_meta_info(ObLobAccessParam& param, - ObLobMetaScanIter &meta_iter, - ObLobQueryResult &result, - ObLobQueryIter *iter, - ObString& read_buf, - ObString &remain_data, - ObString &tmp_buf) -{ - int ret = OB_SUCCESS; - ObLobCtx lob_ctx = lob_ctx_; - bool is_char = param.coll_type_ != common::ObCollationType::CS_TYPE_BINARY; - bool del_piece = false; - if (meta_iter.is_range_begin(result.meta_result_.info_) || - meta_iter.is_range_end(result.meta_result_.info_)) { - // 1. read data - // 2. rebuild data - // 3. write data into lob data oper - // 4. write meta tablet - // 5. write piece tablet - ObLobMetaInfo new_meta_row = result.meta_result_.info_; - ObString read_data; - read_data.assign_buffer(tmp_buf.ptr(), tmp_buf.size()); - - // save variable - uint32_t tmp_st = result.meta_result_.st_; - uint32_t tmp_len = result.meta_result_.len_; - - // read all piece data - result.meta_result_.st_ = 0; - result.meta_result_.len_ = result.meta_result_.info_.char_len_; - if (OB_FAIL(get_real_data(param, result, read_data))) { - LOG_WARN("failed to write data to read buf.", K(ret), K(result)); - } else { - result.meta_result_.st_ = tmp_st; - result.meta_result_.len_ = tmp_len; - - // global pos, from 0 - uint64_t cur_piece_end = meta_iter.get_cur_pos(); - uint64_t cur_piece_begin = cur_piece_end - result.meta_result_.info_.char_len_; - - // local pos, from current piece; - // if is_char, char pos; else byte pos - uint32_t local_begin = param.offset_ - cur_piece_begin; - uint32_t local_end = param.offset_ + param.len_ - cur_piece_begin; - - // char len - uint32_t piece_byte_len = result.meta_result_.info_.byte_len_; - uint32_t piece_char_len = result.meta_result_.info_.char_len_; - - if (OB_FAIL(ret)) { - } else if (meta_iter.is_range_begin(result.meta_result_.info_) && - meta_iter.is_range_end(result.meta_result_.info_)) { - uint32_t replace_char_st = local_begin; - uint32_t replace_char_len = local_end - local_begin; - uint32_t replace_byte_st = replace_char_st; - uint32_t replace_byte_len = replace_char_len; - if (is_char) { - transform_query_result_charset(param.coll_type_, - read_data.ptr(), - read_data.length(), - replace_byte_len, - replace_byte_st); - } else { - replace_byte_len = OB_MIN(replace_byte_len, read_data.length() - replace_byte_st); - } - ObString temp_read_buf; - uint32_t temp_read_size = ObLobMetaUtil::LOB_OPER_PIECE_DATA_SIZE - replace_byte_st; - temp_read_buf.assign_buffer(read_buf.ptr(), temp_read_size); - if (OB_FAIL(iter->get_next_row(temp_read_buf))) { - LOG_WARN("fail to do get next read buffer", K(ret)); - } else if (piece_byte_len - replace_byte_len + temp_read_buf.length() <= ObLobMetaUtil::LOB_OPER_PIECE_DATA_SIZE) { - // after replace, can store in one this meta - if (temp_read_buf.length() != replace_byte_len) { - uint32_t move_len = piece_byte_len - (replace_byte_st + replace_byte_len); - MEMMOVE(read_data.ptr() + replace_byte_st + temp_read_buf.length(), read_data.ptr() + replace_byte_st + replace_byte_len, move_len); - } - MEMCPY(read_data.ptr() + replace_byte_st, temp_read_buf.ptr(), temp_read_buf.length()); - read_data.set_length(piece_byte_len - replace_byte_len + temp_read_buf.length()); - } else { - // after replace, cannot store in one this meta - // keep remain data for next insert - // [0, replace_byte_st] - // [replace_byte_st, replace_byte_st+replace_byte_len] - // [replace_byte_st+replace_byte_len, piece_byte_len] - uint32_t remain_st = replace_byte_st + replace_byte_len; - uint32_t remain_len = piece_byte_len - remain_st; - MEMCPY(remain_data.ptr(), read_data.ptr() + remain_st, remain_len); - remain_data.set_length(remain_len); - // try copy data to meta - int64_t max_byte = ObLobMetaUtil::LOB_OPER_PIECE_DATA_SIZE - replace_byte_st; - int64_t data_char_len = 0; - max_byte = OB_MIN(max_byte, temp_read_buf.length()); - size_t data_by_len = ObCharset::max_bytes_charpos(param.coll_type_, temp_read_buf.ptr(), temp_read_buf.length(), max_byte, data_char_len); - data_by_len = ob_lob_writer_length_validation(param.coll_type_, temp_read_buf.length(), data_by_len, data_char_len); - MEMCPY(read_data.ptr() + replace_byte_st, temp_read_buf.ptr(), data_by_len); - read_data.set_length(replace_byte_st + data_by_len); - if (data_by_len == temp_read_buf.length()) { - // try copy remain data to meta if data has copy fully - max_byte = ObLobMetaUtil::LOB_OPER_PIECE_DATA_SIZE - read_data.length(); - max_byte = OB_MIN(max_byte, remain_data.length()); - size_t rdata_by_len = ObCharset::max_bytes_charpos(param.coll_type_, remain_data.ptr(), remain_data.length(), max_byte, data_char_len); - rdata_by_len = ob_lob_writer_length_validation(param.coll_type_, remain_data.length(), rdata_by_len, data_char_len); - if (rdata_by_len >= remain_data.length()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid len", K(ret), K(remain_data), K(rdata_by_len), K(max_byte)); - } else if (rdata_by_len > 0) { - MEMCPY(read_data.ptr() + read_data.length(), remain_data.ptr(), rdata_by_len); - read_data.set_length(read_data.length() + rdata_by_len); - MEMMOVE(remain_data.ptr(), remain_data.ptr() + rdata_by_len, remain_data.length() - rdata_by_len); - remain_data.set_length(remain_data.length() - rdata_by_len); - } - } - } - if (OB_SUCC(ret)) { - new_meta_row.byte_len_ = read_data.length(); - new_meta_row.char_len_ = ObCharset::strlen_char(param.coll_type_, read_data.ptr(), read_data.length()); - } - } else if (meta_iter.is_range_begin(result.meta_result_.info_)) { - uint32_t by_st = 0; - uint32_t by_len = local_begin; - if (is_char) { - transform_query_result_charset(param.coll_type_, - read_data.ptr(), - read_data.length(), - by_len, - by_st); - } else { - by_len = OB_MIN(by_len, read_data.length() - by_st); - } - // [0, by_len][by_len, 256K] - // try copy data to meta in [by_len, 256K] - ObString temp_read_buf; - uint32_t max_byte = ObLobMetaUtil::LOB_OPER_PIECE_DATA_SIZE - by_len; - temp_read_buf.assign_buffer(read_buf.ptr(), max_byte); - if (iter->is_end()) { - } else if (OB_FAIL(iter->get_next_row(temp_read_buf))) { - if (ret != OB_ITER_END) { - LOG_WARN("fail to do get next read buffer", K(ret)); - } else { - ret = OB_SUCCESS; - } - } - - if (OB_FAIL(ret)) { - } else if (iter->is_end()) { - if (local_begin == 0) { - del_piece = true; - } else { - read_data.assign_ptr(read_data.ptr(), by_len); - new_meta_row.byte_len_ = by_len; - new_meta_row.char_len_ = local_begin; - } - } else { - int64_t data_char_len = 0; - max_byte = OB_MIN(max_byte, temp_read_buf.length()); - size_t data_by_len = ObCharset::max_bytes_charpos(param.coll_type_, temp_read_buf.ptr(), - temp_read_buf.length(), max_byte, data_char_len); - data_by_len = ob_lob_writer_length_validation(param.coll_type_, temp_read_buf.length(), data_by_len, data_char_len); - MEMCPY(read_data.ptr() + by_len, temp_read_buf.ptr(), data_by_len); - read_data.set_length(by_len + data_by_len); - new_meta_row.byte_len_ = read_data.length(); - new_meta_row.char_len_ = local_begin + data_char_len; - } - } else { - uint32_t by_st = 0; - uint32_t by_len = local_end; - if (is_char) { - transform_query_result_charset(param.coll_type_, - read_data.ptr(), - read_data.length(), - by_len, - by_st); - } else { - by_len = OB_MIN(by_len, read_data.length() - by_st); - } - // calc data - ObString temp_read_buf; - int64_t max_byte = ObLobMetaUtil::LOB_OPER_PIECE_DATA_SIZE - (piece_byte_len - by_len); - temp_read_buf.assign_buffer(read_buf.ptr(), max_byte); - if (iter->is_end()) { - } else if (OB_FAIL(iter->get_next_row(temp_read_buf))) { - if (ret != OB_ITER_END) { - LOG_WARN("fail to do get next read buffer", K(ret)); - } else { - ret = OB_SUCCESS; - } - } - - if (OB_FAIL(ret)) { - } else if (iter->is_end()) { - if (local_end == piece_char_len) { - del_piece = true; - } else { - new_meta_row.char_len_ = piece_char_len - local_end; - new_meta_row.byte_len_ = read_data.length() - by_len; - MEMMOVE(read_data.ptr(), read_data.ptr() + by_len, read_data.length() - by_len); - read_data.assign_ptr(read_data.ptr(), read_data.length() - by_len); - } - } else { - // calc data - int64_t data_char_len = 0; - max_byte = OB_MIN(max_byte, temp_read_buf.length()); - size_t data_by_len = ObCharset::max_bytes_charpos(param.coll_type_, temp_read_buf.ptr(), temp_read_buf.length(), max_byte, data_char_len); - data_by_len = ob_lob_writer_length_validation(param.coll_type_, temp_read_buf.length(), data_by_len, data_char_len); - MEMMOVE(read_data.ptr() + data_by_len, read_data.ptr() + by_len, piece_byte_len - by_len); - MEMCPY(read_data.ptr(), temp_read_buf.ptr(), data_by_len); - read_data.set_length(piece_byte_len - by_len + data_by_len); - new_meta_row.char_len_ = piece_char_len - local_end + data_char_len; - new_meta_row.byte_len_ = piece_byte_len - by_len + data_by_len; - } - } - - if (OB_FAIL(ret)) { - } else if (del_piece) { - if (OB_FAIL(erase_one_piece(param, - lob_ctx, - result.meta_result_.info_, - result.piece_info_))) { - LOG_WARN("failed erase one piece", K(ret), K(result)); - } - } else { - if (OB_FAIL(update_one_piece(param, - lob_ctx, - result.meta_result_.info_, - new_meta_row, - result.piece_info_, - read_data))) { - LOG_WARN("failed to update.", K(ret), K(result), K(read_data)); - } - } - } - } else { - // do replace for whole meta - int64_t max_byte = ObLobMetaUtil::LOB_OPER_PIECE_DATA_SIZE; - ObString temp_read_buf; - temp_read_buf.assign_buffer(read_buf.ptr(), max_byte); - if (iter->is_end()) { - } else if (OB_FAIL(iter->get_next_row(temp_read_buf))) { - if (ret != OB_ITER_END) { - LOG_WARN("fail to do get next read buffer", K(ret)); - } else { - ret = OB_SUCCESS; - } - } - - if (OB_FAIL(ret)) { - } else if (!iter->is_end()) { - ObLobMetaInfo new_meta_row = result.meta_result_.info_; - ObString read_data; - read_data.assign_buffer(tmp_buf.ptr(), tmp_buf.size()); - - int64_t data_char_len = 0; - max_byte = OB_MIN(max_byte, temp_read_buf.length()); - size_t data_by_len = ObCharset::max_bytes_charpos(param.coll_type_, temp_read_buf.ptr(), temp_read_buf.length(), max_byte, data_char_len); - data_by_len = ob_lob_writer_length_validation(param.coll_type_, temp_read_buf.length(), data_by_len, data_char_len); - MEMCPY(read_data.ptr(), temp_read_buf.ptr(), data_by_len); - read_data.set_length(data_by_len); - new_meta_row.byte_len_ = read_data.length(); - new_meta_row.char_len_ = data_char_len; - if (OB_FAIL(update_one_piece(param, - lob_ctx, - result.meta_result_.info_, - new_meta_row, - result.piece_info_, - read_data))) { - LOG_WARN("failed to update.", K(ret), K(result), K(read_data)); - } - } else { - if (OB_FAIL(erase_one_piece(param, - lob_ctx, - result.meta_result_.info_, - result.piece_info_))) { - LOG_WARN("failed erase one piece", K(ret), K(result)); - } - } - } - return ret; -} - -int ObLobManager::do_delete_one_piece(ObLobAccessParam& param, ObLobQueryResult &result, ObString &tmp_buff) -{ - int ret = OB_SUCCESS; - ObLobCtx lob_ctx = lob_ctx_; - // do fill zero for whole meta - if (param.is_fill_zero_) { - ObLobMetaInfo new_meta_row = result.meta_result_.info_; - uint32_t fill_len = new_meta_row.char_len_; - char* tmp_buf = tmp_buff.ptr(); - if (OB_ISNULL(tmp_buf)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fill zero buf is null.", K(ret), K(tmp_buff)); - } else { - ObString space = ObCharsetUtils::get_const_str(param.coll_type_, ' '); - if (param.coll_type_ == CS_TYPE_BINARY) { - MEMSET(tmp_buf, 0x00, fill_len); - } else { - uint32_t space_len = space.length(); - if (fill_len%space_len != 0) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid res-len", K(ret), K(fill_len), K(space_len)); - } else if (space_len > 1) { - for (int i = 0; i < fill_len/space_len; i++) { - MEMCPY(tmp_buf + i * space_len, space.ptr(), space_len); - } - } else { - MEMSET(tmp_buf, *space.ptr(), fill_len); - } - } - if (OB_FAIL(ret)) { - } else { - ObString read_data; - read_data.assign_ptr(tmp_buf, fill_len); - new_meta_row.byte_len_ = read_data.length(); - if (OB_FAIL(update_one_piece(param, - lob_ctx, - result.meta_result_.info_, - new_meta_row, - result.piece_info_, - read_data))) { - LOG_WARN("failed to update.", K(ret), K(result), K(read_data)); - } - } - } - } else { - if (OB_FAIL(erase_one_piece(param, - lob_ctx, - result.meta_result_.info_, - result.piece_info_))) { - LOG_WARN("failed erase one piece", K(ret), K(result)); - } - } - return ret; -} - int ObLobManager::fill_zero(char *ptr, uint64_t length, bool is_char, const ObCollationType coll_type, uint32_t byte_len, uint32_t byte_offset, uint32_t &char_len) { @@ -3625,250 +3096,16 @@ int ObLobManager::fill_zero(char *ptr, uint64_t length, bool is_char, return ret; } -int ObLobManager::erase_process_meta_info(ObLobAccessParam& param, ObLobMetaScanIter &meta_iter, - ObLobQueryResult &result, ObString &tmp_buff) +int ObLobManager::batch_delete(ObLobAccessParam& param, ObLobMetaScanIter &iter) { int ret = OB_SUCCESS; - ObLobCtx lob_ctx = lob_ctx_; - bool is_char = param.coll_type_ != common::ObCollationType::CS_TYPE_BINARY; - bool del_piece = false; - if (meta_iter.is_range_begin(result.meta_result_.info_) || - meta_iter.is_range_end(result.meta_result_.info_)) { - // 1. read data - // 2. rebuild data - // 3. write data into lob data oper - // 4. write meta tablet - // 5. write piece tablet - ObLobMetaInfo new_meta_row = result.meta_result_.info_; - char* tmp_buf = tmp_buff.ptr(); - ObString read_data; - if (param.is_fill_zero_) { - read_data.assign_buffer(tmp_buf, tmp_buff.size()); - } else { - read_data = result.meta_result_.info_.lob_data_; - } - - // save variable - uint32_t tmp_st = result.meta_result_.st_; - uint32_t tmp_len = result.meta_result_.len_; - - // read all piece data - result.meta_result_.st_ = 0; - result.meta_result_.len_ = result.meta_result_.info_.char_len_; - if (OB_ISNULL(tmp_buf)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tmp buffer is null.", K(ret), K(tmp_buff)); - } else if (param.is_fill_zero_ && OB_FAIL(get_real_data(param, result, read_data))) { - LOG_WARN("failed to write data to read buf.", K(ret), K(result)); - } else { - result.meta_result_.st_ = tmp_st; - result.meta_result_.len_ = tmp_len; - - // global pos, from 0 - // meta_iter.get_cur_pos use byte_len for binary charset, so cur_piece_begin should use byte_len too for binary charset - // or if char_len is less than for byte_len, will cause cur_piece_begin incorrect - uint64_t cur_piece_end = meta_iter.get_cur_pos(); - uint64_t cur_piece_begin = cur_piece_end - (is_char ? result.meta_result_.info_.char_len_ : result.meta_result_.info_.byte_len_); - - // local pos, from current piece; - // if is_char, char pos; else byte pos - // consider 3 situation according to this lob meta how to hit range - // meta both hit begin and end : cur_piece_begin > offset > offset+len > cur_piece_end - // meta hit begin : cur_piece_begin > offset > cur_piece_end - // meta hit end : cur_piece_begin > offset+len > cur_piece_end - uint32_t local_begin = param.offset_ - cur_piece_begin; - uint32_t local_end = param.offset_ + param.len_ - cur_piece_begin; - - // char len - uint32_t piece_byte_len = result.meta_result_.info_.byte_len_; - uint32_t piece_char_len = result.meta_result_.info_.char_len_; - uint32_t piece_data_st = 0; - - if (is_char) { - transform_query_result_charset(param.coll_type_, - read_data.ptr(), - read_data.length(), - piece_char_len, - piece_data_st); - if (piece_char_len != piece_byte_len || piece_data_st > 0) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to erase.", K(ret), K(result), K(piece_char_len), - K(piece_byte_len), K(piece_data_st), K(read_data)); - } else { - piece_byte_len = piece_char_len; - piece_char_len = result.meta_result_.info_.char_len_; - } - } else { - // consider we process char lob as byte len - // we may get a char len which is diff from byte len - piece_char_len = piece_byte_len; - } - - if (OB_FAIL(ret)) { - } else if (meta_iter.is_range_begin(result.meta_result_.info_) && - meta_iter.is_range_end(result.meta_result_.info_)) { - if (local_end - local_begin != piece_char_len) { - uint32_t by_st = local_begin; - uint32_t by_len = local_end - local_begin; - if (is_char) { - transform_query_result_charset(param.coll_type_, - read_data.ptr(), - read_data.length(), - by_len, - by_st); - } - if (param.is_fill_zero_) { - uint32_t fill_char_len = local_end - local_begin; - if (OB_FAIL(fill_zero(read_data.ptr(), read_data.length(), is_char, param.coll_type_, by_len, by_st, fill_char_len))) { - LOG_WARN("failed to fill zero", K(ret)); - } else { - read_data.set_length(read_data.length() - by_len + fill_char_len); - new_meta_row.byte_len_ = read_data.length(); - } - } else { - // here expect read_data read full lob meta data - read_data.assign_buffer(tmp_buf, tmp_buff.size()); - MEMCPY(read_data.ptr(), result.meta_result_.info_.lob_data_.ptr(), result.meta_result_.info_.lob_data_.length()); - read_data.set_length(result.meta_result_.info_.lob_data_.length()); - new_meta_row.byte_len_ -= (by_len); - new_meta_row.char_len_ -= (local_end - local_begin); - MEMMOVE(read_data.ptr() + by_st, read_data.ptr() + (by_st + by_len), piece_byte_len - (by_st + by_len)); - read_data.assign_ptr(read_data.ptr(), read_data.length() - by_len); - } - } else { - del_piece = true; - } - } else if (meta_iter.is_range_begin(result.meta_result_.info_)) { - if (local_begin == 0) { - del_piece = true; - } else { - uint32_t by_st = 0; - uint32_t by_len = local_begin; - if (is_char) { - transform_query_result_charset(param.coll_type_, - read_data.ptr(), - read_data.length(), - by_len, - by_st); - } - if (param.is_fill_zero_) { - uint32_t fill_char_len = piece_char_len - local_begin; - if (OB_FAIL(fill_zero(read_data.ptr(), read_data.length(), is_char, param.coll_type_, - read_data.length() - by_len, by_len, fill_char_len))) { - LOG_WARN("failed to fill zero", K(ret)); - } else { - read_data.set_length(by_len + fill_char_len); - new_meta_row.byte_len_ = read_data.length(); - } - } else { - read_data.assign_ptr(read_data.ptr(), by_len); - new_meta_row.byte_len_ = by_len; - new_meta_row.char_len_ = local_begin; - } - } - } else { - if (local_end == piece_char_len) { - del_piece = true; - } else { - uint32_t by_st = 0; - uint32_t by_len = local_end; - if (is_char) { - transform_query_result_charset(param.coll_type_, - read_data.ptr(), - read_data.length(), - by_len, - by_st); - } - if (param.is_fill_zero_) { - uint32_t fill_char_len = local_end; - if (OB_FAIL(fill_zero(read_data.ptr(), read_data.length(), is_char, param.coll_type_, by_len, 0, fill_char_len))) { - LOG_WARN("failed to fill zero", K(ret)); - } else { - read_data.set_length(fill_char_len + read_data.length() - by_len); - new_meta_row.byte_len_ = read_data.length(); - } - } else { - // here expect read_data read full lob meta data - read_data.assign_buffer(tmp_buf, tmp_buff.size()); - MEMCPY(read_data.ptr(), result.meta_result_.info_.lob_data_.ptr(), result.meta_result_.info_.lob_data_.length()); - read_data.set_length(result.meta_result_.info_.lob_data_.length()); - new_meta_row.char_len_ = piece_char_len - local_end; - new_meta_row.byte_len_ = read_data.length() - by_len; - MEMMOVE(read_data.ptr(), read_data.ptr() + by_len, read_data.length() - by_len); - read_data.assign_ptr(read_data.ptr(), read_data.length() - by_len); - } - } - } - - if (OB_FAIL(ret)) { - } else if (del_piece) { - if (OB_FAIL(do_delete_one_piece(param, result, tmp_buff))) { - LOG_WARN("failed erase one piece", K(ret), K(result)); - } - } else { - if (OB_FAIL(update_one_piece(param, - lob_ctx, - result.meta_result_.info_, - new_meta_row, - result.piece_info_, - read_data))) { - LOG_WARN("failed to update.", K(ret), K(result), K(read_data)); - } - } - } - } else { - if (OB_FAIL(do_delete_one_piece(param, result, tmp_buff))) { - LOG_WARN("do delete one lob meta failed", K(ret), K(param), K(result)); - } - } - return ret; -} - -int ObLobManager::prepare_erase_buffer(ObLobAccessParam& param, ObString &tmp_buff) -{ - int ret = OB_SUCCESS; - char* tmp_buf = static_cast(param.allocator_->alloc(ObLobMetaUtil::LOB_OPER_PIECE_DATA_SIZE)); - if (OB_ISNULL(tmp_buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("alloc buffer failed.", K(ret)); - } else { - tmp_buff.assign_buffer(tmp_buf, ObLobMetaUtil::LOB_OPER_PIECE_DATA_SIZE); - } - return ret; -} - -int ObLobManager::batch_delete(ObLobAccessParam& param, ObLobMetaScanIter &meta_iter) -{ - int ret = OB_SUCCESS; - ObLobQueryResult result; - meta_iter.set_not_calc_char_len(true); - meta_iter.set_not_need_last_info(true); - while (OB_SUCC(ret)) { - if (OB_FAIL(meta_iter.get_next_row(result.meta_result_))) { - if (ret == OB_ITER_END) { - } else { - LOG_WARN("failed to get next row.", K(ret)); - } - } else if (ObTimeUtility::current_time() > param.timeout_) { - ret = OB_TIMEOUT; - int64_t cur_time = ObTimeUtility::current_time(); - LOG_WARN("query timeout", K(cur_time), K(param.timeout_), K(ret)); - } else if (OB_FAIL(lob_ctx_.lob_meta_mngr_->erase(param, result.meta_result_.info_))) { - LOG_WARN("write lob meta row failed.", K(ret)); - } else if (OB_FAIL(update_out_ctx(param, nullptr, result.meta_result_.info_))) { // old row - LOG_WARN("failed update checksum.", K(ret)); - } else { - param.lob_data_->byte_size_ -= result.meta_result_.info_.byte_len_; - if (lob_handle_has_char_len(param)) { - int64_t *len = get_char_len_ptr(param); - *len = *len - result.meta_result_.info_.char_len_; - OB_ASSERT(*len >= 0); - } - param.byte_size_ = param.lob_data_->byte_size_; - } - } - if (ret == OB_ITER_END) { - ret = OB_SUCCESS; + ObLobPersistDeleteIter delete_iter; + iter.set_not_calc_char_len(true); + iter.set_not_need_last_info(true); + if (OB_FAIL(delete_iter.init(¶m, &iter))) { + LOG_WARN("init insert iter fail", K(ret)); + } else if (OB_FAIL(lob_ctx_.lob_meta_mngr_->batch_delete(param, delete_iter))) { + LOG_WARN("write lob meta row failed.", K(ret)); } return ret; } @@ -3880,7 +3117,10 @@ int ObLobManager::erase_imple_inner(ObLobAccessParam& param) ObLobMetaScanIter meta_iter; ObLobCtx lob_ctx = lob_ctx_; ObString tmp_buff; - if (OB_FAIL(init_out_row_ctx(param, param.lob_data_->byte_size_, param.op_type_))) { + int64_t store_chunk_size = 0; + if (OB_FAIL(param.get_store_chunk_size(store_chunk_size))) { + LOG_WARN("get_store_chunk_size fail", KR(ret), K(param)); + } else if (OB_FAIL(init_out_row_ctx(param, param.lob_data_->byte_size_, param.op_type_))) { LOG_WARN("init lob data out row ctx failed", K(ret)); } else if (OB_FAIL(lob_ctx.lob_meta_mngr_->scan(param, meta_iter))) { LOG_WARN("do lob meta scan failed.", K(ret), K(param)); @@ -3888,7 +3128,7 @@ int ObLobManager::erase_imple_inner(ObLobAccessParam& param) if (OB_FAIL(batch_delete(param, meta_iter))) { LOG_WARN("batch_delete fail", K(ret), K(param)); } - } else if(OB_FAIL(prepare_erase_buffer(param, tmp_buff))) { + } else if(OB_FAIL(prepare_data_buffer(param, tmp_buff, store_chunk_size))) { LOG_WARN("fail to prepare buffers", K(ret), K(param)); } else { ObLobQueryResult result; @@ -3906,7 +3146,7 @@ int ObLobManager::erase_imple_inner(ObLobAccessParam& param) } else if (param.asscess_ptable_ /* TODO: weiyouchao.wyc should set param.asscess_ptable_ as false 2022.4.7 */ && OB_FAIL(lob_ctx.lob_piece_mngr_->get(param, result.meta_result_.info_.piece_id_, result.piece_info_))) { LOG_WARN("get lob piece failed.", K(ret), K(result)); - } else if (OB_FAIL(erase_process_meta_info(param, meta_iter, result, tmp_buff))) { + } else if (OB_FAIL(erase_process_meta_info(param, store_chunk_size, meta_iter, result, tmp_buff))) { LOG_WARN("process erase meta info failed.", K(ret), K(param), K(result)); } } @@ -4000,6 +3240,10 @@ int ObLobManager::erase(ObLobAccessParam& param) } else if (param.lob_locator_ != nullptr && !param.lob_locator_->is_persist_lob()) { ret = OB_NOT_IMPLEMENT; LOG_WARN("Unsupport outrow tmp lob.", K(ret), K(param)); + } else if (param.is_fill_zero_) { + if (OB_FAIL(fill_outrow_with_zero(param))) { + LOG_WARN("fill_outrow_with_zero fail", K(ret), K(param)); + } } else if (OB_FAIL(erase_imple_inner(param))) { LOG_WARN("failed erase", K(ret)); } @@ -4080,6 +3324,312 @@ int ObLobManager::build_lob_param(ObLobAccessParam& param, return ret; } +int ObLobManager::append_outrow(ObLobAccessParam& param, bool ori_is_inrow, ObString &data) +{ + int ret = OB_SUCCESS; + ObLobMetaInfo last_info; + int64_t store_chunk_size = 0; + bool need_get_last_info = ! (ori_is_inrow || param.byte_size_ == 0); + if (param.lob_locator_ != nullptr && !param.lob_locator_->is_persist_lob()) { + ret = OB_NOT_IMPLEMENT; + LOG_WARN("Unsupport outrow tmp lob.", K(ret), K(param)); + } else if (OB_FAIL(param.get_store_chunk_size(store_chunk_size))) { + LOG_WARN("get_store_chunk_size fail", KR(ret), K(param)); + } else if (OB_FAIL(init_out_row_ctx(param, data.length(), param.op_type_))) { + LOG_WARN("init lob data out row ctx failed", K(ret)); + } else if (param.is_full_insert()) { + ObLobMetaWriteIter iter(param.get_tmp_allocator(), store_chunk_size); + if (OB_FAIL(iter.open(param, data))) { + LOG_WARN("Failed to open lob meta write iter.", K(ret), K(param)); + } else if (OB_FAIL(batch_insert(param, iter))) { + LOG_WARN("batch_insert fail", K(ret), K(param)); + } + } else { // outrow + ObLobMetaWriteIter iter(param.get_tmp_allocator(), store_chunk_size); + if (OB_FAIL(iter.open(param, data, need_get_last_info ? lob_ctx_.lob_meta_mngr_ : nullptr))) { + LOG_WARN("Failed to open lob meta write iter.", K(ret), K(param)); + } else if (OB_FAIL(write_outrow_result(param, iter))) { + LOG_WARN("write_outrow_result fail", K(ret), K(param)); + } + } + return ret; +} + +int ObLobManager::append_outrow( + ObLobAccessParam& param, + ObLobLocatorV2& lob, + int64_t append_lob_len, + ObString& ori_inrow_data) +{ + int ret = OB_SUCCESS; + ObLobCtx lob_ctx = lob_ctx_; + ObString read_buffer; + int64_t store_chunk_size = 0; + bool need_get_last_info = ! (ori_inrow_data.length() > 0 || param.byte_size_ == 0); + if (param.lob_locator_ != nullptr && !param.lob_locator_->is_persist_lob()) { + ret = OB_NOT_IMPLEMENT; + LOG_WARN("Unsupport outrow tmp lob.", K(ret), K(param)); + } else if (OB_FAIL(param.get_store_chunk_size(store_chunk_size))) { + LOG_WARN("get_store_chunk_size fail", KR(ret), K(param)); + } else if (OB_FAIL(init_out_row_ctx(param, append_lob_len, param.op_type_))) { + LOG_WARN("init lob data out row ctx failed", K(ret)); + } else if(OB_FAIL(prepare_data_buffer(param, read_buffer, OB_MIN(append_lob_len, store_chunk_size)))) { + LOG_WARN("fail to prepare reaed buffer", K(ret), K(param)); + } else { + SMART_VAR(ObLobAccessParam, read_param) { + read_param.tx_desc_ = param.tx_desc_; + read_param.tenant_id_ = param.src_tenant_id_; + if (OB_FAIL(build_lob_param(read_param, *param.get_tmp_allocator(), param.coll_type_, + 0, UINT64_MAX, param.timeout_, lob))) { + LOG_WARN("fail to build read param", K(ret), K(lob)); + } else { + ObLobQueryIter *iter = nullptr; + if (OB_FAIL(query(read_param, iter))) { + LOG_WARN("do query src by iter failed.", K(ret), K(read_param)); + } else { + ObLobMetaWriteIter write_iter(param.get_tmp_allocator(), store_chunk_size); + ObString remain_buf; + ObString seq_id_st; + ObString seq_id_ed; + if (OB_FAIL(write_iter.open( + param, iter, read_buffer, 0/*padding_size*/, + ori_inrow_data, remain_buf, seq_id_st, seq_id_ed, + need_get_last_info ? lob_ctx.lob_meta_mngr_ : nullptr))) { + LOG_WARN("failed to open meta writer", K(ret), K(param), K(write_iter)); + } else if (OB_FAIL(write_outrow_result(param, write_iter))) { + LOG_WARN("failed to write outrow result", K(ret), K(param), K(write_iter)); + } + } + if (OB_NOT_NULL(iter)) { + iter->reset(); + OB_DELETE(ObLobQueryIter, "unused", iter); + } + } + } + } + return ret; +} + +int ObLobManager::fill_outrow_with_zero(ObLobAccessParam& param) +{ + int ret = OB_SUCCESS; + param.lob_data_ = reinterpret_cast(param.lob_common_->buffer_); + ObLobMetaScanIter meta_iter; + ObString write_data_buffer; + int64_t store_chunk_size = 0; + if (OB_FAIL(param.get_store_chunk_size(store_chunk_size))) { + LOG_WARN("get_store_chunk_size fail", KR(ret), K(param)); + } else if (OB_FAIL(init_out_row_ctx(param, param.lob_data_->byte_size_, param.op_type_))) { + LOG_WARN("init lob data out row ctx failed", K(ret)); + } else if (OB_FAIL(lob_ctx_.lob_meta_mngr_->scan(param, meta_iter))) { + LOG_WARN("do lob meta scan failed.", K(ret), K(param)); + } else if(OB_FAIL(prepare_data_buffer(param, write_data_buffer, store_chunk_size))) { + LOG_WARN("fail to prepare buffers", K(ret), K(param)); + } else { + ObLobQueryResult result; + while (OB_SUCC(ret)) { + if (OB_FAIL(meta_iter.get_next_row(result.meta_result_))) { + if (ret == OB_ITER_END) { + } else { + LOG_WARN("failed to get next row.", K(ret)); + } + } else if (OB_FAIL(param.is_timeout())) { + LOG_WARN("access timeout", K(ret), K(param)); + } else if (OB_FAIL(do_fill_outrow_with_zero(param, store_chunk_size, meta_iter, result, write_data_buffer))) { + LOG_WARN("process erase meta info failed.", K(ret), K(param), K(result)); + } + } + if (ret == OB_ITER_END) { + ret = OB_SUCCESS; + } + } + return ret; +} + +int ObLobManager::do_fill_outrow_with_zero( + ObLobAccessParam& param, + const int64_t store_chunk_size, + ObLobMetaScanIter &meta_iter, + ObLobQueryResult &result, + ObString &write_buf) +{ + int ret = OB_SUCCESS; + bool is_char = param.is_char(); + + ObLobWriteBuffer buffer(param.coll_type_, store_chunk_size, param.is_store_char_len_); + + int64_t cur_piece_end = meta_iter.get_cur_pos(); + int64_t cur_piece_begin = cur_piece_end - (is_char ? result.meta_result_.info_.char_len_ : result.meta_result_.info_.byte_len_); + + int64_t piece_write_begin = param.offset_ > cur_piece_begin ? param.offset_ - cur_piece_begin : 0; + int64_t piece_write_end = param.offset_ + param.len_ > cur_piece_end ? cur_piece_end - cur_piece_begin : param.offset_ + param.len_ - cur_piece_begin; + + uint32_t piece_byte_len = result.meta_result_.info_.byte_len_; + uint32_t piece_char_len = result.meta_result_.info_.char_len_; + + ObLobMetaInfo new_meta_row = result.meta_result_.info_; + + if (OB_FAIL(buffer.set_buffer(write_buf.ptr(), write_buf.size()))) { + LOG_WARN("set_buffer fail", K(ret)); + } else if (OB_FAIL(buffer.append(result.meta_result_.info_.lob_data_.ptr(), result.meta_result_.info_.lob_data_.length()))) { + LOG_WARN("append fail", K(ret), K(param)); + } else if (OB_FAIL(buffer.char_fill_zero( + piece_write_begin, + piece_write_end - piece_write_begin))) { + LOG_WARN("fill_zero empty", K(ret), K(result)); + } else if (OB_FAIL(buffer.to_lob_meta_info(new_meta_row))) { + LOG_WARN("to_lob_meta_info fail", K(ret)); + } else if (OB_FAIL(update_one_piece( + param, + lob_ctx_, + result.meta_result_.info_, + new_meta_row, + result.piece_info_, + new_meta_row.lob_data_))) { + LOG_WARN("update_one_piece fail", K(ret), K(result), K(new_meta_row), + K(cur_piece_begin), K(cur_piece_end), K(piece_write_begin), K(piece_write_end)); + } + return ret; +} + +int ObLobManager::replace_process_meta_info(ObLobAccessParam& param, + const int64_t store_chunk_size, + ObLobMetaScanIter &meta_iter, + ObLobQueryResult &result, + ObLobQueryIter *iter, + ObString& read_buf, + ObString &remain_data, + ObString &write_buf) +{ + int ret = OB_SUCCESS; + ObLobCtx lob_ctx = lob_ctx_; + bool is_char = param.is_char(); + bool del_piece = false; + ObString temp_read_buf; + ObLobMetaInfo new_meta_row = result.meta_result_.info_; + + ObLobWriteBuffer buffer(param.coll_type_, store_chunk_size, param.is_store_char_len_); + + uint64_t cur_piece_end = meta_iter.get_cur_pos(); + uint64_t cur_piece_begin = cur_piece_end - (is_char ? result.meta_result_.info_.char_len_ : result.meta_result_.info_.byte_len_); + + int64_t piece_write_begin = param.offset_ > cur_piece_begin ? param.offset_ - cur_piece_begin : 0; + int64_t piece_write_end = param.offset_ + param.len_ > cur_piece_end ? cur_piece_end - cur_piece_begin : param.offset_ + param.len_ - cur_piece_begin; + + uint32_t piece_byte_len = result.meta_result_.info_.byte_len_; + uint32_t piece_char_len = result.meta_result_.info_.char_len_; + + int64_t replace_byte_st = ObCharset::charpos(param.coll_type_, result.meta_result_.info_.lob_data_.ptr(), result.meta_result_.info_.lob_data_.length(), piece_write_begin); + int64_t temp_read_size = store_chunk_size - replace_byte_st; + temp_read_buf.assign_buffer(read_buf.ptr(), temp_read_size); + if (iter->is_end()) { + } else if (OB_FAIL(iter->get_next_row(temp_read_buf))) { + if (ret != OB_ITER_END) { + LOG_WARN("fail to do get next read buffer", K(ret)); + } else { + ret = OB_SUCCESS; + } + } + + if (OB_FAIL(ret)) { + } else if (iter->is_end() && piece_write_end - piece_write_begin == piece_char_len) { + del_piece = true; + } else if (piece_write_end - piece_write_begin != piece_char_len) { + // use buffer because need concat data + if (OB_FAIL(buffer.set_buffer(write_buf.ptr(), write_buf.size()))) { + LOG_WARN("set_buffer fail", K(ret)); + } else if (OB_FAIL(buffer.append(result.meta_result_.info_.lob_data_.ptr(), result.meta_result_.info_.lob_data_.length()))) { + LOG_WARN("append fail", K(ret), K(param)); + } + } + + if (OB_FAIL(ret) || del_piece) { + } else if (OB_FAIL(buffer.char_write(piece_write_begin, piece_write_end - piece_write_begin, temp_read_buf, remain_data))) { + LOG_WARN("char_write fail", K(ret), K(cur_piece_begin), K(cur_piece_end), K(piece_write_begin), K(piece_write_end)); + } else if (OB_FAIL(buffer.to_lob_meta_info(new_meta_row))) { + LOG_WARN("to_lob_meta_info fail", K(ret)); + } + + if (OB_FAIL(ret)) { + } else if (del_piece) { + if (OB_FAIL(erase_one_piece(param, + lob_ctx, + result.meta_result_.info_, + result.piece_info_))) { + LOG_WARN("failed erase one piece", K(ret), K(result), + K(cur_piece_begin), K(cur_piece_end), K(piece_write_begin), K(piece_write_end), K(buffer)); + } + } else if (OB_FAIL(update_one_piece(param, + lob_ctx, + result.meta_result_.info_, + new_meta_row, + result.piece_info_, + new_meta_row.lob_data_))) { + LOG_WARN("failed to update.", K(ret), K(result), K(new_meta_row), + K(cur_piece_begin), K(cur_piece_end), K(piece_write_begin), K(piece_write_end), K(buffer)); + } + return ret; +} + +int ObLobManager::erase_process_meta_info(ObLobAccessParam& param, const int64_t store_chunk_size, ObLobMetaScanIter &meta_iter, + ObLobQueryResult &result, ObString &write_buf) +{ + int ret = OB_SUCCESS; + ObLobCtx lob_ctx = lob_ctx_; + bool is_char = param.is_char(); + bool del_piece = false; + + uint64_t cur_piece_end = meta_iter.get_cur_pos(); + uint64_t cur_piece_begin = cur_piece_end - (is_char ? result.meta_result_.info_.char_len_ : result.meta_result_.info_.byte_len_); + + int64_t piece_write_begin = param.offset_ > cur_piece_begin ? param.offset_ - cur_piece_begin : 0; + int64_t piece_write_end = param.offset_ + param.len_ > cur_piece_end ? cur_piece_end - cur_piece_begin : param.offset_ + param.len_ - cur_piece_begin; + + uint32_t piece_byte_len = result.meta_result_.info_.byte_len_; + uint32_t piece_char_len = result.meta_result_.info_.char_len_; + + ObLobMetaInfo new_meta_row = result.meta_result_.info_; + + ObLobWriteBuffer buffer(param.coll_type_, store_chunk_size, param.is_store_char_len_); + + if (piece_write_end - piece_write_begin == piece_char_len) { + del_piece = true; + LOG_DEBUG("just delete", K(piece_write_begin), K(piece_write_end), K(piece_char_len)); + } else if (meta_iter.is_range_begin(result.meta_result_.info_)) { + if (OB_FAIL(buffer.char_append(result.meta_result_.info_.lob_data_, 0/*char_offset*/, piece_write_begin /*char_len*/))) { // is_range_begin + LOG_WARN("char_append fail", K(ret)); + } + } else if (meta_iter.is_range_end(result.meta_result_.info_)) { + // use buffer because need copy data + if (OB_FAIL(buffer.set_buffer(write_buf.ptr(), write_buf.size()))) { + LOG_WARN("set_buffer fail", K(ret)); + } else if (OB_FAIL(buffer.char_append(result.meta_result_.info_.lob_data_, piece_write_begin, piece_write_end - piece_write_begin))) { + LOG_WARN("char_append fail", K(ret)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unkown piece fail", K(ret), K(result), K(cur_piece_begin), K(cur_piece_end), K(piece_write_begin), K(piece_write_end)); + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(buffer.to_lob_meta_info(new_meta_row))) { + LOG_WARN("to_lob_meta_info fail", K(ret)); + } else if (del_piece) { + if (OB_FAIL(erase_one_piece(param, lob_ctx, result.meta_result_.info_, result.piece_info_))) { + LOG_WARN("failed erase one piece", K(ret), K(result), + K(cur_piece_begin), K(cur_piece_end), K(piece_write_begin), K(piece_write_end), K(buffer)); + } + } else if (OB_FAIL(update_one_piece(param, + lob_ctx, + result.meta_result_.info_, + new_meta_row, + result.piece_info_, + new_meta_row.lob_data_))) { + LOG_WARN("failed to update.", K(ret), K(result), K(new_meta_row), + K(cur_piece_begin), K(cur_piece_end), K(piece_write_begin), K(piece_write_end), K(buffer)); + } + return ret; +} /*************ObLobQueryIter*****************/ int ObLobQueryIter::open(ObString &data, uint32_t byte_offset, uint32_t byte_len, ObCollationType cs, bool is_reverse) @@ -5057,34 +4607,29 @@ int ObLobPartialUpdateRowIter::get_next_row(int64_t &offset, ObLobMetaInfo *&old old_data = partial_data_.old_data_[idx.old_data_idx_].data_; } if (! idx.is_add_) { - old_meta_info_.lob_id_ = param_->lob_data_->id_; - old_meta_info_.seq_id_ = idx.seq_id_; - // old_meta_info_.offset_ = idx.offset_; - old_meta_info_.byte_len_ = old_data.length(); - // json column schema charset is utf8, may byte_len_ not equal char_len_ - old_meta_info_.char_len_ = old_meta_info_.byte_len_; - old_meta_info_.piece_id_ = ObLobMetaUtil::LOB_META_INLINE_PIECE_ID;; - old_meta_info_.lob_data_ = old_data; - - new_meta_info_.lob_id_ = param_->lob_data_->id_; - new_meta_info_.seq_id_ = idx.seq_id_; - // new_meta_info_.offset_ = idx.offset_; - new_meta_info_.byte_len_ = idx.byte_len_; - new_meta_info_.char_len_ = idx.byte_len_; - new_meta_info_.piece_id_ = ObLobMetaUtil::LOB_META_INLINE_PIECE_ID;; - new_meta_info_.lob_data_.assign_ptr(chunk_data.data_.ptr() + idx.pos_, idx.byte_len_); - - offset = idx.offset_; - old_info = &old_meta_info_; - new_info = &new_meta_info_; + if (OB_FAIL(ObLobMetaUtil::construct( + *param_, param_->lob_data_->id_, idx.seq_id_, + old_data.length(), old_data.length(), old_data, + old_meta_info_))) { + LOG_WARN("construct old lob_meta_info fail", K(ret), K(idx), K(old_data)); + } else if (OB_FAIL(ObLobMetaUtil::construct( + *param_, param_->lob_data_->id_, idx.seq_id_, + idx.byte_len_, idx.byte_len_, + ObString(idx.byte_len_, chunk_data.data_.ptr() + idx.pos_), + new_meta_info_))) { + LOG_WARN("construct new lob_meta_info fail", K(ret), K(idx), K(chunk_data)); + } else { + offset = idx.offset_; + old_info = &old_meta_info_; + new_info = &new_meta_info_; + } + } else if (OB_FAIL(ObLobMetaUtil::construct( + *param_, param_->lob_data_->id_, ObString(), + idx.byte_len_, idx.byte_len_, + ObString(idx.byte_len_, chunk_data.data_.ptr() + idx.pos_), + new_meta_info_))) { + LOG_WARN("construct new lob_meta_info fail", K(ret), K(idx), K(chunk_data)); } else { - new_meta_info_.lob_id_ = param_->lob_data_->id_; - // new_meta_info_.offset_ = idx.offset_; - new_meta_info_.byte_len_ = idx.byte_len_; - new_meta_info_.char_len_ = idx.byte_len_; - new_meta_info_.piece_id_ = ObLobMetaUtil::LOB_META_INLINE_PIECE_ID;; - new_meta_info_.lob_data_.assign_ptr(chunk_data.data_.ptr() + idx.pos_, idx.byte_len_); - offset = idx.offset_; new_info = &new_meta_info_; } diff --git a/src/storage/lob/ob_lob_manager.h b/src/storage/lob/ob_lob_manager.h index bee374cbdb..c1cc9684db 100644 --- a/src/storage/lob/ob_lob_manager.h +++ b/src/storage/lob/ob_lob_manager.h @@ -214,7 +214,7 @@ class ObLobManager public: static const int64_t LOB_AUX_TABLE_COUNT = 2; // lob aux table count for each table static const int64_t LOB_WITH_OUTROW_CTX_SIZE = sizeof(ObLobCommon) + sizeof(ObLobData) + sizeof(ObLobDataOutRowCtx); - static const int64_t LOB_OUTROW_FULL_SIZE = sizeof(ObLobCommon) + sizeof(ObLobData) + sizeof(ObLobDataOutRowCtx) + sizeof(uint64_t); + static const int64_t LOB_OUTROW_FULL_SIZE = ObLobLocatorV2::DISK_LOB_OUTROW_FULL_SIZE; static const uint64_t LOB_READ_BUFFER_LEN = 1024L*1024L; // 1M static const int64_t LOB_IN_ROW_MAX_LENGTH = 4096; // 4K static const uint64_t REMOTE_LOB_QUERY_RETRY_MAX = 10L; // 1M @@ -382,13 +382,13 @@ private: ObLobPieceInfo& piece_info, ObString& data); + int batch_delete(ObLobAccessParam& param, ObLobMetaScanIter &iter); + int batch_insert(ObLobAccessParam& param, ObLobMetaWriteIter &iter); int erase_one_piece(ObLobAccessParam& param, ObLobCtx& lob_ctx, ObLobMetaInfo& meta_info, ObLobPieceInfo& piece_info); - int batch_delete(ObLobAccessParam& param, ObLobMetaScanIter &iter); - void transform_query_result_charset(const common::ObCollationType& coll_type, const char* data, uint32_t len, @@ -402,12 +402,12 @@ private: bool &need_out_row); int init_out_row_ctx(ObLobAccessParam& param, uint64_t len, ObLobDataOutRowCtx::OpType op); int check_handle_size(ObLobAccessParam& param); - int erase_process_meta_info(ObLobAccessParam& param, ObLobMetaScanIter &meta_iter, ObLobQueryResult &result, ObString &tmp_buff); + int erase_process_meta_info(ObLobAccessParam& param, const int64_t store_chunk_size, ObLobMetaScanIter &meta_iter, ObLobQueryResult &result, ObString &tmp_buff); int prepare_for_write(ObLobAccessParam& param, ObString &old_data, bool &need_out_row); - int prepare_write_buffers(ObLobAccessParam& param, ObString &remain_buf, ObString &tmp_buf); int replace_process_meta_info(ObLobAccessParam& param, + const int64_t store_chunk_size, ObLobMetaScanIter &meta_iter, ObLobQueryResult &result, ObLobQueryIter *iter, @@ -420,7 +420,6 @@ private: int query_remote(ObLobAccessParam& param, ObString& data); int getlength_remote(ObLobAccessParam& param, common::ObAddr& dst_addr, uint64_t &len); int do_delete_one_piece(ObLobAccessParam& param, ObLobQueryResult &result, ObString &tmp_buff); - int prepare_erase_buffer(ObLobAccessParam& param, ObString &tmp_buff); int fill_zero(char *ptr, uint64_t length, bool is_char, const ObCollationType coll_type, uint32_t byte_len, uint32_t byte_offset, uint32_t &char_len); int prepare_lob_common(ObLobAccessParam& param, bool &alloc_inside); @@ -431,6 +430,16 @@ private: int64_t& result); int load_all(ObLobAccessParam ¶m, ObLobPartialData &partial_data); void transform_lob_id(uint64_t src, uint64_t &dst); + int append_outrow(ObLobAccessParam& param, ObLobLocatorV2& lob, int64_t append_lob_len, ObString& ori_inrow_data); + int append_outrow(ObLobAccessParam& param, bool ori_is_inrow, ObString &data); + int fill_outrow_with_zero(ObLobAccessParam& param); + int do_fill_outrow_with_zero( + ObLobAccessParam& param, + const int64_t store_chunk_size, + ObLobMetaScanIter &meta_iter, + ObLobQueryResult &result, + ObString &write_data_buffer); + private: static const int64_t DEFAULT_LOB_META_BUCKET_CNT = 1543; const uint64_t tenant_id_; diff --git a/src/storage/lob/ob_lob_meta.cpp b/src/storage/lob/ob_lob_meta.cpp index 0729cc48ca..c06dd16c27 100644 --- a/src/storage/lob/ob_lob_meta.cpp +++ b/src/storage/lob/ob_lob_meta.cpp @@ -11,15 +11,14 @@ */ #define USING_LOG_PREFIX STORAGE - +#include "ob_lob_meta.h" #include "lib/oblog/ob_log.h" #include "lib/objectpool/ob_server_object_pool.h" -#include "ob_lob_meta.h" -#include "ob_lob_seq.h" -#include "ob_lob_manager.h" +#include "storage/lob/ob_lob_seq.h" +#include "storage/lob/ob_lob_manager.h" +#include "storage/lob/ob_lob_write_buffer.h" #include "storage/tx_storage/ob_access_service.h" #include "storage/access/ob_table_scan_iterator.h" - namespace oceanbase { namespace storage @@ -75,7 +74,6 @@ int ObLobMetaScanIter::get_next_row(ObLobMetaInfo &row) } else { cur_info_ = row; if (is_char && row.char_len_ == UINT32_MAX) { - LOG_INFO("found no char_len, this only happen in inner QA upgrade test, can not happen in user situation", K(param_), KPC(this)); if (row.byte_len_ != param_.byte_size_) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected situation", K(ret), K(param_), KPC(this), K(row)); @@ -396,7 +394,7 @@ bool ObLobMetaScanIter::is_in_range(const ObLobMetaInfo& info) void ObLobMetaScanIter::reset() { - if (meta_iter_ != NULL) { + if (meta_iter_ != NULL && param_.access_ctx_ == NULL) { meta_iter_->reset(); if (lob_adatper_ != NULL) { (void)lob_adatper_->revert_scan_iter(meta_iter_); @@ -430,14 +428,12 @@ bool ObLobMetaScanIter::is_range_over(const ObLobMetaInfo& info) return cur_pos_ >= param_.offset_ + param_.len_; } -ObLobMetaWriteIter::ObLobMetaWriteIter(ObIAllocator* allocator) - : seq_id_(allocator), - offset_(0), +ObLobMetaWriteIter::ObLobMetaWriteIter( + ObIAllocator* allocator, + uint32_t piece_block_size + ) : seq_id_(allocator), lob_id_(), - piece_id_(0), - data_(), coll_type_(CS_TYPE_BINARY), - piece_block_size_(ObLobMetaUtil::LOB_OPER_PIECE_DATA_SIZE), scan_iter_(), padding_size_(0), seq_id_end_(allocator), @@ -450,8 +446,12 @@ ObLobMetaWriteIter::ObLobMetaWriteIter(ObIAllocator* allocator) iter_fill_size_(0), read_param_(nullptr), lob_common_(nullptr), - is_end_(false) + is_end_(false), + is_store_char_len_(true) { + offset_ = 0; + piece_id_ = 0; + piece_block_size_ = piece_block_size; } ObLobMetaWriteIter::ObLobMetaWriteIter( @@ -473,7 +473,8 @@ ObLobMetaWriteIter::ObLobMetaWriteIter( iter_fill_size_(0), read_param_(nullptr), lob_common_(nullptr), - is_end_(false) + is_end_(false), + is_store_char_len_(true) { data_ = data; offset_ = 0; @@ -481,12 +482,42 @@ ObLobMetaWriteIter::ObLobMetaWriteIter( piece_block_size_ = piece_block_size; } +int ObLobMetaWriteIter::get_last_meta_info(ObLobAccessParam ¶m, ObLobMetaManager* meta_manager) +{ + int ret = OB_SUCCESS; + // do reverse scan + bool save_is_reverse = param.scan_backward_; + param.scan_backward_ = true; + if (OB_ISNULL(meta_manager)) { + LOG_DEBUG("not scan last info", K(ret), K(param)); + } else if (OB_FAIL(meta_manager->scan(param, scan_iter_))) { + LOG_WARN("do lob meta scan failed.", K(ret), K(param)); + } else { + ObLobMetaScanResult scan_res; + if (OB_FAIL(scan_iter_.get_next_row(scan_res))) { + if (ret == OB_ITER_END) { + LOG_WARN("no found lob_meta_info", K(ret), K(param), K(scan_iter_)); + ret = OB_ENTRY_NOT_EXIST; + } else { + LOG_WARN("failed to get next row.", K(ret), K(param), K(scan_iter_)); + } + } else { + last_info_ = scan_res.info_; + seq_id_.set_seq_id(last_info_.seq_id_); + } + } + param.scan_backward_ = save_is_reverse; + return ret; +} + int ObLobMetaWriteIter::open(ObLobAccessParam ¶m, + ObString &data, uint64_t padding_size, ObString &post_data, ObString &remain_buf, ObString &seq_id_st, - ObString &seq_id_end) + ObString &seq_id_end, + ObLobMetaManager* meta_manager) { int ret = OB_SUCCESS; coll_type_ = param.coll_type_; @@ -496,58 +527,35 @@ int ObLobMetaWriteIter::open(ObLobAccessParam ¶m, seq_id_.set_seq_id(seq_id_st); seq_id_end_.set_seq_id(seq_id_end); post_data_ = post_data; + data_ = data; remain_buf_ = remain_buf; - char *buf = reinterpret_cast(allocator_->alloc(piece_block_size_)); - if (OB_ISNULL(buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("alloc buffer failed.", K(piece_block_size_)); - } else { - inner_buffer_.assign_ptr(buf, piece_block_size_); - } - return ret; -} + is_store_char_len_ = param.is_store_char_len_; + allocator_ = param.get_tmp_allocator(); -int ObLobMetaWriteIter::open(ObLobAccessParam ¶m, ObILobApator* adatper) -{ - int ret = OB_SUCCESS; - bool is_empty_lob = (param.byte_size_ == 0); - if (!is_empty_lob && OB_FAIL(scan_iter_.open(param, adatper))) { - LOG_WARN("failed open scan meta open iter.", K(ret)); - } else { - coll_type_ = param.coll_type_; - lob_id_ = param.lob_data_->id_; - piece_id_ = ObLobMetaUtil::LOB_META_INLINE_PIECE_ID; - padding_size_ = 0; - inner_buffer_.assign_ptr(nullptr, 0); - - // locate first piece - if (!is_empty_lob) { - ObLobMetaScanResult scan_res; - ret = scan_iter_.get_next_row(scan_res); - if (OB_FAIL(ret)) { - if (ret == OB_ITER_END) { - // empty table - ret = OB_SUCCESS; - } else { - LOG_WARN("failed to get next row.", K(ret)); - } - } else { - last_info_ = scan_res.info_; - seq_id_.set_seq_id(scan_res.info_.seq_id_); - } + if (OB_FAIL(get_last_meta_info(param, meta_manager))) { + LOG_WARN("get_last_meta_info fail", K(ret), K(param)); + } else if (0 == padding_size_ && post_data_.empty() && remain_buf_.empty() && last_info_.byte_len_ > 0) { + // need concat data, so alloc buffer + char *buf = reinterpret_cast(allocator_->alloc(piece_block_size_)); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc buffer failed.", K(piece_block_size_)); + } else { + inner_buffer_.assign_ptr(buf, piece_block_size_); } } return ret; } int ObLobMetaWriteIter::open(ObLobAccessParam ¶m, - void *iter, // ObLobQueryIter - ObString &read_buf, + void *iter, // ObLobQueryIter for data + ObString &read_buf, // ObLobQueryIter read buffer uint64_t padding_size, ObString &post_data, ObString &remain_buf, ObString &seq_id_st, - ObString &seq_id_end) + ObString &seq_id_end, + ObLobMetaManager* meta_manager) { int ret = OB_SUCCESS; if (OB_ISNULL(iter)) { @@ -563,9 +571,14 @@ int ObLobMetaWriteIter::open(ObLobAccessParam ¶m, post_data_ = post_data; remain_buf_ = remain_buf; iter_ = iter; + is_store_char_len_ = param.is_store_char_len_; data_.assign_buffer(read_buf.ptr(), piece_block_size_); - char *buf = reinterpret_cast(allocator_->alloc(piece_block_size_)); - if (OB_ISNULL(buf)) { + allocator_ = param.get_tmp_allocator(); + + char *buf = nullptr; + if (OB_FAIL(get_last_meta_info(param, meta_manager))) { + LOG_WARN("get_last_meta_info fail", K(ret), K(param)); + } else if (OB_ISNULL(buf = reinterpret_cast(allocator_->alloc(piece_block_size_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc buffer failed.", K(piece_block_size_)); } else { @@ -575,110 +588,97 @@ int ObLobMetaWriteIter::open(ObLobAccessParam ¶m, return ret; } +int ObLobMetaWriteIter::open(ObLobAccessParam ¶m, + ObString &data, + ObLobMetaManager* meta_manager) +{ + ObString post_data; + ObString remain_buf; + ObString seq_id_st; + ObString seq_id_end; + return open(param, data, 0/*padding_size*/, post_data, remain_buf, seq_id_st, seq_id_end, meta_manager); +} + int ObLobMetaWriteIter::open(ObLobAccessParam ¶m, void *iter, // ObLobQueryIter - void *read_param, // ObLobAccessParam + ObString &read_buf, + ObLobMetaManager* meta_manager) +{ + ObString post_data; + ObString remain_buf; + ObString seq_id_st; + ObString seq_id_end; + return open(param, iter, read_buf, 0/*padding_size*/, post_data, remain_buf, seq_id_st, seq_id_end, meta_manager); +} + +int ObLobMetaWriteIter::open(ObLobAccessParam ¶m, + void *iter, // ObLobQueryIter + ObLobAccessParam *read_param, ObString &read_buf) { int ret = OB_SUCCESS; + ObString post_data; + ObString remain_buf; + ObString seq_id_st; + ObString seq_id_end; if (OB_ISNULL(iter) || OB_ISNULL(read_param)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("null query iter", K(ret)); + } else if (OB_FAIL(open(param, iter, read_buf, 0/*padding_size*/, post_data, remain_buf, seq_id_st, seq_id_end, nullptr))) { + LOG_WARN("open fail", K(ret), K(param), KP(read_param)); } else { - coll_type_ = param.coll_type_; - lob_id_ = param.lob_data_->id_; - piece_id_ = ObLobMetaUtil::LOB_META_INLINE_PIECE_ID; - iter_ = iter; read_param_ = read_param; - allocator_ = param.allocator_; lob_common_ = param.lob_common_; - data_.assign_buffer(read_buf.ptr(), piece_block_size_); - char *buf = reinterpret_cast(allocator_->alloc(piece_block_size_)); - if (OB_ISNULL(buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("alloc buffer failed.", K(piece_block_size_)); - } else { - inner_buffer_.assign_ptr(buf, piece_block_size_); - } } return ret; } -int ObLobMetaWriteIter::try_fill_data( - ObLobMetaWriteResult& row, - bool &use_inner_buffer, - bool &fill_full) +int ObLobMetaWriteIter::try_fill_data(ObLobWriteBuffer &write_buffer, ObLobQueryIter *iter) { int ret = OB_SUCCESS; - ObLobQueryIter *iter = reinterpret_cast(iter_); int64_t by_len = 0; - int64_t char_len = 0; int64_t max_bytes_in_char = 16; - if (use_inner_buffer) { - // prepare tmp read buffer - ObString tmp_read_buf; - tmp_read_buf.assign_buffer(data_.ptr(), row.data_.remain()); - if (OB_FAIL(iter->get_next_row(tmp_read_buf))) { - if (ret != OB_ITER_END) { - LOG_WARN("fail to get next read buff", K(ret)); - } else { - ret = OB_SUCCESS; - } - } else if (tmp_read_buf.length() == 0) { - LOG_DEBUG("[LOB] get empty result"); - } else { - by_len = ObCharset::max_bytes_charpos(coll_type_, - tmp_read_buf.ptr(), - tmp_read_buf.length(), - tmp_read_buf.length(), - char_len); - by_len = ob_lob_writer_length_validation(coll_type_, tmp_read_buf.length(), by_len, char_len); - if (row.data_.write(tmp_read_buf.ptr(), by_len) != by_len) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to write data to inner buffer", K(ret), K(row.data_), K(by_len)); - } else { - fill_full = (row.data_.remain() <= max_bytes_in_char); - } - } + + // prepare read buffer + ObString data; + if (write_buffer.use_buffer()) { + data.assign_buffer(data_.ptr(), write_buffer.remain()); } else { - ObString tmp_read_buf; - tmp_read_buf.assign_buffer(data_.ptr(), piece_block_size_); - if (OB_FAIL(iter->get_next_row(tmp_read_buf))) { - if (ret != OB_ITER_END) { - LOG_WARN("fail to get next read buff", K(ret)); - } else { - ret = OB_SUCCESS; - } - } else if (tmp_read_buf.length() + max_bytes_in_char >= piece_block_size_ || inner_buffer_.ptr() == nullptr) { - // calc len and assign ptr - by_len = ObCharset::max_bytes_charpos(coll_type_, - tmp_read_buf.ptr(), - tmp_read_buf.length(), - tmp_read_buf.length(), - char_len); - by_len = ob_lob_writer_length_validation(coll_type_, tmp_read_buf.length(), by_len, char_len); - row.data_.assign_ptr(tmp_read_buf.ptr(), by_len); - fill_full = true; + data.assign_buffer(data_.ptr(), piece_block_size_); + } + + // read data and prepare write buffer mode + if (OB_FAIL(ret)) { + } else if (OB_FAIL(iter->get_next_row(data))) { + if (ret != OB_ITER_END) { + LOG_WARN("fail to get next read buff", K(ret)); } else { - // use inner buffer - use_inner_buffer = true; - row.data_.assign_buffer(inner_buffer_.ptr(), inner_buffer_.length()); - by_len = ObCharset::max_bytes_charpos(coll_type_, - tmp_read_buf.ptr(), - tmp_read_buf.length(), - tmp_read_buf.length(), - char_len); - by_len = ob_lob_writer_length_validation(coll_type_, tmp_read_buf.length(), by_len, char_len); - if (row.data_.write(tmp_read_buf.ptr(), by_len) != by_len) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to write data to inner buffer", K(ret), K(row.data_), K(by_len)); - } + ret = OB_SUCCESS; + } + } else if (data.length() == 0) { + LOG_INFO("[LOB] get empty result"); + } else if (write_buffer.use_buffer()) { + // already use buffer + } else if (data.length() + max_bytes_in_char >= piece_block_size_ || inner_buffer_.ptr() == nullptr) { + // not use buffer + } else { + // use buffer because need concat data + if (inner_buffer_.length() != piece_block_size_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("inner_buffer_ length not equal piece_block_size", K(ret), K(piece_block_size_), "inner_buffer_length", inner_buffer_.length()); + } else if (OB_FAIL(write_buffer.set_buffer(inner_buffer_.ptr(), piece_block_size_))) { + LOG_WARN("set_buffer fail", K(ret)); } } + + // write data + if (OB_FAIL(ret)) { + } else if (iter->is_end()) { + } else if (OB_FAIL(write_buffer.append(data.ptr(), data.length(), by_len))) { + LOG_WARN("set buffer data fail", K(ret)); + } + if (OB_SUCC(ret)) { - row.info_.byte_len_ += by_len; - row.info_.char_len_ += char_len; - OB_ASSERT(row.info_.byte_len_ >= row.info_.char_len_); offset_ += by_len; iter_fill_size_ += by_len; } @@ -686,129 +686,76 @@ int ObLobMetaWriteIter::try_fill_data( } int ObLobMetaWriteIter::try_fill_data( - ObLobMetaWriteResult& row, + ObLobWriteBuffer &write_buffer, ObString &data, uint64_t base, - bool is_padding, - bool &use_inner_buffer, - bool &fill_full) + bool is_padding) { int ret = OB_SUCCESS; uint64_t offset = offset_ - base; uint32_t left_size = is_padding ? (padding_size_ - offset) : (data.length() - offset); int64_t by_len = 0; - int64_t char_len = 0; - if (use_inner_buffer) { - // try write data to buffer - fill_full = (left_size >= row.data_.remain()); - by_len = (row.data_.remain() > left_size) ? left_size : row.data_.remain(); - if (is_padding) { - char_len = by_len; - if (coll_type_ == CS_TYPE_BINARY) { - MEMSET(row.data_.ptr() + row.data_.length(), '0x00', by_len); - } else { - MEMSET(row.data_.ptr() + row.data_.length(), ' ', by_len); - } - row.data_.set_length(row.data_.length() + by_len); - } else { - by_len = ObCharset::max_bytes_charpos(coll_type_, - data.ptr() + offset, - data.length() - offset, - by_len, - char_len); - by_len = ob_lob_writer_length_validation(coll_type_, data.length() - offset, by_len, char_len); - if (row.data_.write(data_.ptr() + offset, by_len) != by_len) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to write data to inner buffer", K(ret), K(row.data_), K(by_len)); - } - } + + // prepare buffer mode and max_write_byte_len + if (write_buffer.use_buffer()) { + // already use buffer + } else if ((left_size >= piece_block_size_ && !is_padding) || inner_buffer_.ptr() == nullptr) { + // not use buffer } else { - // check need use inner buffer - if ((left_size >= piece_block_size_ && !is_padding) || inner_buffer_.ptr() == nullptr) { - // calc len and assign ptr - int64_t max_bytes = piece_block_size_; - if (max_bytes > data.length() - offset) { - max_bytes = (data.length() - offset); - } - by_len = ObCharset::max_bytes_charpos(coll_type_, - data.ptr() + offset, - data.length() - offset, - max_bytes, - char_len); - by_len = ob_lob_writer_length_validation(coll_type_, data.length() - offset, by_len, char_len); - row.data_.assign_ptr(data_.ptr() + offset, by_len); - fill_full = true; - } else { - // use inner buffer - use_inner_buffer = true; - row.data_.assign_buffer(inner_buffer_.ptr(), inner_buffer_.length()); - if (is_padding) { - by_len = (row.data_.remain() > left_size) ? left_size : row.data_.remain(); - char_len = by_len; - fill_full = (left_size >= row.data_.remain()); - if (coll_type_ == CS_TYPE_BINARY) { - MEMSET(row.data_.ptr(), '0x00', by_len); - } else { - MEMSET(row.data_.ptr(), ' ', by_len); - } - row.data_.set_length(by_len); - } else { - by_len = ObCharset::max_bytes_charpos(coll_type_, - data.ptr() + offset, - data.length() - offset, - left_size, - char_len); - by_len = ob_lob_writer_length_validation(coll_type_, data.length() - offset, by_len, char_len); - if (row.data_.write(data.ptr() + offset, by_len) != by_len) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to write data to inner buffer", K(ret), K(row.data_), K(by_len)); - } - } + // use buffer + if (inner_buffer_.length() != piece_block_size_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("inner_buffer_ length not equal piece_block_size", K(ret), K(piece_block_size_), "inner_buffer_length", inner_buffer_.length()); + } else if (OB_FAIL(write_buffer.set_buffer(inner_buffer_.ptr(), piece_block_size_))) { + LOG_WARN("set_buffer fail", K(ret)); } } + + // write data + if (OB_FAIL(ret)) { + } else if (is_padding) { + // there is a trouble if charset is utf16 + if (OB_FAIL(write_buffer.padding(left_size, by_len))) { + LOG_WARN("padding fail", K(ret), K(left_size)); + } + } else if (OB_FAIL(write_buffer.append(data.ptr() + offset, left_size, by_len))) { + LOG_WARN("set buffer data fail", K(ret), K(left_size)); + } + if (OB_SUCC(ret)) { - row.info_.byte_len_ += by_len; - row.info_.char_len_ += char_len; - OB_ASSERT(row.info_.byte_len_ >= row.info_.char_len_); offset_ += by_len; } return ret; } -int ObLobMetaWriteIter::try_update_last_info(ObLobMetaWriteResult &row) +int ObLobMetaWriteIter::try_update_last_info( + ObLobMetaWriteResult &row, + ObLobWriteBuffer &write_buffer) { int ret = OB_SUCCESS; row.old_info_ = last_info_; row.is_update_ = false; bool is_fill_full = false; if (last_info_.seq_id_.ptr() != nullptr) { - if (last_info_.byte_len_ < piece_block_size_) { - int64_t by_len = piece_block_size_ - last_info_.byte_len_; - int64_t char_len = 0; - int64_t max_bytes = by_len > data_.length() ? data_.length() : by_len; - by_len = ObCharset::max_bytes_charpos(coll_type_, - data_.ptr(), - data_.length(), - max_bytes, - char_len); - if (by_len > 0) { - // prepare buffer - ObString buffer; - char *buf = reinterpret_cast(allocator_->alloc(piece_block_size_)); - if (OB_ISNULL(buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("alloc buffer failed.", K(piece_block_size_)); - } else { - buffer.assign_ptr(buf, piece_block_size_); - MEMCPY(buf, last_info_.lob_data_.ptr(), last_info_.lob_data_.length()); - MEMCPY(buf + last_info_.lob_data_.length(), data_.ptr(), by_len); - row.data_.assign_ptr(buf, by_len + last_info_.lob_data_.length()); - row.info_ = last_info_; - row.is_update_ = true; - row.info_.byte_len_ += by_len; - row.info_.char_len_ += char_len; - offset_ += by_len; - } + if (last_info_.byte_len_ > 0 && last_info_.byte_len_ < piece_block_size_) { + int64_t by_len = 0; + if (inner_buffer_.length() != piece_block_size_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("inner_buffer_ length not equal piece_block_size", K(ret), K(piece_block_size_), "inner_buffer_length", inner_buffer_.length()); + } else if (OB_FAIL(write_buffer.set_buffer(inner_buffer_.ptr(), piece_block_size_))) { + LOG_WARN("set_buffer fail", K(ret)); + } else if (OB_FAIL(write_buffer.append(last_info_.lob_data_.ptr(), last_info_.lob_data_.length(), by_len))) { + LOG_WARN("set buffer data fail", K(ret), K(last_info_), K(write_buffer)); + } else if (by_len != last_info_.lob_data_.length()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("should write all data", K(ret), K(by_len), K(last_info_), K(write_buffer)); + } else { + row.is_update_ = true; + row.info_ = last_info_; + // reset except lob_id, seq_id, piece_id + row.info_.lob_data_.reset(); + row.info_.byte_len_ = 0; + row.info_.char_len_ = 0; } } if (OB_SUCC(ret)) { @@ -822,16 +769,17 @@ int ObLobMetaWriteIter::try_update_last_info(ObLobMetaWriteResult &row) int ObLobMetaWriteIter::get_next_row(ObLobMetaWriteResult &row) { int ret = OB_SUCCESS; + ObLobWriteBuffer write_buffer(coll_type_, piece_block_size_, is_store_char_len_); if (is_end_) { ret = OB_ITER_END; // mock for inrow situation - } else if (OB_FAIL(try_update_last_info(row))) { + } else if (OB_FAIL(try_update_last_info(row, write_buffer))) { LOG_WARN("fail to do try update last info.", K(ret)); } else if (!row.is_update_) { // 1. init common info for ObLobMetaWriteResult row.need_alloc_macro_id_ = false; row.info_.lob_id_ = lob_id_; row.info_.piece_id_ = ObLobMetaUtil::LOB_META_INLINE_PIECE_ID; - row.info_.lob_data_ = ObString(); + row.info_.lob_data_.reset(); row.info_.byte_len_ = 0; row.info_.char_len_ = 0; ObString end_seq_id; @@ -847,55 +795,60 @@ int ObLobMetaWriteIter::get_next_row(ObLobMetaWriteResult &row) LOG_WARN("failed to next seq id.", K(ret)); } } - + } + if (OB_SUCC(ret)) { // 2. fill data to ObLobMetaWriteResult // in sequenced by [post_data][padding][data][remain_buf] - bool use_inner_buffer = false; - bool fill_full = false; - while (OB_SUCC(ret) && !fill_full) { + while (OB_SUCC(ret) && ! write_buffer.is_full()) { if (iter_ == nullptr) { if (post_data_.length() > offset_) { - ret = try_fill_data(row, post_data_, 0, false, use_inner_buffer, fill_full); + ret = try_fill_data(write_buffer, post_data_, 0/*base*/, false/*is_padding**/); } else if (padding_size_ > offset_ - post_data_.length()) { - ret = try_fill_data(row, post_data_, post_data_.length(), true, use_inner_buffer, fill_full); + ret = try_fill_data(write_buffer, post_data_, post_data_.length(), true/*is_padding**/); } else if (data_.length() > offset_ - post_data_.length() - padding_size_) { - ret = try_fill_data(row, data_, post_data_.length() + padding_size_, false, use_inner_buffer, fill_full); + ret = try_fill_data(write_buffer, data_, post_data_.length() + padding_size_, false/*is_padding**/); } else if (remain_buf_.length() > offset_ - post_data_.length() - padding_size_ - data_.length()) { - ret = try_fill_data(row, remain_buf_, post_data_.length() + padding_size_ + data_.length(), false, use_inner_buffer, fill_full); + ret = try_fill_data(write_buffer, remain_buf_, post_data_.length() + padding_size_ + data_.length(), false/*is_padding**/); } else { ret = OB_ITER_END; } } else { ObLobQueryIter *iter = reinterpret_cast(iter_); if (post_data_.length() > offset_) { - ret = try_fill_data(row, post_data_, 0, false, use_inner_buffer, fill_full); + ret = try_fill_data(write_buffer, post_data_, 0/*base*/, false/*is_padding**/); } else if (padding_size_ > offset_ - post_data_.length()) { - ret = try_fill_data(row, post_data_, post_data_.length(), true, use_inner_buffer, fill_full); + ret = try_fill_data(write_buffer, post_data_, post_data_.length(), true/*is_padding**/); } else if (!iter->is_end()) { - ret = try_fill_data(row, use_inner_buffer, fill_full); + ret = try_fill_data(write_buffer, iter); } else if (remain_buf_.length() > offset_ - post_data_.length() - padding_size_ - iter_fill_size_) { - ret = try_fill_data(row, remain_buf_, post_data_.length() + padding_size_ + iter_fill_size_, false, use_inner_buffer, fill_full); + ret = try_fill_data(write_buffer, remain_buf_, post_data_.length() + padding_size_ + iter_fill_size_, false/*is_padding**/); } else { ret = OB_ITER_END; } } } - if (ret == OB_ITER_END && row.info_.byte_len_ > 0) { + if (ret == OB_ITER_END && write_buffer.byte_length() > 0) { ret = OB_SUCCESS; } + + if (OB_SUCC(ret)) { + if (OB_FAIL(write_buffer.to_lob_meta_info(row.info_))) { + LOG_WARN("to_lob_meta_info fail", K(ret)); + } else if (row.info_.lob_data_.length() == 0 + || row.info_.lob_data_.length() > piece_block_size_ + || row.info_.lob_data_.length() != row.info_.byte_len_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result data over piece_block_size", K(ret), K(piece_block_size_), K(row.info_), K(row.is_update_)); + } else { + row.data_ = row.info_.lob_data_; + LOG_DEBUG("to_lob_meta_info", K(row.info_), K(write_buffer), K(offset_)); + } + } + if (OB_SUCC(ret) && OB_NOT_NULL(lob_common_)) { - // refresh byte len - ObLobCommon *lob_common = reinterpret_cast(lob_common_); - ObLobData *lob_data = reinterpret_cast(lob_common->buffer_); - lob_data->byte_size_ += row.info_.byte_len_; - // FIXME @haozheng direct load has no tx desc info, therefore do not need modified outrow ctx here - // maybe later we should need to update outrow ctx here for cdc - // refresh char len - char *ptr = reinterpret_cast(lob_common_); - int64_t *len = reinterpret_cast(ptr + ObLobManager::LOB_WITH_OUTROW_CTX_SIZE); - *len = *len + row.info_.char_len_; - // set lob data - row.info_.lob_data_.assign_ptr(row.data_.ptr(), row.data_.length()); + if (OB_FAIL(update_disk_lob_locator(row))) { + LOG_WARN("update_disk_lob_locator fail", K(ret), KPC(this)); + } } } return ret; @@ -943,11 +896,48 @@ void ObLobMetaWriteIter::reuse() is_end_ = false; } + void ObLobMetaWriteIter::set_data(const ObString& data) { data_ = data; } +int ObLobMetaWriteIter::update_disk_lob_locator(ObLobMetaWriteResult &result) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(lob_common_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("lob_common is null", K(ret), KPC(this), K(result)); + } else { + ObLobData *lob_data = reinterpret_cast(lob_common_->buffer_); + ObLobDataOutRowCtx *lob_outrow_ctx = reinterpret_cast(lob_data->buffer_); + uint64_t *char_len_ptr = reinterpret_cast(lob_outrow_ctx + 1); + + // update byte size + lob_data->byte_size_ += result.info_.byte_len_; + // update char_len + if (is_store_char_len_) { + *char_len_ptr = *char_len_ptr + result.info_.char_len_; + } else if (*char_len_ptr != UINT64_MAX) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("char len is incorrect", K(ret), K(*char_len_ptr), KPC(this), KPC(lob_common_), KPC(lob_outrow_ctx), KPC(lob_data), K(result)); + } + + if (OB_FAIL(ret)) { + } else if (lob_outrow_ctx->seq_no_st_ == 0) { + LOG_DEBUG("non incremental_direct_load does not have seq_no, so skip", KPC(lob_common_), KPC(lob_outrow_ctx), KPC(lob_data), K(result)); + } else { + transaction::ObTxSEQ cur_seq = transaction::ObTxSEQ::cast_from_int(lob_outrow_ctx->seq_no_st_) + lob_outrow_ctx->seq_no_cnt_; + + // update seq_no_cnt + lob_outrow_ctx->seq_no_cnt_++; + LOG_DEBUG("disk lob locator info", K(cur_seq), KPC(lob_common_), KPC(lob_outrow_ctx), KPC(lob_data), K(result)); + } + } + return ret; +} + + int ObLobMetaManager::write(ObLobAccessParam& param, ObLobMetaInfo& in_row) { int ret = OB_SUCCESS; @@ -957,15 +947,30 @@ int ObLobMetaManager::write(ObLobAccessParam& param, ObLobMetaInfo& in_row) return ret; } +int ObLobMetaManager::batch_insert(ObLobAccessParam& param, ObNewRowIterator &iter) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(persistent_lob_adapter_.write_lob_meta(param, iter))) { + LOG_WARN("batch write lob meta failed.", K(ret), K(param)); + } + return ret; +} + +int ObLobMetaManager::batch_delete(ObLobAccessParam& param, ObNewRowIterator &iter) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(persistent_lob_adapter_.erase_lob_meta(param, iter))) { + LOG_WARN("batch write lob meta failed.", K(ret), K(param)); + } + return ret; +} + // append int ObLobMetaManager::append(ObLobAccessParam& param, ObLobMetaWriteIter& iter) { int ret = OB_SUCCESS; - param.scan_backward_ = true; - - if (OB_FAIL(iter.open(param, &persistent_lob_adapter_))) { - LOG_WARN("fail open write iter.", K(ret), K(param)); - } + UNUSED(param); + UNUSED(iter); return ret; } @@ -1039,73 +1044,6 @@ int ObLobMetaManager::fetch_lob_id(ObLobAccessParam& param, uint64_t &lob_id) return ret; } - -int ObLobMetaWriteRowIter::get_next_row(ObNewRow *&row) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(meta_iter_->get_next_row(result_))) { - if (ret != OB_ITER_END) { - LOG_WARN("get next meta info failed.", K(ret)); - } - } else { - result_.info_.lob_data_.assign_ptr(result_.data_.ptr(), result_.data_.length()); - for (int64_t i = 0; i < ObLobMetaUtil::LOB_META_COLUMN_CNT; ++i) { - row_cell_[i].reset(); - row_cell_[i].set_nop_value(); - } - row_cell_[ObLobMetaUtil::LOB_ID_COL_ID].set_varchar(reinterpret_cast(&result_.info_.lob_id_), sizeof(ObLobId)); - row_cell_[ObLobMetaUtil::LOB_ID_COL_ID].set_collation_type(common::ObCollationType::CS_TYPE_BINARY); - row_cell_[ObLobMetaUtil::SEQ_ID_COL_ID].set_varchar(result_.info_.seq_id_); - row_cell_[ObLobMetaUtil::SEQ_ID_COL_ID].set_collation_type(common::ObCollationType::CS_TYPE_BINARY); - row_cell_[ObLobMetaUtil::BYTE_LEN_COL_ID].set_uint32(result_.info_.byte_len_); - row_cell_[ObLobMetaUtil::CHAR_LEN_COL_ID].set_uint32(result_.info_.char_len_); - row_cell_[ObLobMetaUtil::PIECE_ID_COL_ID].set_uint64(result_.info_.piece_id_); - - row_cell_[ObLobMetaUtil::LOB_DATA_COL_ID].set_varchar(result_.info_.lob_data_); - row_cell_[ObLobMetaUtil::LOB_DATA_COL_ID].set_collation_type(common::ObCollationType::CS_TYPE_BINARY); - - new_row_.assign(row_cell_, ObLobMetaUtil::LOB_META_COLUMN_CNT); - // refresh param - // refrsh outrow ctx for cdc - if (OB_FAIL(ret)) { - } else if (OB_FAIL(ObLobManager::update_out_ctx(*param_, nullptr, result_.info_))) { // new row - LOG_WARN("failed update checksum.", K(ret)); - } else if (OB_FAIL(update_seq_no())) { - LOG_WARN("update_seq_no fail", K(ret)); - } else { - param_->lob_data_->byte_size_ += result_.info_.byte_len_; - param_->byte_size_ = param_->lob_data_->byte_size_; - if (ObLobManager::lob_handle_has_char_len(*param_)) { - int64_t *len = ObLobManager::get_char_len_ptr(*param_); - *len = *len + result_.info_.char_len_; - OB_ASSERT(*len >= 0); - } - // set output - row = &new_row_; - } - } - return ret; -} - -int ObLobMetaWriteRowIter::update_seq_no() -{ - int ret = OB_SUCCESS; - if (param_->seq_no_st_.is_valid()) { - if (param_->used_seq_cnt_ < param_->total_seq_cnt_) { - param_->dml_base_param_->spec_seq_no_ = param_->seq_no_st_ + param_->used_seq_cnt_; - param_->used_seq_cnt_++; - LOG_DEBUG("dml lob meta with seq no", K(param_->dml_base_param_->spec_seq_no_)); - } else { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("failed to get seq no from param.", K(ret), KPC(param_)); - } - } else { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid seq no from param.", K(ret), KPC(param_)); - } - return ret; -} - ObLobMetaSingleGetter::~ObLobMetaSingleGetter() { if (OB_NOT_NULL(row_objs_)) { @@ -1204,6 +1142,5 @@ int ObLobMetaSingleGetter::get_next_row(ObString &seq_id, ObLobMetaInfo &info) return ret; } - } } diff --git a/src/storage/lob/ob_lob_meta.h b/src/storage/lob/ob_lob_meta.h index 290b6b67ac..191ed2f508 100644 --- a/src/storage/lob/ob_lob_meta.h +++ b/src/storage/lob/ob_lob_meta.h @@ -43,6 +43,25 @@ public: public: static int transform_from_info_to_row(ObLobMetaInfo &info, blocksstable::ObDatumRow *row, bool with_extra_rowkey); static int transform_from_row_to_info(const blocksstable::ObDatumRow *row, ObLobMetaInfo &info, bool with_extra_rowkey); + static int construct( + ObLobAccessParam ¶m, + const ObLobId &lob_id, + const ObString &seq_id, + const uint32_t &byte_len, + const uint32_t &char_len, + const ObString &lob_data, + ObLobMetaInfo &info) + { + int ret = OB_SUCCESS; + info.lob_id_ = lob_id; + info.seq_id_ = seq_id; + info.byte_len_ = byte_len; + info.char_len_ = char_len; + info.piece_id_ = ObLobMetaUtil::LOB_META_INLINE_PIECE_ID; + info.lob_data_ = lob_data; + return ret; + } + private: // from_row_to_info. static int transform_lob_id(const blocksstable::ObDatumRow *row, ObLobMetaInfo &info); @@ -102,6 +121,10 @@ private: bool not_need_last_info_; }; +class ObLobWriteBuffer; +class ObLobQueryIter; +class ObLobMetaManager; + struct ObLobMetaWriteResult { ObLobMetaWriteResult() : info_(), data_(), need_alloc_macro_id_(false), is_update_(false), old_info_() {} ObLobMetaInfo info_; @@ -109,20 +132,23 @@ struct ObLobMetaWriteResult { bool need_alloc_macro_id_; bool is_update_; ObLobMetaInfo old_info_; + + TO_STRING_KV(K_(is_update), K_(info), K_(old_info), K_(data)); }; class ObLobMetaWriteIter { public: - ObLobMetaWriteIter(ObIAllocator* allocator); + ObLobMetaWriteIter(ObIAllocator* allocator, uint32_t piece_block_size); ObLobMetaWriteIter(const ObString& data, ObIAllocator* allocator, uint32_t piece_block_size); ~ObLobMetaWriteIter() { close(); } int open(ObLobAccessParam ¶m, + ObString &data, uint64_t padding_size, ObString &post_data, ObString &remain_buf, ObString &seq_id_st, - ObString &seq_id_end); - int open(ObLobAccessParam ¶m, ObILobApator* adapter); + ObString &seq_id_end, + ObLobMetaManager* meta_manager = nullptr); int open(ObLobAccessParam ¶m, void *iter, // ObLobQueryIter ObString &read_buf, @@ -130,10 +156,18 @@ public: ObString &post_data, ObString &remain_buf, ObString &seq_id_st, - ObString &seq_id_end); + ObString &seq_id_end, + ObLobMetaManager* meta_manager = nullptr); + int open(ObLobAccessParam ¶m, + ObString &data, + ObLobMetaManager* meta_manager = nullptr); int open(ObLobAccessParam ¶m, void *iter, // ObLobQueryIter - void *read_param, // ObLobAccessParam + ObString &read_buf, + ObLobMetaManager* meta_manager = nullptr); + int open(ObLobAccessParam ¶m, + void *iter, // ObLobQueryIter + ObLobAccessParam *read_param, // ObLobAccessParam ObString &read_buf); int get_next_row(ObLobMetaWriteResult &row); int close(); @@ -141,20 +175,21 @@ public: void reuse(); void set_data(const ObString& data); TO_STRING_KV(K_(seq_id), K_(offset), K_(lob_id), K_(piece_id), K_(coll_type), K_(piece_block_size), - K_(scan_iter), K_(padding_size), K_(seq_id_end), K_(last_info)); + K_(scan_iter), K_(padding_size), K_(seq_id_end), K_(last_info), K_(is_store_char_len)); private: int try_fill_data( - ObLobMetaWriteResult& row, + ObLobWriteBuffer& write_buffer, ObString &data, uint64_t base, - bool is_padding, - bool &use_inner_buffer, - bool &fill_full); + bool is_padding); int try_fill_data( - ObLobMetaWriteResult& row, - bool &use_inner_buffer, - bool &fill_full); - int try_update_last_info(ObLobMetaWriteResult &row); + ObLobWriteBuffer& write_buffer, + ObLobQueryIter *iter); + int get_last_meta_info(ObLobAccessParam ¶m, ObLobMetaManager* meta_manager); + int try_update_last_info(ObLobMetaWriteResult &row, ObLobWriteBuffer& write_buffer); + + int update_disk_lob_locator(ObLobMetaWriteResult &new_info); + private: ObLobSeqId seq_id_; // seq id uint64_t offset_; // write or append offset in macro block @@ -173,9 +208,10 @@ private: ObLobMetaInfo last_info_; void *iter_; // ObLobQueryIter uint64_t iter_fill_size_; - void *read_param_; // ObLobAccessParam - void* lob_common_; // ObLobCommon + ObLobAccessParam *read_param_; + ObLobCommon* lob_common_; bool is_end_; + bool is_store_char_len_; }; class ObLobMetaManager { @@ -186,6 +222,8 @@ public: ~ObLobMetaManager() {} // write one lob meta row int write(ObLobAccessParam& param, ObLobMetaInfo& in_row); + int batch_insert(ObLobAccessParam& param, ObNewRowIterator &iter); + int batch_delete(ObLobAccessParam& param, ObNewRowIterator &iter); // append int append(ObLobAccessParam& param, ObLobMetaWriteIter& iter); // return ObLobMetaWriteResult diff --git a/src/storage/lob/ob_lob_persistent_adaptor.cpp b/src/storage/lob/ob_lob_persistent_adaptor.cpp index ec5ed640cb..ea0777e248 100644 --- a/src/storage/lob/ob_lob_persistent_adaptor.cpp +++ b/src/storage/lob/ob_lob_persistent_adaptor.cpp @@ -17,6 +17,8 @@ #include "ob_lob_persistent_adaptor.h" #include "ob_lob_piece.h" #include "ob_lob_meta.h" +#include "ob_lob_persistent_iterator.h" +#include "ob_lob_persistent_reader.h" #include "storage/meta_mem/ob_tenant_meta_mem_mgr.h" #include "share/schema/ob_table_dml_param.h" #include "share/schema/ob_tenant_schema_service.h" @@ -184,75 +186,6 @@ int ObPersistentLobApator::prepare_piece_table_param( return ret; } -int ObPersistentLobApator::scan_lob_meta( - ObLobAccessParam ¶m, - ObTableScanParam &scan_param, - common::ObNewRowIterator *&meta_iter) -{ - int ret = OB_SUCCESS; - // 1. get tablets - ObTabletHandle data_tablet; - ObTabletHandle lob_meta_tablet; - ObTabletHandle lob_piece_tablet; - if (OB_FAIL(get_lob_tablets(param, data_tablet, lob_meta_tablet, lob_piece_tablet))) { - LOG_WARN("failed to get tablets.", K(ret), K(param)); - } else { - uint64_t tenant_id = param.tenant_id_; - // 2. prepare tablet scan param - scan_param.tablet_id_ = lob_meta_tablet.get_obj()->get_tablet_meta().tablet_id_; - scan_param.schema_version_ = lob_meta_tablet.get_obj()->get_tablet_meta().max_sync_storage_schema_version_; - const uint64_t table_id = 0; - scan_param.table_param_ = param.meta_tablet_param_; - if (OB_FAIL(build_common_scan_param(param, table_id, false, ObLobMetaUtil::LOB_META_COLUMN_CNT, scan_param))) { - LOG_WARN("build common scan param failed.", K(ret)); - } else if (OB_FAIL(prepare_table_param(param, scan_param, true))) { - LOG_WARN("prepare lob meta table param failed.", K(ret)); - } else { - // build key range - void *buf = param.allocator_->alloc(sizeof(ObObj) * 4); - if (OB_ISNULL(buf)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("alloc range obj failed.", K(ret)); - } else { - ObObj *row_objs = reinterpret_cast(buf); - const char *lob_id_ptr_ = reinterpret_cast(¶m.lob_data_->id_); - row_objs[0].reset(); - row_objs[0].set_varchar(lob_id_ptr_, sizeof(ObLobId)); // lob_id - row_objs[0].set_collation_type(common::ObCollationType::CS_TYPE_BINARY); - row_objs[1] = ObObj::make_min_obj(); // seq_id set min - ObRowkey min_row_key(row_objs, 2); - - row_objs[2].reset(); - row_objs[2].set_varchar(lob_id_ptr_, sizeof(ObLobId)); // lob_id - row_objs[2].set_collation_type(common::ObCollationType::CS_TYPE_BINARY); - row_objs[3] = ObObj::make_max_obj(); // seq_id set max - ObRowkey max_row_key(row_objs + 2, 2); - - common::ObNewRange range; - range.table_id_ = table_id; - range.start_key_ = min_row_key; - range.end_key_ = max_row_key; - range.border_flag_.set_inclusive_start(); - range.border_flag_.set_inclusive_end(); - scan_param.key_ranges_.reset(); - if (OB_FAIL(scan_param.key_ranges_.push_back(range))) { - LOG_WARN("failed to push key range.", K(ret), K(scan_param), K(range)); - } else { - ObAccessService *oas = MTL(ObAccessService*); - if (OB_ISNULL(oas)) { - ret = OB_ERR_INTERVAL_INVALID; - LOG_WARN("get access service failed.", K(ret)); - } else if (OB_FAIL(oas->table_scan(scan_param, meta_iter))) { - LOG_WARN("do table scan falied.", K(ret), K(scan_param)); - } - } - } - param.allocator_->free(buf); - } - } - return ret; -} - int ObPersistentLobApator::get_lob_data( ObLobAccessParam ¶m, uint64_t piece_id, @@ -278,7 +211,7 @@ int ObPersistentLobApator::get_lob_data( const uint64_t table_id = 0; bool tmp_scan_backward = param.scan_backward_; param.scan_backward_ = false; - if (OB_FAIL(build_common_scan_param(param, table_id, false, ObLobPieceUtil::LOB_PIECE_COLUMN_CNT, scan_param))) { + if (OB_FAIL(build_common_scan_param(param, false, ObLobPieceUtil::LOB_PIECE_COLUMN_CNT, scan_param))) { LOG_WARN("build common scan param failed.", K(ret)); } else if (OB_FAIL(prepare_table_param(param, scan_param, false))) { LOG_WARN("prepare lob meta table param failed.", K(ret)); @@ -372,54 +305,30 @@ int ObPersistentLobApator::fetch_lob_id(ObLobAccessParam& param, uint64_t &lob_i return ret; } -int ObPersistentLobApator::prepare_lob_meta_dml( - ObLobAccessParam& param, - const uint64_t tenant_id, - const ObTabletHandle& data_tablet, - const ObTabletHandle& lob_meta_tablet) +int ObPersistentLobApator::prepare_lob_meta_dml(ObLobAccessParam& param) { int ret = OB_SUCCESS; if (param.dml_base_param_ == nullptr) { - share::schema::ObTableDMLParam* table_dml_param = nullptr; ObStoreCtxGuard *store_ctx_guard = nullptr; void *buf = param.allocator_->alloc(sizeof(ObDMLBaseParam) + sizeof(ObStoreCtxGuard)); if (OB_ISNULL(buf)) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc dml base param", K(ret)); + LOG_WARN("failed to alloc dml base param", K(ret), K(param)); } else { param.dml_base_param_ = new(buf)ObDMLBaseParam(); store_ctx_guard = new((char*)buf + sizeof(ObDMLBaseParam)) ObStoreCtxGuard(); - - buf = param.allocator_->alloc(sizeof(share::schema::ObTableDMLParam)); - if (OB_ISNULL(buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc table dml param", K(ret)); - } else { - table_dml_param = new(buf)share::schema::ObTableDMLParam(*param.allocator_); - } } if (OB_FAIL(ret)) { - } else if (OB_FAIL(build_lob_meta_table_dml(param, tenant_id, table_dml_param, - *param.dml_base_param_, store_ctx_guard, - param.column_ids_, data_tablet, lob_meta_tablet))) { - LOG_WARN("failed to build meta schema", K(ret), K(data_tablet), K(lob_meta_tablet)); - } - } else { - if (param.seq_no_st_.is_valid()) { - if (param.used_seq_cnt_ < param.total_seq_cnt_) { - param.dml_base_param_->spec_seq_no_ = param.seq_no_st_ + param.used_seq_cnt_; - param.used_seq_cnt_++; - LOG_DEBUG("dml lob meta with seq no", K(param.dml_base_param_->spec_seq_no_)); - } else { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("failed to get seq no from param.", K(ret), K(param)); - } - } else { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid seq no from param.", K(ret), K(param)); + } else if (OB_FAIL(build_lob_meta_table_dml(param, *param.dml_base_param_, store_ctx_guard, param.column_ids_))) { + LOG_WARN("failed to build lob meta table dml param", K(ret), K(param)); } } + + if (OB_SUCC(ret) && OB_FAIL(set_dml_seq_no(param))) { + LOG_WARN("update_seq_no fail", K(ret), K(param)); + } + if (OB_SUCC(ret)) { param.dml_base_param_->store_ctx_guard_->reset(); ObAccessService *oas = MTL(ObAccessService *); @@ -438,13 +347,9 @@ int ObPersistentLobApator::prepare_lob_meta_dml( int ObPersistentLobApator::build_lob_meta_table_dml( ObLobAccessParam& param, - const uint64_t tenant_id, - ObTableDMLParam* dml_param, ObDMLBaseParam& dml_base_param, ObStoreCtxGuard *store_ctx_guard, - ObSEArray& column_ids, - const ObTabletHandle& data_tablet, - const ObTabletHandle& lob_meta_tablet) + ObSEArray& column_ids) { int ret = OB_SUCCESS; // dml base @@ -455,44 +360,16 @@ int ObPersistentLobApator::build_lob_meta_table_dml( dml_base_param.encrypt_meta_ = &dml_base_param.encrypt_meta_legacy_; dml_base_param.snapshot_ = param.snapshot_; dml_base_param.check_schema_version_ = false; // lob tablet should not check schema version + dml_base_param.schema_version_ = 0; dml_base_param.store_ctx_guard_ = store_ctx_guard; dml_base_param.write_flag_.set_is_insert_up(); dml_base_param.write_flag_.set_lob_aux(); - - if (param.seq_no_st_.is_valid()) { - if (param.used_seq_cnt_ < param.total_seq_cnt_) { - dml_base_param.spec_seq_no_ = param.seq_no_st_ + param.used_seq_cnt_; - param.used_seq_cnt_++; - LOG_DEBUG("dml lob meta with seq no", K(dml_base_param.spec_seq_no_)); - } else { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("failed to get seq no from param.", K(ret), K(param)); - } - } else { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid seq no from param.", K(ret), K(param)); - } - { - const ObTableSchema* table_schema = param.meta_table_schema_; - for (int i = 0; OB_SUCC(ret) && i < ObLobMetaUtil::LOB_META_COLUMN_CNT; ++i) { if (OB_FAIL(column_ids.push_back(OB_APP_MIN_COLUMN_ID + i))) { LOG_WARN("push column ids failed.", K(ret), K(i)); } } - - if (OB_FAIL(ret)) { - } else if (table_schema == nullptr) { - dml_base_param.schema_version_ = lob_meta_tablet.get_obj()->get_tablet_meta().max_sync_storage_schema_version_; - } else { - /** - * for test current - */ - dml_base_param.schema_version_ = share::OB_CORE_SCHEMA_VERSION + 1; - dml_base_param.tenant_schema_version_ = share::OB_CORE_SCHEMA_VERSION + 1; - } - if (OB_FAIL(ret)) { } else if (OB_FAIL(get_meta_table_dml_param(dml_base_param.table_param_))) { LOG_WARN("get_meta_table_dml_param fail", KR(ret)); @@ -501,56 +378,6 @@ int ObPersistentLobApator::build_lob_meta_table_dml( return ret; } -int ObPersistentLobApator::erase_lob_meta(ObLobAccessParam ¶m, ObLobMetaInfo& in_row) -{ - int ret = OB_SUCCESS; - - // 1. get tablets - ObTabletHandle data_tablet; - ObTabletHandle lob_meta_tablet; - ObTabletHandle lob_piece_tablet; - // get Access service - ObAccessService *oas = MTL(ObAccessService*); - - if (OB_ISNULL(oas)) { - ret = OB_ERR_INTERVAL_INVALID; - LOG_WARN("get access service failed.", K(ret), KP(oas)); - } else if (OB_FAIL(get_lob_tablets(param, - data_tablet, - lob_meta_tablet, - lob_piece_tablet))) { - LOG_WARN("failed to get tablets.", K(ret), K(param)); - } else if (OB_ISNULL(param.tx_desc_)) { - ret = OB_ERR_NULL_VALUE; - LOG_WARN("get tx desc null.", K(ret), K(param)); - } else { - uint64_t tenant_id = param.tenant_id_; - if (OB_FAIL(prepare_lob_meta_dml(param, tenant_id, data_tablet, lob_meta_tablet))) { - LOG_WARN("failed to prepare lob meta dml", K(ret)); - } else { - // get tx desc - transaction::ObTxDesc* tx_desc = param.tx_desc_; - - int64_t affected_rows = 0; - ObObj cell[ObLobMetaUtil::LOB_META_COLUMN_CNT]; - ObNewRow new_row; - common::ObSingleRowIteratorWrapper single_iter; - set_lob_meta_row(cell, new_row, in_row); - single_iter.set_row(&new_row); - if (OB_FAIL(oas->delete_rows(param.ls_id_, - lob_meta_tablet.get_obj()->get_tablet_meta().tablet_id_, - *tx_desc, - *param.dml_base_param_, - param.column_ids_, - &single_iter, - affected_rows))) { - LOG_WARN("failed to insert row.", K(ret)); - } - } - } - return ret; -} - int ObPersistentLobApator::erase_lob_piece_tablet(ObLobAccessParam& param, ObLobPieceInfo& in_row) { int ret = OB_SUCCESS; @@ -614,120 +441,6 @@ int ObPersistentLobApator::erase_lob_piece_tablet(ObLobAccessParam& param, ObLob return ret; } -int ObPersistentLobApator::write_lob_meta(ObLobAccessParam& param, ObLobMetaInfo& in_row) -{ - int ret = OB_SUCCESS; - - // 1. get tablets - ObTabletHandle data_tablet; - ObTabletHandle lob_meta_tablet; - ObTabletHandle lob_piece_tablet; - - ObAccessService *oas = MTL(ObAccessService*); - - if (OB_ISNULL(oas)) { - ret = OB_ERR_INTERVAL_INVALID; - LOG_WARN("get access service failed.", K(ret), KP(oas)); - } else if (OB_FAIL(get_lob_tablets(param, - data_tablet, - lob_meta_tablet, - lob_piece_tablet))) { - LOG_WARN("failed to get tablets.", K(ret), K(param)); - } else if (OB_ISNULL(param.tx_desc_)) { - ret = OB_ERR_NULL_VALUE; - LOG_WARN("get tx desc null.", K(ret), K(param)); - } else { - uint64_t tenant_id = param.tenant_id_; - if (OB_FAIL(prepare_lob_meta_dml(param, tenant_id, data_tablet, lob_meta_tablet))) { - LOG_WARN("failed to prepare lob meta dml.", K(ret)); - } else { - // get tx desc - transaction::ObTxDesc* tx_desc = param.tx_desc_;; - - int64_t affected_rows = 0; - ObObj cell[ObLobMetaUtil::LOB_META_COLUMN_CNT]; - ObNewRow new_row; - common::ObSingleRowIteratorWrapper single_iter; - set_lob_meta_row(cell, new_row, in_row); - single_iter.set_row(&new_row); - if (OB_FAIL(oas->insert_rows(param.ls_id_, - lob_meta_tablet.get_obj()->get_tablet_meta().tablet_id_, - *tx_desc, - *param.dml_base_param_, - param.column_ids_, - &single_iter, - affected_rows))) { - LOG_WARN("failed to insert row.", K(ret)); - } - } - } - return ret; -} - -int ObPersistentLobApator::update_lob_meta(ObLobAccessParam& param, ObLobMetaInfo& old_row, ObLobMetaInfo& new_row) -{ - int ret = OB_SUCCESS; - - // 1. get tablets - ObTabletHandle data_tablet; - ObTabletHandle lob_meta_tablet; - ObTabletHandle lob_piece_tablet; - - // get Access service - ObAccessService *oas = MTL(ObAccessService*); - - if (OB_ISNULL(oas)) { - ret = OB_ERR_INTERVAL_INVALID; - LOG_WARN("get access service failed.", K(ret), KP(oas)); - } else if (OB_FAIL(get_lob_tablets(param, - data_tablet, - lob_meta_tablet, - lob_piece_tablet))) { - LOG_WARN("failed to get tablets.", K(ret), K(param)); - } else if (OB_ISNULL(param.tx_desc_)) { - ret = OB_ERR_NULL_VALUE; - LOG_WARN("get tx desc null.", K(ret), K(param)); - } else { - uint64_t tenant_id = param.tenant_id_; - ObSEArray update_column_ids; - - for (int i = 2; OB_SUCC(ret) && i < ObLobMetaUtil::LOB_META_COLUMN_CNT; ++i) { - if (OB_FAIL(update_column_ids.push_back(OB_APP_MIN_COLUMN_ID + i))) { - LOG_WARN("push column ids failed.", K(ret), K(i)); - } - } - - if (OB_FAIL(ret)) { - } else if (OB_FAIL(prepare_lob_meta_dml(param, tenant_id, data_tablet, lob_meta_tablet))) { - LOG_WARN("failed to prepare lob meta dml", K(ret)); - } else { - // get tx desc - transaction::ObTxDesc* tx_desc = param.tx_desc_; - - int64_t affected_rows = 0; - ObObj new_row_cell[ObLobMetaUtil::LOB_META_COLUMN_CNT]; - ObNewRow new_tbl_row; - set_lob_meta_row(new_row_cell, new_tbl_row, new_row); - ObObj old_row_cell[ObLobMetaUtil::LOB_META_COLUMN_CNT]; - ObNewRow old_tbl_row; - set_lob_meta_row(old_row_cell, old_tbl_row, old_row); - ObLobUpdIterator upd_iter(&old_tbl_row, &new_tbl_row); - if (OB_FAIL(oas->update_rows(param.ls_id_, - lob_meta_tablet.get_obj()->get_tablet_meta().tablet_id_, - *tx_desc, - *param.dml_base_param_, - param.column_ids_, - update_column_ids, - &upd_iter, - affected_rows))) { - LOG_WARN("failed to insert row.", K(ret)); - } - } - } - return ret; -} - - int ObPersistentLobApator::build_lob_piece_table_dml( ObLobAccessParam& param, const uint64_t tenant_id, @@ -943,8 +656,7 @@ int ObPersistentLobApator::get_lob_tablet_schema( int ObPersistentLobApator::build_common_scan_param( const ObLobAccessParam ¶m, - const uint64_t table_id, - bool is_get, + const bool is_get, uint32_t col_num, ObTableScanParam& scan_param) { @@ -963,10 +675,11 @@ int ObPersistentLobApator::build_common_scan_param( false // read_latest ); query_flag.disable_cache(); + if (param.enable_block_cache()) query_flag.set_use_block_cache(); query_flag.scan_order_ = param.scan_backward_ ? ObQueryFlag::Reverse : ObQueryFlag::Forward; scan_param.scan_flag_.flag_ = query_flag.flag_; // set column ids - scan_param.column_ids_.reset(); + scan_param.column_ids_.reuse(); for (uint32_t i = 0; OB_SUCC(ret) && i < col_num; i++) { if (OB_FAIL(scan_param.column_ids_.push_back(OB_APP_MIN_COLUMN_ID + i))) { LOG_WARN("push col id failed.", K(ret), K(i)); @@ -976,7 +689,7 @@ int ObPersistentLobApator::build_common_scan_param( if (OB_SUCC(ret)) { scan_param.reserved_cell_count_ = scan_param.column_ids_.count(); // table param - scan_param.index_id_ = table_id; // table id + scan_param.index_id_ = 0; // table id scan_param.is_get_ = is_get; // set timeout scan_param.timeout_ = param.timeout_; @@ -990,10 +703,11 @@ int ObPersistentLobApator::build_common_scan_param( } scan_param.sql_mode_ = param.sql_mode_; // common set - scan_param.allocator_ = param.allocator_; + ObIAllocator *allocator = param.access_ctx_ != nullptr ? &(param.access_ctx_->reader_cache_.get_allocator()) : param.allocator_; + scan_param.allocator_ = allocator; scan_param.for_update_ = false; scan_param.for_update_wait_timeout_ = scan_param.timeout_; - scan_param.scan_allocator_ = param.allocator_; + scan_param.scan_allocator_ = allocator; scan_param.frozen_version_ = -1; scan_param.force_refresh_lc_ = false; scan_param.output_exprs_ = nullptr; @@ -1158,21 +872,279 @@ int ObPersistentLobApator::set_lob_piece_row( return ret; } -int ObLobUpdIterator::get_next_row(ObNewRow *&row) +int ObPersistentLobApator::scan_lob_meta_with_ctx( + ObLobAccessParam ¶m, + ObTableScanParam &scan_param, + ObNewRowIterator *&meta_iter) { int ret = OB_SUCCESS; - if (OB_ISNULL(old_row_) || OB_ISNULL(new_row_)) { - ret = OB_ERR_NULL_VALUE; - LOG_WARN("get next row failed, null val.", K(ret), K(old_row_), K(new_row_)); - } else if (is_iter_end_) { - ret = OB_ITER_END; - } else if (!got_old_row_) { - row = old_row_; - got_old_row_ = true; + bool is_hit = false; + ObPersistLobReaderCacheKey key; + ObPersistLobReader *reader = nullptr; + ObPersistLobReaderCache &cache = param.access_ctx_->reader_cache_; + key.ls_id_ = param.ls_id_; + key.snapshot_ = param.snapshot_.core_.version_.get_val_for_tx(); + key.tablet_id_ = param.tablet_id_; + key.is_get_ = param.has_single_chunk(); + if (OB_FAIL(cache.get(key, reader))) { + LOG_WARN("get read from cache fail", K(ret), K(key)); + } else if (nullptr != reader) { // use cache + is_hit = true; + if (OB_FAIL(reader->rescan(param, meta_iter))) { + LOG_WARN("rescan reader fail", K(ret), K(key)); + } + } else if (OB_ISNULL(reader = cache.alloc_reader())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc_reader fail", K(ret)); + } else if (OB_FAIL(prepare_lob_tablet_id(param))) { + LOG_WARN("prepare_lob_tablet_id fail", K(ret)); + } else if (OB_FAIL(reader->open(this, param, meta_iter))) { + LOG_WARN("open reader fail", K(ret), K(key)); + } else if (OB_FAIL(cache.put(key, reader))) { + LOG_WARN("put reader to cache fail", K(ret), K(key)); + } + + if (OB_FAIL(ret)) { + if (! is_hit && OB_NOT_NULL(reader)) { + reader->~ObPersistLobReader(); + } + } + return ret; +} + +int ObPersistentLobApator::do_scan_lob_meta( + ObLobAccessParam ¶m, + ObTableScanParam &scan_param, + common::ObNewRowIterator *&meta_iter) +{ + int ret = OB_SUCCESS; + ObTabletHandle data_tablet; + ObTabletHandle lob_meta_tablet; + ObTabletHandle lob_piece_tablet; + if (OB_FAIL(get_lob_tablets(param, data_tablet, lob_meta_tablet, lob_piece_tablet))) { + LOG_WARN("failed to get tablets.", K(ret), K(param)); } else { - row = new_row_; - got_old_row_ = false; - is_iter_end_ = true; + ObAccessService *oas = MTL(ObAccessService*); + scan_param.tablet_id_ = param.lob_meta_tablet_id_; + scan_param.schema_version_ = lob_meta_tablet.get_obj()->get_tablet_meta().max_sync_storage_schema_version_; + if (OB_ISNULL(oas)) { + ret = OB_ERR_INTERVAL_INVALID; + LOG_WARN("get access service failed.", K(ret)); + } else if (OB_FAIL(build_common_scan_param(param, param.has_single_chunk(), ObLobMetaUtil::LOB_META_COLUMN_CNT, scan_param))) { + LOG_WARN("build common scan param failed.", K(ret), K(param)); + } else if (OB_FAIL(prepare_table_param(param, scan_param, true/*is_meta*/))) { + LOG_WARN("prepare lob meta table param failed.", K(ret), K(param)); + } else if (OB_FAIL(oas->table_scan(scan_param, meta_iter))) { + LOG_WARN("do table scan falied.", K(ret), K(scan_param), K(param)); + } + } + return ret; +} + +int ObPersistentLobApator::scan_lob_meta( + ObLobAccessParam ¶m, + ObTableScanParam &scan_param, + common::ObNewRowIterator *&meta_iter) +{ + int ret = OB_SUCCESS; + ObNewRange range; + ObObj *rowkey_objs = nullptr; + if (nullptr != param.access_ctx_) { + if (OB_FAIL(scan_lob_meta_with_ctx(param, scan_param, meta_iter))) { + LOG_WARN("scan_lob_meta_with_ctx fail", K(ret)); + } + } else if (OB_FAIL(prepare_lob_tablet_id(param))) { + LOG_WARN("prepare_lob_tablet_id fail", K(ret)); + } else if (OB_ISNULL(rowkey_objs = reinterpret_cast(param.allocator_->alloc(sizeof(ObObj) * 4)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc range obj failed.", K(ret), "size", (sizeof(ObObj) * 4)); + } else if (OB_FAIL(param.get_rowkey_range(rowkey_objs, range))) { + LOG_WARN("get_rowkey_range fail", K(ret)); + } else if (OB_FAIL(scan_param.key_ranges_.push_back(range))) { + LOG_WARN("failed to push key range.", K(ret), K(scan_param), K(range)); + } else if (OB_FAIL(do_scan_lob_meta(param, scan_param, meta_iter))) { + LOG_WARN("do_scan_lob_meta fail", K(ret)); + } + return ret; +} + +int ObPersistentLobApator::prepare_lob_tablet_id(ObLobAccessParam& param) +{ + int ret = OB_SUCCESS; + ObTabletHandle data_tablet; + ObTabletBindingMdsUserData ddl_data; + if (param.lob_meta_tablet_id_.is_valid() && param.lob_piece_tablet_id_.is_valid()) { + } else if (OB_FAIL(inner_get_tablet(param, param.tablet_id_, data_tablet))) { + LOG_WARN("failed to get data tablet", K(ret), K(param.ls_id_), K(param.tablet_id_)); + } else if (OB_FAIL(data_tablet.get_obj()->ObITabletMdsInterface::get_ddl_data(share::SCN::max_scn(), ddl_data))) { + LOG_WARN("failed to get ddl data from tablet", K(ret), K(data_tablet)); + } else if (OB_UNLIKELY(check_lob_tablet_id(param.tablet_id_, ddl_data.lob_meta_tablet_id_, ddl_data.lob_piece_tablet_id_))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("failed to check lob_tablet_id", K(ret), K(param), K(ddl_data.lob_meta_tablet_id_), K(ddl_data.lob_piece_tablet_id_)); + } else { + param.lob_meta_tablet_id_ = ddl_data.lob_meta_tablet_id_; + param.lob_piece_tablet_id_ = ddl_data.lob_piece_tablet_id_; + } + return ret; +} + +int ObPersistentLobApator::set_dml_seq_no(ObLobAccessParam ¶m) +{ + int ret = OB_SUCCESS; + if (param.seq_no_st_.is_valid()) { + if (param.used_seq_cnt_ < param.total_seq_cnt_) { + param.dml_base_param_->spec_seq_no_ = param.seq_no_st_ + param.used_seq_cnt_; + // param.used_seq_cnt_++; + LOG_DEBUG("dml lob meta with seq no", K(param.dml_base_param_->spec_seq_no_)); + } else { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("failed to get seq no from param.", K(ret), K(param)); + } + } else { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid seq no from param.", K(ret), K(param)); + } + return ret; +} + +int ObPersistentLobApator::erase_lob_meta(ObLobAccessParam ¶m, ObNewRowIterator& row_iter) +{ + int ret = OB_SUCCESS; + int64_t affected_rows = 0; + ObAccessService *oas = MTL(ObAccessService*); + if (OB_ISNULL(oas)) { + ret = OB_ERR_INTERVAL_INVALID; + LOG_WARN("get access service failed.", K(ret), KP(oas)); + } else if (OB_ISNULL(param.tx_desc_)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("get tx desc null.", K(ret), K(param)); + } else if (OB_FAIL(prepare_lob_tablet_id(param))) { + LOG_WARN("failed to get tablets.", K(ret), K(param)); + } else if (OB_FAIL(prepare_lob_meta_dml(param))) { + LOG_WARN("failed to prepare lob meta dml", K(ret)); + } else if (OB_FAIL(oas->delete_rows( + param.ls_id_, + param.lob_meta_tablet_id_, + *param.tx_desc_, + *param.dml_base_param_, + param.column_ids_, + &row_iter, + affected_rows))) { + LOG_WARN("delete_rows fail", K(ret)); + } + return ret; +} + +int ObPersistentLobApator::write_lob_meta(ObLobAccessParam& param, ObNewRowIterator& row_iter) +{ + int ret = OB_SUCCESS; + int64_t affected_rows = 0; + ObAccessService *oas = MTL(ObAccessService*); + if (OB_ISNULL(oas)) { + ret = OB_ERR_INTERVAL_INVALID; + LOG_WARN("get access service failed.", K(ret), KP(oas)); + } else if (OB_ISNULL(param.tx_desc_)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("get tx desc null.", K(ret), K(param)); + } else if (OB_FAIL(prepare_lob_tablet_id(param))) { + LOG_WARN("failed to get tablets.", K(ret), K(param)); + } else if (OB_FAIL(prepare_lob_meta_dml(param))) { + LOG_WARN("failed to prepare lob meta dml", K(ret)); + } else if (OB_FAIL(oas->insert_rows( + param.ls_id_, + param.lob_meta_tablet_id_, + *param.tx_desc_, + *param.dml_base_param_, + param.column_ids_, + &row_iter, + affected_rows))) { + LOG_WARN("insert_rows fail", K(ret)); + } + return ret; +} + +int ObPersistentLobApator::update_lob_meta(ObLobAccessParam& param, ObNewRowIterator &row_iter) +{ + int ret = OB_SUCCESS; + int64_t affected_rows = 0; + ObAccessService *oas = MTL(ObAccessService*); + if (OB_ISNULL(oas)) { + ret = OB_ERR_INTERVAL_INVALID; + LOG_WARN("get access service failed.", K(ret), KP(oas)); + } else if (OB_ISNULL(param.tx_desc_)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("get tx desc null.", K(ret), K(param)); + } else if (OB_FAIL(prepare_lob_tablet_id(param))) { + LOG_WARN("failed to get tablets.", K(ret), K(param)); + } else if (OB_FAIL(prepare_lob_meta_dml(param))) { + LOG_WARN("failed to prepare lob meta dml", K(ret)); + } else { + ObSEArray update_column_ids; + for (int i = 2; OB_SUCC(ret) && i < ObLobMetaUtil::LOB_META_COLUMN_CNT; ++i) { + if (OB_FAIL(update_column_ids.push_back(OB_APP_MIN_COLUMN_ID + i))) { + LOG_WARN("push column ids failed.", K(ret), K(i)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(oas->update_rows( + param.ls_id_, + param.lob_meta_tablet_id_, + *param.tx_desc_, + *param.dml_base_param_, + param.column_ids_, + update_column_ids, + &row_iter, + affected_rows))) { + LOG_WARN("update_rows fail", K(ret)); + } + } + return ret; +} + +int ObPersistentLobApator::write_lob_meta(ObLobAccessParam ¶m, ObLobMetaInfo& row_info) +{ + int ret = OB_SUCCESS; + ObObj cell[ObLobMetaUtil::LOB_META_COLUMN_CNT]; + ObNewRow new_row; + ObLobPersistInsertSingleRowIter single_iter; + set_lob_meta_row(cell, new_row, row_info); + if (OB_FAIL(single_iter.init(¶m, &new_row))) { + LOG_WARN("single_iter init fail", K(ret)); + } else if (OB_FAIL(write_lob_meta(param, single_iter))) { + LOG_WARN("write_lob_meta fail", K(ret)); + } + return ret; +} + +int ObPersistentLobApator::erase_lob_meta(ObLobAccessParam ¶m, ObLobMetaInfo& row_info) +{ + int ret = OB_SUCCESS; + ObObj cell[ObLobMetaUtil::LOB_META_COLUMN_CNT]; + ObNewRow new_row; + ObLobPersistDeleteSingleRowIter single_iter; + set_lob_meta_row(cell, new_row, row_info); + if (OB_FAIL(single_iter.init(¶m, &new_row))) { + LOG_WARN("single_iter init fail", K(ret)); + } else if (OB_FAIL(erase_lob_meta(param, single_iter))) { + LOG_WARN("erase_lob_meta fail", K(ret)); + } + return ret; +} + +int ObPersistentLobApator::update_lob_meta(ObLobAccessParam& param, ObLobMetaInfo& old_row, ObLobMetaInfo& new_row) +{ + int ret = OB_SUCCESS; + ObObj new_row_cell[ObLobMetaUtil::LOB_META_COLUMN_CNT]; + ObNewRow new_tbl_row; + set_lob_meta_row(new_row_cell, new_tbl_row, new_row); + ObObj old_row_cell[ObLobMetaUtil::LOB_META_COLUMN_CNT]; + ObNewRow old_tbl_row; + set_lob_meta_row(old_row_cell, old_tbl_row, old_row); + ObLobPersistUpdateSingleRowIter upd_iter; + if (OB_FAIL(upd_iter.init(¶m, &old_tbl_row, &new_tbl_row))) { + LOG_WARN("upd_iter init fail", K(ret)); + } else if (OB_FAIL(update_lob_meta(param, upd_iter))) { + LOG_WARN("update_lob_meta fail", K(ret)); } return ret; } @@ -1193,7 +1165,7 @@ int ObPersistentLobApator::prepare_single_get( scan_param.tablet_id_ = lob_meta_tablet.get_obj()->get_tablet_meta().tablet_id_; scan_param.schema_version_ = lob_meta_tablet.get_obj()->get_tablet_meta().max_sync_storage_schema_version_; scan_param.table_param_ = param.meta_tablet_param_; - if (OB_FAIL(build_common_scan_param(param, table_id, true, ObLobMetaUtil::LOB_META_COLUMN_CNT, scan_param))) { + if (OB_FAIL(build_common_scan_param(param, true, ObLobMetaUtil::LOB_META_COLUMN_CNT, scan_param))) { LOG_WARN("build common scan param failed.", K(ret)); } else if (OB_FAIL(prepare_table_param(param, scan_param, true))) { LOG_WARN("prepare lob meta table param failed.", K(ret)); diff --git a/src/storage/lob/ob_lob_persistent_adaptor.h b/src/storage/lob/ob_lob_persistent_adaptor.h index 2550320224..fb41e92bbf 100644 --- a/src/storage/lob/ob_lob_persistent_adaptor.h +++ b/src/storage/lob/ob_lob_persistent_adaptor.h @@ -65,6 +65,16 @@ public: virtual int scan_lob_meta(ObLobAccessParam ¶m, ObTableScanParam &scan_param, common::ObNewRowIterator *&meta_iter) override; + + int scan_lob_meta_with_ctx( + ObLobAccessParam ¶m, + ObTableScanParam &scan_param, + common::ObNewRowIterator *&meta_iter); + int do_scan_lob_meta( + ObLobAccessParam ¶m, + ObTableScanParam &scan_param, + common::ObNewRowIterator *&meta_iter); + virtual int get_lob_data(ObLobAccessParam ¶m, uint64_t piece_id, ObLobPieceInfo& info) override; @@ -82,6 +92,22 @@ public: int update_lob_piece_tablet(ObLobAccessParam& param, ObLobPieceInfo& in_row); // update lob meta tablet item virtual int update_lob_meta(ObLobAccessParam& param, ObLobMetaInfo& old_row, ObLobMetaInfo& new_row) override; + + static void set_lob_meta_row( + ObObj* cell, + ObNewRow& new_row, + ObLobMetaInfo& in_row); + + int write_lob_meta(ObLobAccessParam ¶m, ObNewRowIterator& row_iter); + int erase_lob_meta(ObLobAccessParam ¶m, ObNewRowIterator& row_iter); + int update_lob_meta(ObLobAccessParam& param, ObNewRowIterator &row_iter); + + int build_common_scan_param( + const ObLobAccessParam ¶m, + const bool is_get, + uint32_t col_num, + ObTableScanParam& scan_param); + private: // get schema from schema service int get_lob_tablet_schema( @@ -104,12 +130,6 @@ private: const ObLobAccessParam ¶m, ObTableScanParam &scan_param, bool is_meta); - int build_common_scan_param( - const ObLobAccessParam ¶m, - const uint64_t table_id, - bool is_get, - uint32_t col_num, - ObTableScanParam& scan_param); int inner_get_tablet( const ObLobAccessParam ¶m, const common::ObTabletID &tablet_id, @@ -128,26 +148,13 @@ private: const ObTabletHandle& data_tablet, const ObTabletHandle& lob_piece_tablet); - int prepare_lob_meta_dml( - ObLobAccessParam& param, - const uint64_t tenant_id, - const ObTabletHandle& data_tablet, - const ObTabletHandle& lob_meta_tablet); + int prepare_lob_meta_dml(ObLobAccessParam& param); int build_lob_meta_table_dml( ObLobAccessParam& param, - const uint64_t tenant_id, - ObTableDMLParam* dml_param, ObDMLBaseParam& dml_base_param, ObStoreCtxGuard *store_ctx_guard, - ObSEArray& column_ids, - const ObTabletHandle& data_tablet, - const ObTabletHandle& lob_meta_tablet); - - void set_lob_meta_row( - ObObj* cell, - ObNewRow& new_row, - ObLobMetaInfo& in_row); + ObSEArray& column_ids); int set_lob_piece_row( char* buf, @@ -162,6 +169,8 @@ private: int prepare_piece_table_param( const ObLobAccessParam ¶m, ObTableScanParam &scan_param); + int prepare_lob_tablet_id(ObLobAccessParam& param); + int set_dml_seq_no(ObLobAccessParam ¶m); int prepare_single_get( ObLobAccessParam ¶m, diff --git a/src/storage/lob/ob_lob_persistent_iterator.cpp b/src/storage/lob/ob_lob_persistent_iterator.cpp new file mode 100644 index 0000000000..03fcf147d4 --- /dev/null +++ b/src/storage/lob/ob_lob_persistent_iterator.cpp @@ -0,0 +1,232 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "ob_lob_persistent_iterator.h" + +namespace oceanbase +{ +namespace storage +{ + + +int ObLobPersistWriteIter::update_seq_no() +{ + int ret = OB_SUCCESS; + if (param_->seq_no_st_.is_valid()) { + if (param_->used_seq_cnt_ < param_->total_seq_cnt_) { + param_->dml_base_param_->spec_seq_no_ = param_->seq_no_st_ + param_->used_seq_cnt_; + param_->used_seq_cnt_++; + LOG_DEBUG("dml lob meta with seq no", K(param_->dml_base_param_->spec_seq_no_)); + } else { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("failed to get seq no from param.", K(ret), KPC(param_)); + } + } else { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid seq no from param.", K(ret), KPC(param_)); + } + return ret; +} + +int ObLobPersistWriteIter::inc_lob_size(ObLobMetaInfo &info) +{ + int ret = OB_SUCCESS; + param_->lob_data_->byte_size_ += info.byte_len_; + param_->byte_size_ = param_->lob_data_->byte_size_; + if (ObLobManager::lob_handle_has_char_len(*param_)) { + int64_t *len = ObLobManager::get_char_len_ptr(*param_); + *len = *len + info.char_len_; + OB_ASSERT(*len >= 0); + } + return ret; +} + +int ObLobPersistWriteIter::dec_lob_size(ObLobMetaInfo &info) +{ + int ret = OB_SUCCESS; + param_->lob_data_->byte_size_ -= info.byte_len_; + if (ObLobManager::lob_handle_has_char_len(*param_)) { + int64_t *len = ObLobManager::get_char_len_ptr(*param_); + *len = *len - info.char_len_; + OB_ASSERT(*len >= 0); + } + param_->byte_size_ = param_->lob_data_->byte_size_; + return ret; +} + +int ObLobPersistUpdateSingleRowIter::init(ObLobAccessParam *param, ObNewRow *old_row, ObNewRow *new_row) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(param) || OB_ISNULL(old_row) || OB_ISNULL(new_row)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("param or row is null", K(ret), KP(param), KP(old_row), KP(new_row)); + } else { + param_ = param; + old_row_ = old_row; + new_row_ = new_row; + } + return ret; +} + +int ObLobPersistUpdateSingleRowIter::get_next_row(ObNewRow *&row) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(old_row_) || OB_ISNULL(new_row_)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("get next row failed, null val.", K(ret), K(old_row_), K(new_row_)); + } else if (is_iter_end_) { + ret = OB_ITER_END; + } else if (!got_old_row_) { + row = old_row_; + got_old_row_ = true; + } else if (OB_FAIL(update_seq_no())) { + LOG_WARN("update_seq_no fail", K(ret)); + } else { + row = new_row_; + got_old_row_ = false; + is_iter_end_ = true; + } + return ret; +} + +int ObLobPersistInsertSingleRowIter::init(ObLobAccessParam *param, ObNewRow *row) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(param) || OB_ISNULL(row)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("param or row is null", K(ret), KP(param), KP(row)); + } else { + param_ = param; + row_ = row; + } + return ret; +} + +int ObLobPersistInsertSingleRowIter::get_next_row(ObNewRow *&row) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(param_) || OB_ISNULL(row_)) { + ret = OB_NOT_INIT; + LOG_WARN("param or row is null", K(ret), KP(param_), KP(row_)); + } else if (iter_end_) { + ret = OB_ITER_END; + } else if (OB_FAIL(update_seq_no())) { + LOG_WARN("update_seq_no fail", K(ret)); + } else { + row = row_; + iter_end_ = true; + } + return ret; +} + +int ObLobPersistDeleteSingleRowIter::init(ObLobAccessParam *param, ObNewRow *row) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(param) || OB_ISNULL(row)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("param or row is null", K(ret), KP(param), KP(row)); + } else { + param_ = param; + row_ = row; + } + return ret; +} + +int ObLobPersistDeleteSingleRowIter::get_next_row(ObNewRow *&row) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(param_) || OB_ISNULL(row_)) { + ret = OB_NOT_INIT; + LOG_WARN("param or row is null", K(ret), KP(param_), KP(row_)); + } else if (iter_end_) { + ret = OB_ITER_END; + } else if (OB_FAIL(update_seq_no())) { + LOG_WARN("update_seq_no fail", K(ret)); + } else { + row = row_; + iter_end_ = true; + } + return ret; +} + +int ObLobPersistInsertIter::init(ObLobAccessParam *param, ObLobMetaWriteIter *meta_iter) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(param) || OB_ISNULL(meta_iter)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("param or meta_iter is null", K(ret), KP(param), KP(meta_iter)); + } else { + param_ = param; + meta_iter_ = meta_iter; + } + return ret; +} + +int ObLobPersistInsertIter::get_next_row(ObNewRow *&row) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(meta_iter_->get_next_row(result_))) { + if (ret != OB_ITER_END) { + LOG_WARN("get next meta info failed.", K(ret)); + } + } else if (OB_FALSE_IT(result_.info_.lob_data_.assign_ptr(result_.data_.ptr(), result_.data_.length()))) { + } else if (OB_FAIL(update_seq_no())) { + LOG_WARN("update_seq_no fail", K(ret)); + } else if (OB_FAIL(ObLobManager::update_out_ctx(*param_, nullptr, result_.info_))) { // new row + LOG_WARN("failed update checksum.", K(ret)); + } else if (OB_FAIL(inc_lob_size(result_.info_))) { + LOG_WARN("inc_lob_size fail", K(ret)); + } else { + ObPersistentLobApator::set_lob_meta_row(row_cell_, new_row_, result_.info_); + row = &new_row_; + } + return ret; +} + +int ObLobPersistDeleteIter::init(ObLobAccessParam *param, ObLobMetaScanIter *meta_iter) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(param) || OB_ISNULL(meta_iter)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("param or meta_iter is null", K(ret), KP(param), KP(meta_iter)); + } else { + param_ = param; + meta_iter_ = meta_iter; + } + return ret; +} + +int ObLobPersistDeleteIter::get_next_row(ObNewRow *&row) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(meta_iter_->get_next_row(result_))) { + if (ret != OB_ITER_END) { + LOG_WARN("get next meta info failed.", K(ret)); + } + } else if (OB_FAIL(update_seq_no())) { + LOG_WARN("update_seq_no fail", K(ret)); + } else if (OB_FAIL(ObLobManager::update_out_ctx(*param_, nullptr, result_.info_))) { // new row + LOG_WARN("failed update checksum.", K(ret)); + } else if (OB_FAIL(dec_lob_size(result_.info_))) { + LOG_WARN("dec_lob_size fail", K(ret)); + } else { + ObPersistentLobApator::set_lob_meta_row(row_cell_, new_row_, result_.info_); + row = &new_row_; + } + return ret; +} + + +} // storage +} // oceanbase diff --git a/src/storage/lob/ob_lob_persistent_iterator.h b/src/storage/lob/ob_lob_persistent_iterator.h new file mode 100644 index 0000000000..d3be04c164 --- /dev/null +++ b/src/storage/lob/ob_lob_persistent_iterator.h @@ -0,0 +1,160 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEABASE_STORAGE_OB_LOB_PERSISTENT_ITERATOR_ +#define OCEABASE_STORAGE_OB_LOB_PERSISTENT_ITERATOR_ + +namespace oceanbase +{ +namespace storage +{ + +class ObLobPersistWriteIter : public ObNewRowIterator +{ +public: + ObLobPersistWriteIter(): param_(nullptr) {} + virtual ~ObLobPersistWriteIter() {} + + virtual int get_next_row() override { return OB_NOT_IMPLEMENT; } + +protected: + int update_seq_no(); + int dec_lob_size(ObLobMetaInfo &info); + int inc_lob_size(ObLobMetaInfo &info); + +protected: + ObLobAccessParam *param_; + +}; + + +class ObLobPersistInsertSingleRowIter: public ObLobPersistWriteIter +{ +public: + ObLobPersistInsertSingleRowIter(): + row_(nullptr), + iter_end_(false) + {} + int init(ObLobAccessParam *param, ObNewRow *row); + + virtual ~ObLobPersistInsertSingleRowIter() {} + virtual int get_next_row(ObNewRow *&row); + virtual void reset() { iter_end_ = false; } + +private: + // disallow copy + DISALLOW_COPY_AND_ASSIGN(ObLobPersistInsertSingleRowIter); +private: + // data members + ObNewRow *row_; + bool iter_end_; +}; + + +class ObLobPersistDeleteSingleRowIter: public ObLobPersistWriteIter +{ +public: + ObLobPersistDeleteSingleRowIter(): + row_(nullptr), + iter_end_(false) + {} + int init(ObLobAccessParam *param, ObNewRow *row); + + virtual ~ObLobPersistDeleteSingleRowIter() {} + virtual int get_next_row(ObNewRow *&row); + virtual void reset() { iter_end_ = false; } + +private: + // disallow copy + DISALLOW_COPY_AND_ASSIGN(ObLobPersistDeleteSingleRowIter); +private: + // data members + ObNewRow *row_; + bool iter_end_; +}; + +class ObLobPersistUpdateSingleRowIter : public ObLobPersistWriteIter +{ +public: + ObLobPersistUpdateSingleRowIter() + : old_row_(nullptr), + new_row_(nullptr), + got_old_row_(false), + is_iter_end_(false) + {} + + virtual ~ObLobPersistUpdateSingleRowIter() {} + + int init(ObLobAccessParam *param, ObNewRow *old_row, ObNewRow *new_row); + + virtual int get_next_row(ObNewRow *&row) override; + virtual void reset() override {} + +private: + // disallow copy + DISALLOW_COPY_AND_ASSIGN(ObLobPersistUpdateSingleRowIter); + +private: + ObNewRow *old_row_; + ObNewRow *new_row_; + bool got_old_row_; + bool is_iter_end_; +}; + + +class ObLobPersistInsertIter: public ObLobPersistWriteIter +{ +public: + ObLobPersistInsertIter() : meta_iter_(nullptr), new_row_(), row_cell_(), result_() {} + int init(ObLobAccessParam *param, ObLobMetaWriteIter *meta_iter); + virtual ~ObLobPersistInsertIter() {} + virtual int get_next_row(ObNewRow *&row); + virtual void reset() { new_row_.reset(); } + +private: + // disallow copy + DISALLOW_COPY_AND_ASSIGN(ObLobPersistInsertIter); +private: + // data members + ObLobMetaWriteIter *meta_iter_; + ObNewRow new_row_; + ObObj row_cell_[ObLobMetaUtil::LOB_META_COLUMN_CNT]; + ObLobMetaWriteResult result_; +}; + + +class ObLobPersistDeleteIter: public ObLobPersistWriteIter +{ +public: + ObLobPersistDeleteIter() : meta_iter_(nullptr), new_row_(), row_cell_(), result_() {} + int init(ObLobAccessParam *param, ObLobMetaScanIter *meta_iter); + virtual ~ObLobPersistDeleteIter() {} + virtual int get_next_row(ObNewRow *&row); + virtual void reset() { new_row_.reset(); } + + +private: + // disallow copy + DISALLOW_COPY_AND_ASSIGN(ObLobPersistDeleteIter); +private: + // data members + ObLobMetaScanIter *meta_iter_; + ObNewRow new_row_; + ObObj row_cell_[ObLobMetaUtil::LOB_META_COLUMN_CNT]; + ObLobMetaScanResult result_; +}; + + +} // storage +} // oceanbase + +#endif \ No newline at end of file diff --git a/src/storage/lob/ob_lob_persistent_reader.cpp b/src/storage/lob/ob_lob_persistent_reader.cpp new file mode 100644 index 0000000000..2cc6564482 --- /dev/null +++ b/src/storage/lob/ob_lob_persistent_reader.cpp @@ -0,0 +1,168 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "ob_lob_persistent_reader.h" + +namespace oceanbase +{ +namespace storage +{ + +ObPersistLobReaderCache::~ObPersistLobReaderCache() +{ + int ret = OB_SUCCESS; + DLIST_FOREACH(curr, list_) { + curr->reader_->~ObPersistLobReader(); + curr->reader_ = nullptr; + } + list_.clear(); +} + +int ObPersistLobReaderCache::get(ObPersistLobReaderCacheKey key, ObPersistLobReader *&reader) +{ + int ret = OB_SUCCESS; + DLIST_FOREACH_X(curr, list_, OB_SUCC(ret) && nullptr == reader) { + if (OB_ISNULL(curr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("curr is null", K(ret)); + } else if (! (curr->key_ == key)) { // next + } else if (false == list_.move_to_last(curr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("move_to_last fail", K(ret), K(key)); + } else if (OB_ISNULL(reader = curr->reader_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("reader is null", K(ret), K(key)); + } + } + return ret; +} + +int ObPersistLobReaderCache::put(ObPersistLobReaderCacheKey key, ObPersistLobReader *reader) +{ + int ret = OB_SUCCESS; + ObPersistLobReaderCacheNode *node = nullptr; + if (OB_ISNULL(reader)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("reader is null", K(ret), K(key)); + } else if (OB_ISNULL(node = OB_NEWx(ObPersistLobReaderCacheNode, &allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc fail", K(ret)); + } else { + node->key_ = key; + node->reader_ = reader; + if (list_.get_size() >= cap_ && OB_FAIL(remove_first())) { + LOG_WARN("remove_first fail", K(ret), K(list_), K(cap_)); + } else if (false == list_.add_last(node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("add_last fail", K(ret), K(key)); + } + } + + if (OB_FAIL(ret)) { + if (OB_NOT_NULL(node)) { + allocator_.free(node); + } + } + return ret; +} + +int ObPersistLobReaderCache::remove_first() +{ + int ret = OB_SUCCESS; + ObPersistLobReaderCacheNode *node = list_.remove_first(); + if (OB_ISNULL(node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("node is null", K(ret), K(list_)); + } else { + node->reader_->~ObPersistLobReader(); + allocator_.free(node->reader_); + node->reader_ = nullptr; + allocator_.free(node); + } + return ret; +} + +ObPersistLobReader::~ObPersistLobReader() +{ + if (OB_NOT_NULL(adaptor_)) { + if (OB_NOT_NULL(row_iter_)) { + adaptor_->revert_scan_iter(row_iter_); + row_iter_ = nullptr; + } + } +} + +ObPersistLobReader* ObPersistLobReaderCache::alloc_reader() +{ + return OB_NEWx(ObPersistLobReader, &allocator_); +} + +int ObPersistLobReader::open(ObPersistentLobApator* adaptor, ObLobAccessParam ¶m, ObNewRowIterator *&meta_iter) +{ + int ret = OB_SUCCESS; + // scan may be fail, must be sure adaptor not null + // then deconstrcutor can release correctly + adaptor_ = adaptor; + + ObNewRange range; + if (OB_ISNULL(adaptor)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("adaptor is null", KR(ret), K(param)); + } else if (! param.lob_meta_tablet_id_.is_valid() || ! param.lob_piece_tablet_id_.is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("tablet_id invalid", KR(ret), K(param)); + } else if (OB_FAIL(param.get_rowkey_range(rowkey_objs_, range))) { + LOG_WARN("get_rowkey_range fail", K(ret)); + } else if (OB_FAIL(scan_param_.key_ranges_.push_back(range))) { + LOG_WARN("failed to push key range.", K(ret), K(scan_param_), K(range)); + } else if (OB_FAIL(adaptor->do_scan_lob_meta(param, scan_param_, row_iter_))) { + LOG_WARN("do_scan_lob_meta fail", K(ret)); + } else { + meta_iter = row_iter_; + lob_meta_tablet_id_ = param.lob_meta_tablet_id_; + lob_piece_tablet_id_ = param.lob_piece_tablet_id_; + } + return ret; +} + +int ObPersistLobReader::rescan(ObLobAccessParam ¶m, ObNewRowIterator *&meta_iter) +{ + int ret = OB_SUCCESS; + ObNewRange range; + ObAccessService *oas = MTL(ObAccessService*); + + param.lob_meta_tablet_id_ = lob_meta_tablet_id_; + param.lob_piece_tablet_id_ = lob_piece_tablet_id_; + + if (OB_FAIL(oas->reuse_scan_iter(false/*tablet id same*/, row_iter_))) { + LOG_WARN("fail to reuse scan iter", K(ret)); + } else if (OB_FAIL(param.get_rowkey_range(rowkey_objs_, range))) { + LOG_WARN("get_rowkey_range fail", K(ret)); + } else if (OB_FAIL(adaptor_->build_common_scan_param(param, param.has_single_chunk(), ObLobMetaUtil::LOB_META_COLUMN_CNT, scan_param_))) { + LOG_WARN("build common scan param failed.", K(ret)); + } else { + scan_param_.key_ranges_.reset(); + if (OB_FAIL(scan_param_.key_ranges_.push_back(range))) { + LOG_WARN("failed to push key range.", K(ret), K(scan_param_), K(range)); + } else if (OB_FAIL(oas->table_rescan(scan_param_, row_iter_))) { + LOG_WARN("fail to do table rescan", K(ret), K(scan_param_)); + } else { + meta_iter = row_iter_; + } + } + return ret; +} + +} // storage +} // oceanbase \ No newline at end of file diff --git a/src/storage/lob/ob_lob_persistent_reader.h b/src/storage/lob/ob_lob_persistent_reader.h new file mode 100644 index 0000000000..5232aa6c01 --- /dev/null +++ b/src/storage/lob/ob_lob_persistent_reader.h @@ -0,0 +1,118 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEABASE_STORAGE_OB_LOB_PERSISTENT_READER_ +#define OCEABASE_STORAGE_OB_LOB_PERSISTENT_READER_ + +namespace oceanbase +{ +namespace storage +{ + + +class ObPersistLobReader +{ +public: + ObPersistLobReader(): + adaptor_(nullptr), + row_iter_(nullptr), + scan_param_(), + lob_meta_tablet_id_(), + lob_piece_tablet_id_() + {} + ~ObPersistLobReader(); + int rescan(ObLobAccessParam ¶m, ObNewRowIterator *&meta_iter); + int open(ObPersistentLobApator* adpter, ObLobAccessParam ¶m, ObNewRowIterator *&meta_iter); + +private: + ObPersistentLobApator *adaptor_; + ObNewRowIterator *row_iter_; + ObObj rowkey_objs_[4]; + ObTableScanParam scan_param_; + common::ObTabletID lob_meta_tablet_id_; + common::ObTabletID lob_piece_tablet_id_; + +}; + +struct ObPersistLobReaderCacheKey +{ + ObPersistLobReaderCacheKey(): + ls_id_(), + tablet_id_(), + snapshot_(0), + is_get_(false) + {} + + share::ObLSID ls_id_; + common::ObTabletID tablet_id_; + int64_t snapshot_; + bool is_get_; + bool operator==(const ObPersistLobReaderCacheKey &other) const + { + return snapshot_ == other.snapshot_ && tablet_id_ == other.tablet_id_ && ls_id_ == other.ls_id_ && is_get_ == other.is_get_; + } + + TO_STRING_KV(K(ls_id_), K(tablet_id_), K(snapshot_)); +}; + +struct ObPersistLobReaderCacheNode : public ObDLinkBase +{ + ObPersistLobReaderCacheNode(): + key_(), + reader_(nullptr) + {} + + ObPersistLobReaderCacheKey key_; + ObPersistLobReader *reader_; +}; + +class ObPersistLobReaderCache +{ +public: + static const int DEFAULT_CAP = 10; + +public: + ObPersistLobReaderCache(int32_t cap = DEFAULT_CAP): + allocator_(ObModIds::OB_LOB_READER, OB_MALLOC_NORMAL_BLOCK_SIZE/*8KB*/, MTL_ID()), + cap_(cap) + {} + ~ObPersistLobReaderCache(); + + int get(ObPersistLobReaderCacheKey key, ObPersistLobReader *&reader); + int put(ObPersistLobReaderCacheKey key, ObPersistLobReader *reader); + + ObPersistLobReader* alloc_reader(); + ObIAllocator& get_allocator() { return allocator_; } + +private: + int remove_first(); + +private: + ObArenaAllocator allocator_; + const int32_t cap_; + ObDList list_; +}; + + +struct ObLobAccessCtx +{ + ObLobAccessCtx(): + reader_cache_() + {} + ObPersistLobReaderCache reader_cache_; +}; + + +} // storage +} // oceanbase + +#endif \ No newline at end of file diff --git a/src/storage/lob/ob_lob_util.cpp b/src/storage/lob/ob_lob_util.cpp index 44a6aa540e..346a1c45d7 100644 --- a/src/storage/lob/ob_lob_util.cpp +++ b/src/storage/lob/ob_lob_util.cpp @@ -16,7 +16,7 @@ #include "ob_lob_manager.h" #include "storage/tx/ob_trans_service.h" #include "storage/blocksstable/ob_datum_row.h" -#include "ob_lob_meta.h" +#include "storage/lob/ob_lob_meta.h" #include "storage/tx_storage/ob_access_service.h" namespace oceanbase @@ -125,6 +125,17 @@ int64_t ObLobAccessParam::get_inrow_threshold() return res; } +int ObLobAccessParam::is_timeout() +{ + int ret = OB_SUCCESS; + int64_t cur_time = ObTimeUtility::current_time(); + if (cur_time > timeout_) { + ret = OB_TIMEOUT; + LOG_WARN("query timeout", K(ret), K(cur_time), K(timeout_)); + } + return ret; +} + int ObInsertLobColumnHelper::start_trans(const share::ObLSID &ls_id, const bool is_for_read, const int64_t timeout_ts, diff --git a/src/storage/lob/ob_lob_util.h b/src/storage/lob/ob_lob_util.h index 437f21bd8a..cebe538649 100644 --- a/src/storage/lob/ob_lob_util.h +++ b/src/storage/lob/ob_lob_util.h @@ -49,9 +49,16 @@ struct ObLobStorageParam int64_t inrow_threshold_; }; +class ObLobAccessCtx; + struct ObLobAccessParam { + +public: + static const int32_t DEFAULT_QUERY_CACHE_THRESHOLD = 256 * 1024; +public: + ObLobAccessParam() - : tx_desc_(nullptr), snapshot_(), tx_id_(), read_latest_(0), + : tmp_allocator_(nullptr), tx_desc_(nullptr), snapshot_(), tx_id_(), read_latest_(0), sql_mode_(SMO_DEFAULT), allocator_(nullptr), dml_base_param_(nullptr), column_ids_(), meta_table_schema_(nullptr), piece_table_schema_(nullptr), @@ -64,18 +71,22 @@ struct ObLobAccessParam { scan_backward_(false), asscess_ptable_(false), offset_(0), len_(0), parent_seq_no_(), seq_no_st_(), used_seq_cnt_(0), total_seq_cnt_(0), checksum_(0), update_len_(0), op_type_(ObLobDataOutRowCtx::OpType::SQL), is_fill_zero_(false), from_rpc_(false), - inrow_read_nocopy_(false), inrow_threshold_(OB_DEFAULT_LOB_INROW_THRESHOLD), schema_chunk_size_(OB_DEFAULT_LOB_CHUNK_SIZE), spec_lob_id_(), - remote_query_ctx_(nullptr) + inrow_read_nocopy_(false), inrow_threshold_(OB_DEFAULT_LOB_INROW_THRESHOLD), schema_chunk_size_(OB_DEFAULT_LOB_CHUNK_SIZE), + access_ctx_(nullptr), is_store_char_len_(true), spec_lob_id_(), remote_query_ctx_(nullptr) {} ~ObLobAccessParam(); public: - bool is_full_read() const { return op_type_ == ObLobDataOutRowCtx::OpType::SQL && 0 == offset_ && (len_ == byte_size_ || INT64_MAX == len_ || UINT64_MAX == len_); } bool is_full_delete() const { return op_type_ == ObLobDataOutRowCtx::OpType::SQL && 0 == offset_ && len_ >= byte_size_; } bool is_full_insert() const { return op_type_ == ObLobDataOutRowCtx::OpType::SQL && 0 == offset_ && 0 == byte_size_; } + bool has_single_chunk() const; + bool enable_block_cache() const; + int set_lob_locator(common::ObLobLocatorV2 *lob_locator); + int is_timeout(); + bool is_char() { return coll_type_ != common::ObCollationType::CS_TYPE_BINARY; } // chunk size can be changed online. // that means lob data that has been writed may have different chunk size with schema @@ -89,12 +100,20 @@ public: } int64_t get_inrow_threshold(); + int get_rowkey_range(ObObj key_objs[4], ObNewRange &range); + + void set_tmp_allocator(ObIAllocator *tmp_allocator) { tmp_allocator_ = tmp_allocator; } + ObIAllocator* get_tmp_allocator() { return nullptr != tmp_allocator_ ? tmp_allocator_ : allocator_; } TO_STRING_KV(K_(tenant_id), K_(src_tenant_id), K_(ls_id), K_(tablet_id), K_(lob_meta_tablet_id), K_(lob_piece_tablet_id), - KPC_(lob_locator), KPC_(lob_common), KPC_(lob_data), K_(byte_size), K_(handle_size), + KPC_(lob_locator), KPC_(lob_common), KPC_(lob_data), K_(byte_size), K_(handle_size), K_(timeout), KP_(allocator), KP_(tmp_allocator), K_(coll_type), K_(scan_backward), K_(offset), K_(len), K_(parent_seq_no), K_(seq_no_st), K_(used_seq_cnt), K_(total_seq_cnt), K_(checksum), K_(update_len), K_(op_type), K_(is_fill_zero), K_(from_rpc), K_(snapshot), K_(tx_id), K_(read_latest), - K_(inrow_read_nocopy), K_(schema_chunk_size), K_(inrow_threshold), K_(spec_lob_id)); + K_(inrow_read_nocopy), K_(schema_chunk_size), K_(inrow_threshold), K_(is_store_char_len), KP_(remote_query_ctx), + KP_(access_ctx), K_(spec_lob_id)); + +private: + ObIAllocator *tmp_allocator_; public: transaction::ObTxDesc *tx_desc_; // for write/update/delete @@ -146,6 +165,8 @@ public: int64_t inrow_threshold_; int64_t schema_chunk_size_; ObObj ext_info_log_; + ObLobAccessCtx *access_ctx_; + bool is_store_char_len_; ObLobId spec_lob_id_; // remote query ctx void *remote_query_ctx_; diff --git a/src/storage/lob/ob_lob_write_buffer.cpp b/src/storage/lob/ob_lob_write_buffer.cpp new file mode 100644 index 0000000000..af235d048d --- /dev/null +++ b/src/storage/lob/ob_lob_write_buffer.cpp @@ -0,0 +1,478 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "ob_lob_write_buffer.h" + +namespace oceanbase +{ +namespace storage +{ + +size_t ObLobWriteBuffer::max_bytes_charpos( + const ObCollationType collation_type, + const char *str, + const int64_t str_len, + const int64_t max_bytes, /* max buffer size can be writed */ + int64_t &char_len) +{ + int64_t byte_len = max_bytes; + byte_len = ObCharset::max_bytes_charpos(collation_type, str, str_len, byte_len, char_len); + byte_len = ob_lob_writer_length_validation(collation_type, str_len, byte_len, char_len); + return byte_len; +} + +int ObLobWriteBuffer::align_write_postion(char *data_ptr, int64_t data_byte_len, int64_t &byte_len, int64_t &char_len) +{ + int ret = OB_SUCCESS; + if (data_byte_len > max_byte_len_) { + byte_len = ObCharset::max_bytes_charpos(coll_type_, + data_ptr, + data_byte_len, + max_byte_len_, + char_len); + byte_len = ob_lob_writer_length_validation(coll_type_, data_byte_len, byte_len, char_len); + } else { + byte_len = data_byte_len; + char_len = -1; + } + return ret; +} + +int ObLobWriteBuffer::get_byte_range( + const char *data_ptr, + const int64_t data_len, + const int64_t char_offset, + const int64_t char_len, + int64_t &byte_offset, + int64_t &byte_len) const +{ + int ret = OB_SUCCESS; + byte_offset = ObCharset::charpos(coll_type_, data_ptr, data_len, char_offset); + byte_len = ObCharset::charpos(coll_type_, data_ptr + byte_offset, data_len - byte_offset, char_len); + if (byte_offset < char_offset || byte_len < char_len) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("oversize", K(ret), K(byte_offset), K(char_offset), K(byte_len), K(char_len), K(data_len), KP(data_ptr)); + } + return ret; +} + +int ObLobWriteBuffer::get_byte_range( + const int64_t char_offset, + const int64_t char_len, + int64_t &byte_offset, + int64_t &byte_len) const +{ + return get_byte_range(buffer_ptr(), byte_length(), char_offset, char_len, byte_offset, byte_len); +} + +int ObLobWriteBuffer::move_to_remain_buffer( + ObString &remain_buf, + int64_t move_src_byte_offset, + int64_t total_move_byte_len, + int64_t &real_move_byte_len/*in out param*/, + int64_t &real_move_char_len) +{ + int ret = OB_SUCCESS; + int64_t remain_byte_len = total_move_byte_len - real_move_byte_len; + if (remain_byte_len < 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("remain_byte_len invalid", K(ret), K(remain_byte_len), K(total_move_byte_len), K(real_move_byte_len)); + } else if (remain_byte_len == 0) { + // no need move + LOG_DEBUG("remain_byte_len is zero, not move", K(remain_byte_len), K(total_move_byte_len), K(real_move_byte_len)); + } else { + // need ensure remain buffer data is full char + real_move_byte_len = ObLobWriteBuffer::max_bytes_charpos( + coll_type_, + buffer_ptr() + move_src_byte_offset, + total_move_byte_len, + real_move_byte_len, + real_move_char_len); + // move_byte_len may be less after max_bytes_charpos, so need update remain_byte_len + remain_byte_len = total_move_byte_len - real_move_byte_len; + if (remain_byte_len > remain_buf.size()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("remain buffer oversize", K(ret), K(remain_byte_len), "remain_buffer size", remain_buf.size(), K(total_move_byte_len), K(real_move_byte_len)); + } else if (remain_buf.write(buffer_ptr() + move_src_byte_offset + real_move_byte_len, remain_byte_len) != remain_byte_len) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("write data to remain buffer fail", K(ret), K(move_src_byte_offset), K(real_move_byte_len), K(remain_byte_len), K(remain_buf)); + } + } + return ret; +} + +int ObLobWriteBuffer::move_data_for_write( + int64_t write_byte_offset, + int64_t write_old_byte_len, + int64_t write_new_byte_len, + ObString &remain_buf) +{ + int ret = OB_SUCCESS; + int64_t move_src_byte_offset = write_byte_offset + write_old_byte_len; + int64_t move_dst_byte_offset = write_byte_offset + write_new_byte_len; + int64_t total_move_byte_len = byte_length() - move_src_byte_offset; + int64_t real_move_byte_len = move_dst_byte_offset <= move_src_byte_offset ? total_move_byte_len : (byte_length() <= move_dst_byte_offset ? 0 : byte_length() - move_dst_byte_offset); + int64_t real_move_char_len = real_move_byte_len; + + if (total_move_byte_len < 0 || real_move_byte_len < 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid length", K(ret), K(total_move_byte_len), K(real_move_byte_len), K(write_byte_offset), K(write_old_byte_len), K(write_new_byte_len), K(byte_length())); + } else if (move_src_byte_offset == move_dst_byte_offset) { + // src and dst is same, no need move + LOG_DEBUG("src and dst is same, so not move", K(move_src_byte_offset), K(move_dst_byte_offset), K(total_move_byte_len)); + } else if (OB_FAIL(move_to_remain_buffer(remain_buf, move_src_byte_offset, total_move_byte_len, real_move_byte_len, real_move_char_len))) { + LOG_WARN("move_to_remain_buffer fail", K(ret), K(move_src_byte_offset), K(total_move_byte_len), K(real_move_byte_len), K(real_move_char_len)); + } else { + if (real_move_byte_len > 0) { + MEMMOVE(buffer_ptr() + move_dst_byte_offset, buffer_ptr() + move_src_byte_offset, real_move_byte_len); + } else { + LOG_DEBUG("no data move", K(total_move_byte_len), K(real_move_byte_len), K(write_byte_offset), K(write_old_byte_len), K(write_new_byte_len), K(byte_length())); + } + if (OB_FAIL(ret)) { + } else if (inner_buffer_.set_length(move_dst_byte_offset + real_move_byte_len) != move_dst_byte_offset + real_move_byte_len) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("set_length fail", K(ret), K(inner_buffer_.size()), K(inner_buffer_.length()), K(move_dst_byte_offset), K(real_move_byte_len)); + } + } + return ret; +} + +int ObLobWriteBuffer::move_data_for_write( + int64_t write_byte_offset, + int64_t write_old_byte_len, + int64_t write_new_byte_len) +{ + ObString remain_buf; + return move_data_for_write(write_byte_offset, write_old_byte_len, write_new_byte_len, remain_buf); +} + +int ObLobWriteBuffer::do_write( + const int64_t write_byte_offset, + ObString &write_data) +{ + int ret = OB_SUCCESS; + if (write_data.empty()) { + LOG_DEBUG("not write because write_data is empty"); + } else if (write_byte_offset + write_data.length() > buffer_size()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("out of buffer range", K(ret), K(write_byte_offset), K(buffer_size()), K(write_data)); + } else { + MEMCPY(buffer_ptr() + write_byte_offset, write_data.ptr(), write_data.length()); + if (write_byte_offset + write_data.length() > inner_buffer_.length()) { + if (inner_buffer_.set_length(write_byte_offset + write_data.length()) != write_byte_offset + write_data.length()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("set_length fail", K(ret), K(inner_buffer_.size()), K(inner_buffer_.length()), K(write_byte_offset), K(write_data.length())); + } + } + } + return ret; +} + +int ObLobWriteBuffer::byte_write( + int64_t write_byte_offset, + int64_t write_old_byte_len, + ObString &write_data, + ObString &remain_buf) +{ + int ret = OB_SUCCESS; + if (! use_buffer() ) { + if (write_byte_offset > 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("write not at start pos", K(ret), K(write_byte_offset), K(write_old_byte_len), K(write_data)); + } else if (OB_FAIL(set_data(write_data.ptr(), write_data.length()))) { + LOG_WARN("set_data fail", K(ret)); + } + } else if (write_byte_offset + write_data.length() > buffer_size()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("out of buffer range", K(ret), K(write_byte_offset), K(buffer_size()), K(write_data)); + } else if (OB_FAIL(move_data_for_write(write_byte_offset, write_old_byte_len, write_data.length(), remain_buf))) { + LOG_WARN("move_data_for_write fail", K(ret), K(write_byte_offset), K(write_old_byte_len), K(write_data)); + } else if (OB_FAIL(do_write(write_byte_offset, write_data))) { + LOG_WARN("do_write fail", K(ret), K(write_byte_offset)); + } + return ret; +} + + +int ObLobWriteBuffer::char_write( + int64_t write_char_offset, + int64_t write_char_len, + ObString &write_data, + ObString &remain_buf) +{ + int ret = OB_SUCCESS; + int64_t write_byte_offset = write_char_offset; + int64_t write_old_byte_len = write_char_len; + if (! is_char()) { + if (OB_FAIL(byte_write(write_byte_offset, write_old_byte_len, write_data, remain_buf))) { + LOG_WARN("byte_write fail", K(ret), K(coll_type_), K(write_byte_offset), K(write_old_byte_len), K(write_data)); + } + } else if (use_buffer()) { + if (OB_FAIL(get_byte_range(write_char_offset, write_char_len, write_byte_offset, write_old_byte_len))) { + LOG_WARN("get_byte_range fail", K(ret), K(write_char_offset), K(write_char_len), K(coll_type_), K(inner_buffer_)); + } else if (OB_FAIL(byte_write(write_byte_offset, write_old_byte_len, write_data, remain_buf))) { + LOG_WARN("byte_write fail", K(ret), K(coll_type_), K(write_byte_offset), K(write_old_byte_len)); + } + } else if (write_char_offset > 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("write not at start pos", K(ret), K(write_char_offset), K(write_char_len), K(write_data)); + } else if (OB_FAIL(set_data(write_data.ptr(), write_data.length()))) { + LOG_WARN("set_data fail", K(ret)); + } + return ret; +} + +int ObLobWriteBuffer::fill_zero_data( + const int64_t byte_offset, + const int64_t byte_len, + const int64_t char_len, + ObString &space) +{ + int ret = OB_SUCCESS; + if (char_len * space.length() != byte_len) { + LOG_WARN("fill zero length invalid", K(ret), K(space.length()), K(char_len), K(byte_len)); + } else if (space.length() == 1) { + MEMSET(buffer_ptr() + byte_offset, space.ptr()[0], byte_len); + } else { + // used for utf16 currently + int64_t space_len = space.length(); + for (int i = 0; i < char_len; i++) { + MEMCPY(buffer_ptr() + byte_offset + i * space_len, space.ptr(), space_len); + } + } + return ret; +} + +ObString ObLobWriteBuffer::get_const_str() const +{ + ObString space; + if (is_char()) { + space = ObCharsetUtils::get_const_str(coll_type_, ' '); + } else { + space = ObCharsetUtils::get_const_str(coll_type_, '\x00'); + } + return space; +} + +int ObLobWriteBuffer::byte_fill_zero( + int64_t fill_byte_offset, + int64_t fill_old_byte_len, + int64_t fill_char_len) +{ + int ret = OB_SUCCESS; + ObString space = get_const_str(); + int64_t fill_new_byte_len = space.length() * fill_char_len; + if (! use_buffer()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("should be use_buffer mode", K(ret)); + } else if (OB_FAIL(move_data_for_write(fill_byte_offset, fill_old_byte_len, fill_new_byte_len))) { + LOG_WARN("move_data_for_write fail", K(ret)); + } else if (OB_FAIL(fill_zero_data(fill_byte_offset, fill_new_byte_len, fill_char_len, space))){ + LOG_WARN("fill_zero_data fail", K(ret)); + } + return ret; +} + +int ObLobWriteBuffer::char_fill_zero( + int64_t fill_char_offset, + int64_t fill_char_len) +{ + int ret = OB_SUCCESS; + int64_t fill_byte_offset = fill_char_offset; + int64_t fill_old_byte_len = fill_char_len; + if (! is_char()) { + if (OB_FAIL(byte_fill_zero(fill_byte_offset, fill_old_byte_len, fill_char_len))) { + LOG_WARN("byte_fill_zero fail", K(ret), K(coll_type_), K(fill_char_offset), K(fill_char_len), K(fill_byte_offset), K(fill_old_byte_len)); + } + } else if (OB_FAIL(get_byte_range(fill_char_offset, fill_char_len, fill_byte_offset, fill_old_byte_len))) { + LOG_WARN("get_byte_range fail", K(ret), K(fill_char_offset), K(fill_char_len), K(coll_type_), K(inner_buffer_)); + } else if (OB_FAIL(byte_fill_zero(fill_byte_offset, fill_old_byte_len, fill_char_len))) { + LOG_WARN("byte_fill_zero fail", K(ret)); + } + return ret; +} + +int ObLobWriteBuffer::padding(int64_t char_len, int64_t &real_write_byte_len) +{ + int ret = OB_SUCCESS; + ObString space = ObCharsetUtils::get_const_str(coll_type_, ' '); + int64_t space_len = space.length(); + int64_t byte_len = OB_MIN(space_len * char_len, remain()); + int64_t inner_buffer_length = inner_buffer_.length(); + if (! use_buffer()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("should be use_buffer mode when padding", K(ret)); + } else if (is_char()) { + if (space.length() == 1) { + MEMSET(inner_buffer_.ptr() + inner_buffer_length, space.ptr()[0], byte_len); + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not support", K(ret), K(coll_type_), K(space)); + } + } else { + MEMSET(inner_buffer_.ptr() + inner_buffer_length, 0x00, byte_len); + } + + if (OB_FAIL(ret)) { + } else if (inner_buffer_.set_length(inner_buffer_length + byte_len) != inner_buffer_length + byte_len) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("set_length fail", K(ret), K(inner_buffer_.size()), K(inner_buffer_.length()), K(inner_buffer_length), K(byte_len)); + } else { + real_write_byte_len = byte_len; + is_full_ = (is_char() && inner_buffer_.remain() < max_bytes_in_char_); + } + return ret; +} + +int ObLobWriteBuffer::append(char *data_ptr, int64_t data_byte_len, int64_t &real_write_byte_len) +{ + int ret = OB_SUCCESS; + int64_t write_byte_len = 0; + int64_t write_char_len = 0; + + if (! use_buffer()) { + if (OB_FAIL(set_data(data_ptr, data_byte_len, real_write_byte_len))) { + LOG_WARN("set_data fail", K(ret), K(data_byte_len), K(real_write_byte_len)); + } + } else if (OB_FAIL(align_write_postion(data_ptr, data_byte_len, write_byte_len, write_char_len))) { + LOG_WARN("align_write_postion fail", K(ret), K(data_byte_len), K(inner_buffer_)); + } else if (inner_buffer_.write(data_ptr, write_byte_len) != write_byte_len) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("write data to inner buffer fail", K(ret), K(data_byte_len), K(write_byte_len), K(inner_buffer_)); + } else { + real_write_byte_len = write_byte_len; + is_full_ = (is_char() && inner_buffer_.remain() < max_bytes_in_char_); + } + return ret; +} + +int ObLobWriteBuffer::append(char *data_ptr, int64_t data_byte_len) +{ + int ret = OB_SUCCESS; + int64_t real_write_byte_len = 0; + if (OB_FAIL(append(data_ptr, data_byte_len, real_write_byte_len))) { + LOG_WARN("append fail", K(ret), K(data_byte_len), KP(data_ptr)); + } else if (real_write_byte_len != data_byte_len) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("write fail", K(ret), K(real_write_byte_len), K(data_byte_len), KP(data_ptr)); + } + return ret; +} + +int ObLobWriteBuffer::char_append(ObString &data, int64_t write_char_offset, int64_t write_char_len) +{ + int ret = OB_SUCCESS; + int64_t write_byte_offset = 0; + int64_t write_byte_len = 0; + if (OB_FAIL(get_byte_range(data.ptr(), data.length(), write_char_offset, write_char_len, write_byte_offset, write_byte_len))) { + LOG_WARN("get_byte_range fail", K(ret), K(write_char_offset), K(write_char_len), K(coll_type_), K(inner_buffer_)); + } else if (OB_FAIL(append(data.ptr() + write_byte_offset, write_byte_len))) { + LOG_WARN("append fail", K(ret), K(write_byte_offset), K(write_byte_len), K(write_char_offset), K(write_char_len)); + } + return ret; +} + +int ObLobWriteBuffer::set_data(char *data_ptr, int64_t data_byte_len, int64_t &real_set_byte_len) +{ + int ret = OB_SUCCESS; + int64_t char_len = 0; + int64_t byte_len = 0; + if (OB_FAIL(align_write_postion(data_ptr, data_byte_len, byte_len, char_len))) { + LOG_WARN("align_write_postion fail", K(ret), K(data_byte_len), KP(data_ptr)); + } else if (char_len != -1 && OB_FAIL(set_char_len(char_len))) { + LOG_WARN("set_char_len fail", K(ret), K(char_len), K(data_byte_len), K(byte_len)); + } else { + inner_buffer_.assign_ptr(data_ptr, byte_len); + use_buffer_ = false; + is_full_ = true; + real_set_byte_len = byte_len; + } + return ret; +} + +int ObLobWriteBuffer::set_data(char *data_ptr, int64_t data_byte_len) +{ + int ret = OB_SUCCESS; + int64_t real_set_byte_len = 0; + if (OB_FAIL(set_data(data_ptr, data_byte_len, real_set_byte_len))) { + LOG_WARN("set_data fail", K(ret), K(data_ptr), KP(data_ptr)); + } else if (real_set_byte_len != data_byte_len) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("write fail", K(ret), K(real_set_byte_len), K(data_byte_len), KP(data_ptr)); + } + return ret; +} + +int ObLobWriteBuffer::set_buffer(char *buf_ptr, int64_t buf_len) +{ + int ret = OB_SUCCESS; + if (buf_len != max_byte_len_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("buf_len should equal to max_byte_len", K(ret), K(buf_len), K(max_byte_len_)); + } else { + inner_buffer_.assign_buffer(buf_ptr, buf_len); + use_buffer_ = true; + is_full_ = false; + is_char_len_valid_ = false; + } + return ret; +} + +int ObLobWriteBuffer::to_lob_meta_info(ObLobMetaInfo &meta_info) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(get_char_len(meta_info.char_len_))) { + LOG_WARN("get_char_len fail", K(ret), KPC(this)); + } else { + meta_info.byte_len_ = inner_buffer_.length(); + meta_info.lob_data_ = inner_buffer_; + } + return ret; +} + +int ObLobWriteBuffer::set_char_len(const int64_t char_len) +{ + int ret = OB_SUCCESS; + if (is_char_len_valid_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("can not set because char_len is valid", K(ret), K(is_char_len_valid_), K(char_len_), K(char_len)); + } else { + char_len_ = char_len; + is_char_len_valid_ = true; + } + return ret; +} + +int ObLobWriteBuffer::get_char_len(uint32_t &char_len) const +{ + int ret = OB_SUCCESS; + if (is_store_char_len_) { + if (is_char_len_valid_) { + char_len = char_len_; + } else { + char_len = ObCharset::strlen_char(coll_type_, inner_buffer_.ptr(), inner_buffer_.length()); + } + } else { + if (is_char_len_valid_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unkown situtation", K(ret), KPC(this)); + } else { + char_len = UINT32_MAX; + } + } + return ret; +} + +} +} diff --git a/src/storage/lob/ob_lob_write_buffer.h b/src/storage/lob/ob_lob_write_buffer.h new file mode 100644 index 0000000000..878c2f258e --- /dev/null +++ b/src/storage/lob/ob_lob_write_buffer.h @@ -0,0 +1,157 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEABASE_STORAGE_OB_LOB_WRITE_BUFFER_ +#define OCEABASE_STORAGE_OB_LOB_WRITE_BUFFER_ + +#include "storage/lob/ob_lob_meta.h" +#include "storage/lob/ob_lob_util.h" + +namespace oceanbase +{ +namespace storage +{ + +struct ObLobWriteBuffer +{ +public: + static size_t max_bytes_charpos( + const ObCollationType collation_type, + const char *str, + const int64_t str_len, + const int64_t max_bytes, /* max buffer size can be writed */ + int64_t &char_len); + +public: + ObLobWriteBuffer(ObCollationType coll_type, int64_t max_byte_len, bool is_store_char_len): + coll_type_(coll_type), + max_byte_len_(max_byte_len), + inner_buffer_(), + use_buffer_(false), + is_full_(false), + is_store_char_len_(is_store_char_len), + is_char_len_valid_(false), + char_len_(0), + max_bytes_in_char_(16) + {} + + bool is_full() const { return is_full_; } + bool use_buffer() const { return use_buffer_; } + bool is_char() const { return coll_type_ != common::ObCollationType::CS_TYPE_BINARY; } + int64_t byte_length() const { return inner_buffer_.length(); } + + int64_t remain() const { return inner_buffer_.remain(); } + char* buffer_ptr() { return inner_buffer_.ptr(); } + const char* buffer_ptr() const { return inner_buffer_.ptr(); } + int64_t buffer_size() const { return inner_buffer_.size(); } + + int align_write_postion(char *data_ptr, int64_t data_byte_len, int64_t &byte_len, int64_t &char_len); + + int to_lob_meta_info(ObLobMetaInfo &meta_info) const; + +private: + ObString get_const_str() const; + int get_byte_range( + const int64_t char_offset, + const int64_t char_len, + int64_t &byte_offset, + int64_t &byte_len) const; + int get_byte_range( + const char *data_ptr, + const int64_t data_len, + const int64_t char_offset, + const int64_t char_len, + int64_t &byte_offset, + int64_t &byte_len) const; + + int move_to_remain_buffer( + ObString &remain_buf, + int64_t move_src_byte_offset, + int64_t total_move_byte_len, + int64_t &real_move_byte_len/*in out param*/, + int64_t &real_move_char_len); + + int move_data_for_write( + int64_t write_byte_offset, + int64_t write_old_byte_len, + int64_t write_new_byte_len, + ObString &remain_buf); + + int move_data_for_write( + int64_t write_byte_offset, + int64_t write_old_byte_len, + int64_t write_new_byte_len); + + int do_write( + const int64_t write_byte_offset, + ObString &write_data); + + int set_char_len(const int64_t char_len); + int get_char_len(uint32_t &char_len) const; + +public: + int byte_write( + int64_t write_byte_offset, + int64_t write_old_byte_len, + ObString &write_data, + ObString &remain_buf); + + int char_write( + int64_t write_char_offset, + int64_t write_char_len, + ObString &write_data, + ObString &remain_buf); + + int fill_zero_data( + const int64_t byte_offset, + const int64_t byte_len, + const int64_t char_len, + ObString &space); + + int byte_fill_zero( + int64_t fill_byte_offset, + int64_t fill_old_byte_len, + int64_t fill_char_len); + + int char_fill_zero( + int64_t fill_char_offset, + int64_t fill_char_len); + + int padding(int64_t char_len, int64_t &real_write_byte_len); + + int set_data(char *data_ptr, int64_t data_byte_len, int64_t &real_set_byte_len); + int set_data(char *data_ptr, int64_t data_byte_len); + int set_buffer(char *buf_ptr, int64_t buf_len); + + int append(char *data_ptr, int64_t data_byte_len, int64_t &real_write_byte_len); + int append(char *data_ptr, int64_t data_byte_len); + int char_append(ObString &data, int64_t write_char_offset, int64_t write_char_len); + + ObCollationType coll_type_; + int64_t max_byte_len_; + ObString inner_buffer_; + bool use_buffer_; + bool is_full_; + bool is_store_char_len_; + bool is_char_len_valid_; + int64_t char_len_; + int64_t max_bytes_in_char_; + + TO_STRING_KV(K_(coll_type), K_(max_byte_len), K_(use_buffer), K_(is_full), K_(inner_buffer), K_(is_store_char_len), + K_(is_char_len_valid), K_(char_len)); +}; + + +} // storage +} // oceanbase + +#endif diff --git a/src/storage/ls/ob_ls_tablet_service.cpp b/src/storage/ls/ob_ls_tablet_service.cpp index 2a33196bb7..448e7dac2e 100644 --- a/src/storage/ls/ob_ls_tablet_service.cpp +++ b/src/storage/ls/ob_ls_tablet_service.cpp @@ -4184,23 +4184,6 @@ int ObLSTabletService::insert_lob_col( return ret; } -int ObLSTabletService::check_lob_tablet_valid(ObTabletHandle &data_tablet) -{ - int ret = OB_SUCCESS; - bool is_valid_aux_lob_table = false; - ObTabletBindingMdsUserData ddl_data; - if (OB_FAIL(data_tablet.get_obj()->ObITabletMdsInterface::get_ddl_data(share::SCN::max_scn(), ddl_data))) { - LOG_WARN("failed to get ddl data from tablet", K(ret), K(data_tablet)); - } else { - is_valid_aux_lob_table = ddl_data.lob_meta_tablet_id_.is_valid() && ddl_data.lob_piece_tablet_id_.is_valid(); - if (!is_valid_aux_lob_table) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("aux lob table must valid when lob column exist", K(ret), K(ddl_data)); - } - } - return ret; -} - int ObLSTabletService::insert_lob_tablet_row( ObTabletHandle &data_tablet, ObDMLRunningCtx &run_ctx, @@ -4209,7 +4192,6 @@ int ObLSTabletService::insert_lob_tablet_row( int ret = OB_SUCCESS; int64_t col_cnt = run_ctx.col_descs_->count(); ObLobManager *lob_mngr = MTL(ObLobManager*); - bool check_lob = false; if (OB_ISNULL(lob_mngr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("[STORAGE_LOB]failed to get lob manager handle.", K(ret)); @@ -4223,15 +4205,7 @@ int ObLSTabletService::insert_lob_tablet_row( if (obj.is_null() || obj.is_nop_value()) { // do nothing } else if (column.col_type_.is_lob_storage()) { - if (!check_lob) { - if (OB_FAIL(check_lob_tablet_valid(data_tablet))) { - LOG_WARN("failed to check_lob_tablet_valid", K(ret), K(data_tablet)); - } else { - check_lob = true; - } - } - if (OB_FAIL(ret)) { - } else if(OB_FAIL(insert_lob_col(run_ctx, column, obj, nullptr, nullptr))) { + if(OB_FAIL(insert_lob_col(run_ctx, column, obj, nullptr, nullptr))) { LOG_WARN("[STORAGE_LOB]failed to insert lob col.", K(ret), K(row), K(i)); } } @@ -4240,6 +4214,33 @@ int ObLSTabletService::insert_lob_tablet_row( return ret; } +int update_lob_meta_table_seq_no(ObDMLRunningCtx &run_ctx, int64_t row_count) +{ + int ret = OB_SUCCESS; + const ObDMLBaseParam &dml_param = run_ctx.dml_param_; + const ObTableDMLParam *table_param = dml_param.table_param_; + if (OB_ISNULL(table_param)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table_param is null", K(ret)); + } else if (! table_param->get_data_table().is_lob_meta_table()) { + // skip if not lob meta table + } else if (row_count != 1) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("lob meta table row_count incorrect", K(ret), K(row_count)); + } else if (! dml_param.spec_seq_no_.is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("spec_seq_no_ is invalid", K(ret), K(row_count), K(dml_param)); + } else if (! run_ctx.store_ctx_.mvcc_acc_ctx_.tx_scn_.is_valid() + || run_ctx.store_ctx_.mvcc_acc_ctx_.tx_scn_ > dml_param.spec_seq_no_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("seq_no unexpected", K(run_ctx.store_ctx_.mvcc_acc_ctx_.tx_scn_), K(dml_param.spec_seq_no_)); + } else { + LOG_DEBUG("set seq_no", K(run_ctx.store_ctx_.mvcc_acc_ctx_.tx_scn_), K(dml_param.spec_seq_no_)); + run_ctx.store_ctx_.mvcc_acc_ctx_.tx_scn_ = dml_param.spec_seq_no_; + } + return ret; +} + int ObLSTabletService::insert_lob_tablet_rows( ObTabletHandle &data_tablet, ObDMLRunningCtx &run_ctx, @@ -4252,6 +4253,8 @@ int ObLSTabletService::insert_lob_tablet_rows( if (OB_ISNULL(lob_mngr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("[STORAGE_LOB]failed to get lob manager handle.", K(ret)); + } else if (OB_FAIL(update_lob_meta_table_seq_no(run_ctx, row_count))) { + LOG_WARN("update_lob_meta_table_seq_no fail", K(ret), K(run_ctx.dml_param_)); } else { int64_t col_cnt = run_ctx.col_descs_->count(); for (int64_t k = 0; OB_SUCC(ret) && k < row_count; k++) { @@ -4582,13 +4585,14 @@ int ObLSTabletService::process_lob_row( ObStoreRow &new_row) { int ret = OB_SUCCESS; - bool check_lob = false; if (OB_UNLIKELY(old_row.row_val_.get_count() != new_row.row_val_.get_count())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("[STORAGE_LOB]invalid args", K(old_row), K(new_row), K(ret)); } else if (OB_UNLIKELY(old_row.row_val_.get_count() != run_ctx.col_descs_->count())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("[STORAGE_LOB]invalid args", K(old_row), K(new_row), KPC(run_ctx.col_descs_)); + } else if (OB_FAIL(update_lob_meta_table_seq_no(run_ctx, 1/*row_count*/))) { + LOG_WARN("update_lob_meta_table_seq_no fail", K(ret), K(run_ctx.dml_param_)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < old_row.row_val_.get_count(); ++i) { if (run_ctx.col_descs_->at(i).col_type_.is_lob_storage()) { @@ -4602,13 +4606,6 @@ int ObLSTabletService::process_lob_row( } } if (is_update) { - if (!check_lob) { - if (OB_FAIL(check_lob_tablet_valid(tablet_handle))) { - LOG_WARN("failed to check_lob_tablet_valid", K(ret), K(tablet_handle)); - } else { - check_lob = true; - } - } // get new lob locator ObString new_lob_str = (new_obj.is_null() || new_obj.is_nop_value()) ? ObString(0, nullptr) : new_obj.get_string(); @@ -5621,6 +5618,8 @@ int ObLSTabletService::delete_lob_tablet_rows( if (tbl_row.row_val_.count_ != col_cnt) { ret = OB_ERR_UNEXPECTED; LOG_WARN("[STORAGE_LOB]Invliad row col cnt", K(col_cnt), K(tbl_row)); + } else if (OB_FAIL(update_lob_meta_table_seq_no(run_ctx, 1/*row_count*/))) { + LOG_WARN("update_lob_meta_table_seq_no fail", K(ret), K(run_ctx.dml_param_)); } else { ObLobCommon *lob_common = nullptr; for (int64_t i = 0; OB_SUCC(ret) && i < col_cnt; ++i) { diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result index 8651001cf1..333e8f1ccd 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result @@ -125,6 +125,7 @@ large_query_threshold large_query_worker_percentage leak_mod_to_check lease_time +lob_enable_block_cache_threshold local_ip location_cache_cpu_quota location_cache_refresh_min_interval