diff --git a/src/logservice/libobcdc/src/ob_log_formatter.cpp b/src/logservice/libobcdc/src/ob_log_formatter.cpp index 87dadf5ca9..4a7cd8fde8 100644 --- a/src/logservice/libobcdc/src/ob_log_formatter.cpp +++ b/src/logservice/libobcdc/src/ob_log_formatter.cpp @@ -2167,6 +2167,7 @@ int ObLogFormatter::parse_aux_lob_meta_table_( LOG_ERROR("get_cols fail", KR(ret), K(stmt_task)); } else { const uint64_t tenant_id = log_entry_task->get_tenant_id(); + const DmlRedoLogNode *redo_log_node = log_entry_task->get_redo_log_node(); if (stmt_task.is_insert()) { if (OB_FAIL(parse_aux_lob_meta_table_insert_(*log_entry_task, stmt_task, *new_cols))) { @@ -2174,6 +2175,11 @@ int ObLogFormatter::parse_aux_lob_meta_table_( } } else if (stmt_task.is_update()) { // lob meta update data isn't used, just skip + } else if (OB_ISNULL(redo_log_node)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("redo_log_node is NULL", KR(ret)); + } else if (redo_log_node->is_direct_load_inc_log() && stmt_task.is_delete()) { + // lob meta delete isn't used in direct load inc case, just skip } else if (stmt_task.is_delete()) { if (OB_FAIL(parse_aux_lob_meta_table_delete_(*log_entry_task, stmt_task, *old_cols))) { LOG_ERROR("parse_aux_lob_meta_table_delete_ failed", KR(ret)); diff --git a/src/logservice/libobcdc/src/ob_log_part_trans_task.cpp b/src/logservice/libobcdc/src/ob_log_part_trans_task.cpp index 84db09b94f..2a08d74fad 100644 --- a/src/logservice/libobcdc/src/ob_log_part_trans_task.cpp +++ b/src/logservice/libobcdc/src/ob_log_part_trans_task.cpp @@ -108,6 +108,7 @@ int MutatorRow::parse_columns_( const ObTimeZoneInfoWrap *tz_info_wrap, const bool enable_output_hidden_primary_key, const ObLogAllDdlOperationSchemaInfo *all_ddl_operation_table_schema_info, + const bool is_macroblock_row, ColValueList &cols) { int ret = OB_SUCCESS; @@ -129,7 +130,7 @@ int MutatorRow::parse_columns_( int64_t column_offset = 0; for (int64_t column_stored_idx = 0; OB_SUCC(ret) && column_stored_idx < datum_row.get_column_count(); column_stored_idx++) { - if (dml_flag.is_delete_insert()) { + if (is_macroblock_row) { // filter extra rowkey column if (column_stored_idx >= rowkey_cnt && column_stored_idx < macroblock_row_user_column_idx) continue; if (column_stored_idx == macroblock_row_user_column_idx) { @@ -546,7 +547,9 @@ template int MutatorRow::parse_columns_( const bool is_parse_new_col, const blocksstable::ObDatumRow &datum_row, + const int64_t rowkey_cnt, const CDC_INNER_TABLE_SCHEMA &inner_table_schema, + const bool is_macroblock_row, ColValueList &cols) { int ret = OB_SUCCESS; @@ -560,11 +563,25 @@ int MutatorRow::parse_columns_( OBLOG_FORMATTER_LOG(DEBUG, "parse_columns_", K(is_parse_new_col), K(datum_row)); // Iterate through all Cells using Cell Reader + // The normal row and macroblock row is different: + // Normal Row: [rowkey column | user column] + // MacroBlock Row: [rowkey column | extra rowkey column | user column] + // Formatter need to filter the extra rowkey column + const int64_t macroblock_row_user_column_idx = rowkey_cnt + OB_MAX_EXTRA_ROWKEY_COLUMN_NUMBER; + // record the offset of the stored column between column + int64_t column_offset = 0; for (int64_t i = 0; OB_SUCC(ret) && i < datum_row.get_column_count(); i++) { + if (is_macroblock_row) { + // filter extra rowkey column + if (i >= rowkey_cnt && i < macroblock_row_user_column_idx) continue; + if (i == macroblock_row_user_column_idx) { + column_offset = OB_MAX_EXTRA_ROWKEY_COLUMN_NUMBER; + } + } uint64_t column_id = OB_INVALID_ID; const ObObj *value = NULL; blocksstable::ObStorageDatum &datum = datum_row.storage_datums_[i]; - column_id = col_des_array[i].col_id_; + column_id = col_des_array[i - column_offset].col_id_; if (OB_FAIL(deep_copy_encoded_column_value_(datum))) { LOG_ERROR("deep_copy_encoded_column_value_ failed", KR(ret), "column_stored_idx", i, K(datum), K(is_parse_new_col)); @@ -579,7 +596,7 @@ int MutatorRow::parse_columns_( ObObjMeta obj_meta; ObObj obj; - if (OB_FAIL(set_obj_propertie_(column_id, i, table_schema, obj_meta, obj))) { + if (OB_FAIL(set_obj_propertie_(column_id, i - column_offset, table_schema, obj_meta, obj))) { LOG_ERROR("set_obj_propertie_ failed", K(column_id), K(i), K(obj_meta), K(obj)); } else if (OB_FAIL(datum.to_obj_enhance(obj, obj_meta))) { LOG_ERROR("transfer datum to obj failed", KR(ret), K(datum), K(obj_meta)); @@ -859,6 +876,7 @@ int MacroBlockMutatorRow::parse_cols( tz_info_wrap, enable_output_hidden_primary_key, all_ddl_operation_table_schema_info, + true/*is_macroblock_row*/, new_cols_))) { LOG_ERROR("parse new columns fail", KR(ret), K(tenant_id), K(table_id), K(obj2str_helper), K(tb_schema_info), K(enable_output_hidden_primary_key)); @@ -912,7 +930,9 @@ int MacroBlockMutatorRow::parse_cols(const ObCDCLobAuxTableSchemaInfo &inner_tab if (OB_FAIL(parse_columns_( true/*is_parse_new_col*/, row_, + row_key_.get_obj_cnt(), inner_table_schema_info, + true/*is_macroblock_row*/, new_cols_))) { LOG_ERROR("parse new columns fail", KR(ret), K(table_schema)); } else { @@ -920,6 +940,10 @@ int MacroBlockMutatorRow::parse_cols(const ObCDCLobAuxTableSchemaInfo &inner_tab } } + if (OB_SUCC(ret)) { + cols_parsed_ = true; + } + return ret; } @@ -1088,6 +1112,7 @@ int MemtableMutatorRow::parse_cols( tz_info_wrap, enable_output_hidden_primary_key, all_ddl_operation_table_schema_info, + false/*is_macroblock_row*/, new_cols_))) { LOG_ERROR("parse new columns fail", KR(ret), K(tenant_id), K(table_id), K(new_row_), K(obj2str_helper), K(tb_schema_info), K(enable_output_hidden_primary_key)); @@ -1115,6 +1140,7 @@ int MemtableMutatorRow::parse_cols( tz_info_wrap, enable_output_hidden_primary_key, all_ddl_operation_table_schema_info, + false/*is_macroblock_row*/, old_cols_))) { LOG_ERROR("parse old columns fail", KR(ret), K(tenant_id), K(table_id), K(old_row_), K(obj2str_helper), K(tb_schema_info), K(enable_output_hidden_primary_key)); @@ -1177,7 +1203,9 @@ int MemtableMutatorRow::parse_cols(const ObCDCLobAuxTableSchemaInfo &inner_table } else if (OB_FAIL(parse_columns_( true/*is_parse_new_col*/, datum_row, + rowkey_.get_obj_cnt(), inner_table_schema_info, + false/*is_macroblock_row*/, new_cols_))) { LOG_ERROR("parse new columns fail", KR(ret), K(new_row_), K(table_schema)); } else { @@ -1195,7 +1223,9 @@ int MemtableMutatorRow::parse_cols(const ObCDCLobAuxTableSchemaInfo &inner_table } else if (OB_FAIL(parse_columns_( false/*is_parse_new_col*/, datum_row, + rowkey_.get_obj_cnt(), inner_table_schema_info, + false/*is_macroblock_row*/, old_cols_))) { LOG_ERROR("parse old columns fail", KR(ret), K(old_row_), K(table_schema)); } else { diff --git a/src/logservice/libobcdc/src/ob_log_part_trans_task.h b/src/logservice/libobcdc/src/ob_log_part_trans_task.h index b6c46a5e10..11919a3e1f 100644 --- a/src/logservice/libobcdc/src/ob_log_part_trans_task.h +++ b/src/logservice/libobcdc/src/ob_log_part_trans_task.h @@ -262,6 +262,7 @@ protected: const ObTimeZoneInfoWrap *tz_info_wrap, const bool enable_output_hidden_primary_key, const ObLogAllDdlOperationSchemaInfo *all_ddl_operation_table_schema_info, + const bool is_macroblock_row, ColValueList &cols); int parse_rowkey_( ColValueList &rowkey_cols, @@ -306,7 +307,9 @@ protected: int parse_columns_( const bool is_parse_new_col, const blocksstable::ObDatumRow &datum_row, + const int64_t rowkey_cnt, const CDC_INNER_TABLE_SCHEMA &inner_table_schema, + const bool is_macroblock_row, ColValueList &cols); template int parse_rowkey_( diff --git a/src/observer/table_load/ob_table_load_mem_compactor.cpp b/src/observer/table_load/ob_table_load_mem_compactor.cpp index 0c756824ca..84c1746f24 100644 --- a/src/observer/table_load/ob_table_load_mem_compactor.cpp +++ b/src/observer/table_load/ob_table_load_mem_compactor.cpp @@ -276,11 +276,20 @@ int ObTableLoadMemCompactor::inner_init() if (mem_ctx_.mem_dump_task_count_ == 0) { mem_ctx_.mem_dump_task_count_ = 1; } - mem_ctx_.table_data_desc_ = store_ctx_->table_data_desc_; - mem_ctx_.datum_utils_ = &(store_ctx_->ctx_->schema_.datum_utils_); - mem_ctx_.need_sort_ = param_->need_sort_; + + if (compact_ctx_->compact_config_->is_sort_lobid_) { + mem_ctx_.table_data_desc_ = store_ctx_->lob_id_table_data_desc_; + mem_ctx_.datum_utils_ = &(store_ctx_->ctx_->schema_.lob_meta_datum_utils_); + mem_ctx_.need_sort_ = true; + mem_ctx_.column_count_ = 1; + } else { + mem_ctx_.table_data_desc_ = store_ctx_->table_data_desc_; + mem_ctx_.datum_utils_ = &(store_ctx_->ctx_->schema_.datum_utils_); + mem_ctx_.need_sort_ = param_->need_sort_; + mem_ctx_.column_count_ = param_->column_count_; + } + mem_ctx_.mem_load_task_count_ = param_->session_count_; - mem_ctx_.column_count_ = param_->column_count_; mem_ctx_.dml_row_handler_ = store_ctx_->error_row_handler_; mem_ctx_.file_mgr_ = store_ctx_->tmp_file_mgr_; mem_ctx_.dup_action_ = param_->dup_action_; @@ -331,26 +340,27 @@ int ObTableLoadMemCompactor::start() int ObTableLoadMemCompactor::construct_compactors() { int ret = OB_SUCCESS; - ObArray trans_store_array; - trans_store_array.set_tenant_id(MTL_ID()); - if (OB_FAIL(store_ctx_->get_committed_trans_stores(trans_store_array))) { - LOG_WARN("fail to get committed trans stores", KR(ret)); + ObArenaAllocator allocator("TLD_CompTables"); + ObArray table_array; + allocator.set_tenant_id(MTL_ID()); + table_array.set_block_allocator(ModulePageAllocator(allocator)); + if (OB_FAIL(compact_ctx_->compact_config_->get_tables(table_array, allocator))) { + LOG_WARN("fail to get tables", KR(ret)); } - for (int64_t i = 0; OB_SUCC(ret) && i < trans_store_array.count(); ++i) { - ObTableLoadTransStore *trans_store = trans_store_array.at(i); - for (int64_t j = 0; OB_SUCC(ret) && j < trans_store->session_store_array_.count(); ++j) { - const ObTableLoadTransStore::SessionStore *session_store = trans_store->session_store_array_.at(j); - for (int64_t k = 0; OB_SUCC(ret) && k < session_store->partition_table_array_.count(); ++k) { - ObIDirectLoadPartitionTable *table = session_store->partition_table_array_.at(k); - if (OB_FAIL(add_tablet_table(table))) { - LOG_WARN("fail to add tablet table", KR(ret)); - } - } + for (int64_t i = 0; OB_SUCC(ret) && i < table_array.count(); ++i) { + ObIDirectLoadPartitionTable *table = table_array.at(i); + if (OB_FAIL(add_tablet_table(table))) { + LOG_WARN("fail to add tablet table", KR(ret)); } } - if (OB_SUCC(ret)) { - store_ctx_->clear_committed_trans_stores(); + // release tables + for (int64_t i = 0; i < table_array.count(); ++i) { + ObIDirectLoadPartitionTable *table = table_array.at(i); + table->~ObIDirectLoadPartitionTable(); + allocator.free(table); + table = nullptr; } + table_array.reset(); return ret; } @@ -585,7 +595,7 @@ int ObTableLoadMemCompactor::build_result_for_heap_table() ObIDirectLoadPartitionTable *table = mem_ctx_.tables_.at(i); ObDirectLoadExternalTable *external_table = nullptr; ObDirectLoadExternalTable *copied_external_table = nullptr; - if (OB_ISNULL(external_table = dynamic_cast(table))) { + if (OB_ISNULL(external_table = static_cast(table))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected table", KR(ret), K(i), KPC(table)); } else if (OB_ISNULL(copied_external_table = @@ -617,7 +627,7 @@ int ObTableLoadMemCompactor::add_table_to_parallel_merge_ctx() for (int64_t i = 0; OB_SUCC(ret) && i < mem_ctx_.tables_.count(); i++) { ObIDirectLoadPartitionTable *table = mem_ctx_.tables_.at(i); ObDirectLoadMultipleSSTable *sstable = nullptr; - if (OB_ISNULL(sstable = dynamic_cast(table))) { + if (OB_ISNULL(sstable = static_cast(table))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected table", KR(ret), K(i), KPC(table)); } else if (OB_FAIL(parallel_merge_ctx_.add_tablet_sstable(sstable))) { diff --git a/src/observer/table_load/ob_table_load_merger.cpp b/src/observer/table_load/ob_table_load_merger.cpp index dddedf1649..6aae976309 100644 --- a/src/observer/table_load/ob_table_load_merger.cpp +++ b/src/observer/table_load/ob_table_load_merger.cpp @@ -174,6 +174,74 @@ private: ObTableLoadMerger *const merger_; }; +class ObTableLoadMerger::DelLobTaskProcessor : public ObITableLoadTaskProcessor +{ +public: + DelLobTaskProcessor(ObTableLoadTask &task, ObTableLoadTableCtx *ctx, ObTableLoadMerger *merger) + : ObITableLoadTaskProcessor(task), ctx_(ctx), merger_(merger) + { + ctx_->inc_ref_count(); + } + virtual ~DelLobTaskProcessor() + { + ObTableLoadService::put_ctx(ctx_); + } + int process() override + { + int ret = OB_SUCCESS; + ObDirectLoadPartitionDelLobTask *del_lob_task = nullptr; + while (OB_SUCC(ret)) { + del_lob_task = nullptr; + if (OB_FAIL(merger_->get_next_del_lob_task(del_lob_task))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("fail to get next del lob task", KR(ret)); + } else { + ret = OB_SUCCESS; + break; + } + } else if (OB_FAIL(del_lob_task->process())) { + LOG_WARN("fail to process del lob task", KR(ret)); + } + if (nullptr != del_lob_task) { + merger_->handle_del_lob_task_finish(del_lob_task); + } + } + return ret; + } +private: + ObTableLoadTableCtx *const ctx_; + ObTableLoadMerger *const merger_; +}; + +class ObTableLoadMerger::DelLobTaskCallback : public ObITableLoadTaskCallback +{ +public: + DelLobTaskCallback(ObTableLoadTableCtx *ctx, ObTableLoadMerger *merger) + : ctx_(ctx), merger_(merger) + { + ctx_->inc_ref_count(); + } + virtual ~DelLobTaskCallback() + { + ObTableLoadService::put_ctx(ctx_); + } + void callback(int ret_code, ObTableLoadTask *task) override + { + int ret = OB_SUCCESS; + if (OB_FAIL(merger_->handle_del_lob_thread_finish(ret_code))) { + LOG_WARN("fail to handle del lob thread finish", KR(ret)); + } + if (OB_FAIL(ret)) { + ctx_->store_ctx_->set_status_error(ret); + } + ctx_->free_task(task); + OB_TABLE_LOAD_STATISTICS_PRINT_AND_RESET(); + } +private: + ObTableLoadTableCtx *const ctx_; + ObTableLoadMerger *const merger_; +}; + /** * ObTableLoadMerger */ @@ -201,7 +269,11 @@ int ObTableLoadMerger::init() ret = OB_INIT_TWICE; LOG_WARN("ObTableLoadMerger init twice", KR(ret), KP(this)); } else { - if (OB_FAIL(table_compact_ctx_.init(store_ctx_, *this))) { + if (OB_FAIL(table_compact_config_.init(store_ctx_, *this))) { + LOG_WARN("fail to init table_compact_config_", KR(ret)); + } else if (OB_FAIL(lob_id_compact_config_.init(*this))) { + LOG_WARN("fail to init lob id compact config", KR(ret)); + } else if (OB_FAIL(table_compact_ctx_.init(store_ctx_, &table_compact_config_))) { LOG_WARN("fail to init table compact ctx", KR(ret)); } else { is_inited_ = true; @@ -238,16 +310,21 @@ void ObTableLoadMerger::stop() is_stop_ = true; // 停止table合并 table_compact_ctx_.stop(); + lob_id_compact_ctx_.stop(); // 遍历合并中的任务队列, 调用stop ObMutexGuard guard(mutex_); ObDirectLoadPartitionMergeTask *merge_task = nullptr; ObDirectLoadPartitionRescanTask *rescan_task = nullptr; + ObDirectLoadPartitionDelLobTask *del_lob_task = nullptr; DLIST_FOREACH_NORET(merge_task, merging_list_) { merge_task->stop(); } DLIST_FOREACH_NORET(rescan_task, rescan_list_) { rescan_task->stop(); } + DLIST_FOREACH_NORET(del_lob_task, del_lob_list_) { + del_lob_task->stop(); + } } int ObTableLoadMerger::handle_table_compact_success() @@ -264,26 +341,47 @@ int ObTableLoadMerger::handle_table_compact_success() return ret; } +int ObTableLoadMerger::handle_lob_id_compact_success() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(is_stop_)) { + } else { + bool need_del_lob = true; + if (OB_FAIL(build_del_lob_ctx(need_del_lob))) { + LOG_WARN("fail to build del lob ctx", KR(ret)); + } else if (need_del_lob && OB_FAIL(start_del_lob())) { + LOG_WARN("fail to start del lob", KR(ret)); + } + } + return ret; +} + int ObTableLoadMerger::build_merge_ctx() { int ret = OB_SUCCESS; ObDirectLoadMergeParam merge_param; merge_param.table_id_ = param_.table_id_; + merge_param.lob_meta_table_id_ = store_ctx_->ctx_->schema_.lob_meta_table_id_; merge_param.target_table_id_ = store_ctx_->ctx_->ddl_param_.dest_table_id_; merge_param.rowkey_column_num_ = store_ctx_->ctx_->schema_.rowkey_column_count_; merge_param.store_column_count_ = store_ctx_->ctx_->schema_.store_column_count_; merge_param.fill_cg_thread_cnt_ = param_.session_count_; + merge_param.lob_column_idxs_ = &(store_ctx_->ctx_->schema_.lob_column_idxs_); merge_param.table_data_desc_ = store_ctx_->table_data_desc_; merge_param.datum_utils_ = &(store_ctx_->ctx_->schema_.datum_utils_); + merge_param.lob_column_idxs_ = &(store_ctx_->ctx_->schema_.lob_column_idxs_); merge_param.col_descs_ = &(store_ctx_->ctx_->schema_.column_descs_); + merge_param.lob_id_table_data_desc_ = store_ctx_->lob_id_table_data_desc_; + merge_param.lob_meta_datum_utils_ = &(store_ctx_->ctx_->schema_.lob_meta_datum_utils_); + merge_param.lob_meta_col_descs_ = &(store_ctx_->ctx_->schema_.lob_meta_column_descs_); merge_param.is_heap_table_ = store_ctx_->ctx_->schema_.is_heap_table_; merge_param.is_fast_heap_table_ = store_ctx_->is_fast_heap_table_; merge_param.is_incremental_ = ObDirectLoadMethod::is_incremental(param_.method_); merge_param.insert_mode_ = param_.insert_mode_; merge_param.insert_table_ctx_ = store_ctx_->insert_table_ctx_; merge_param.dml_row_handler_ = store_ctx_->error_row_handler_; - if (OB_FAIL(merge_ctx_.init(store_ctx_->ctx_, merge_param, store_ctx_->ls_partition_ids_, - store_ctx_->target_ls_partition_ids_))) { + merge_param.file_mgr_ = store_ctx_->tmp_file_mgr_; + if (OB_FAIL(merge_ctx_.init(store_ctx_->ctx_, merge_param, store_ctx_->ls_partition_ids_))) { LOG_WARN("fail to init merge ctx", KR(ret)); } else if (store_ctx_->is_multiple_mode_) { const ObIArray &tablet_merge_ctxs = @@ -314,7 +412,7 @@ int ObTableLoadMerger::build_merge_ctx() multiple_sstable_array.set_tenant_id(MTL_ID()); for (int64_t i = 0; OB_SUCC(ret) && i < table_array->count(); ++i) { ObDirectLoadMultipleSSTable *sstable = nullptr; - if (OB_ISNULL(sstable = dynamic_cast(table_array->at(i)))) { + if (OB_ISNULL(sstable = static_cast(table_array->at(i)))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected table", KR(ret), K(i), K(table_array)); } else if (OB_FAIL(multiple_sstable_array.push_back(sstable))) { @@ -410,7 +508,75 @@ int ObTableLoadMerger::build_rescan_ctx() } } return ret; +} +int ObTableLoadMerger::build_del_lob_ctx(bool &need_del_lob) +{ + int ret = OB_SUCCESS; + need_del_lob = true; + const ObIArray &tablet_merge_ctxs = + merge_ctx_.get_tablet_merge_ctxs(); + ObArray empty_table_array; + ObIArray *table_array = nullptr; + if (lob_id_compact_ctx_.result_.tablet_result_map_.size() > 0) { + abort_unless(1 == lob_id_compact_ctx_.result_.tablet_result_map_.size()); + ObTableLoadTableCompactResult::TabletResultMap::Iterator iter( + lob_id_compact_ctx_.result_.tablet_result_map_); + ObTableLoadTableCompactTabletResult *tablet_result = nullptr; + if (OB_ISNULL(tablet_result = iter.next(tablet_result))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null tablet result", KR(ret)); + } else { + table_array = &tablet_result->table_array_; + } + if (OB_NOT_NULL(tablet_result)) { + lob_id_compact_ctx_.result_.tablet_result_map_.revert(tablet_result); + } + } else { + table_array = &empty_table_array; + } + if (OB_SUCC(ret)) { + if (table_array->empty()) { + FLOG_INFO("LOAD NO NEED DEL LOB"); + need_del_lob = false; + if (OB_FAIL(store_ctx_->set_status_merged())) { + LOG_WARN("fail to set status", K(ret)); + } + } else { + ObArray multiple_sstable_array; + ObDirectLoadMultipleMergeRangeSplitter range_splitter; + multiple_sstable_array.set_tenant_id(MTL_ID()); + for (int64_t i = 0; OB_SUCC(ret) && i < table_array->count(); ++i) { + ObDirectLoadMultipleSSTable *sstable = nullptr; + if (OB_ISNULL(sstable = static_cast(table_array->at(i)))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected table", KR(ret), K(i), K(table_array)); + } else if (OB_FAIL(multiple_sstable_array.push_back(sstable))) { + LOG_WARN("fail to push back sstable", KR(ret)); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(range_splitter.init(multiple_sstable_array, store_ctx_->lob_id_table_data_desc_, + &(store_ctx_->ctx_->schema_.lob_meta_datum_utils_), + store_ctx_->ctx_->schema_.lob_meta_column_descs_))) { + LOG_WARN("fail to init range splitter", KR(ret)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < tablet_merge_ctxs.count(); ++i) { + ObDirectLoadTabletMergeCtx *tablet_merge_ctx = tablet_merge_ctxs.at(i); + if (OB_FAIL(tablet_merge_ctx->build_del_lob_task( + multiple_sstable_array, range_splitter, param_.session_count_))) { + LOG_WARN("fail to build merge task for multiple pk table", KR(ret)); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(del_lob_task_iter_.init(&merge_ctx_))) { + LOG_WARN("fail to build del lob task", KR(ret)); + } + } + } + } + return ret; } int ObTableLoadMerger::start_merge() @@ -495,21 +661,43 @@ int ObTableLoadMerger::start_rescan() return ret; } -int ObTableLoadMerger::get_next_rescan_task(ObDirectLoadPartitionRescanTask *&rescan_task) +int ObTableLoadMerger::start_del_lob() { int ret = OB_SUCCESS; - rescan_task = nullptr; - if (OB_UNLIKELY(is_stop_ || has_error_)) { - ret = OB_ITER_END; + const int64_t thread_count = store_ctx_->task_scheduler_->get_thread_count(); + ObTableLoadTableCtx *ctx = store_ctx_->ctx_; + if (OB_UNLIKELY(0 != running_thread_count_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected running thread count not zero", KR(ret), K(running_thread_count_)); } else { - ObMutexGuard guard(mutex_); - if (OB_FAIL(rescan_task_iter_.get_next_task(rescan_task))) { - if (OB_UNLIKELY(OB_ITER_END != ret)) { - LOG_WARN("fail to get next task", KR(ret)); - } - } else { - OB_ASSERT(rescan_list_.add_last(rescan_task)); + running_thread_count_ = thread_count; + } + for (int32_t thread_idx = 0; OB_SUCC(ret) && thread_idx < thread_count; ++thread_idx) { + ObTableLoadTask *task = nullptr; + // 1. 分配task + if (OB_FAIL(ctx->alloc_task(task))) { + LOG_WARN("fail to alloc task", KR(ret)); } + // 2. 设置processor + else if (OB_FAIL(task->set_processor(ctx, this))) { + LOG_WARN("fail to set del lob task processor", KR(ret)); + } + // 3. 设置callback + else if (OB_FAIL(task->set_callback(ctx, this))) { + LOG_WARN("fail to set del lob task callback", KR(ret)); + } + // 4. 把task放入调度器 + else if (OB_FAIL(store_ctx_->task_scheduler_->add_task(thread_idx, task))) { + LOG_WARN("fail to add task", KR(ret), K(thread_idx), KPC(task)); + } + if (OB_FAIL(ret)) { + if (nullptr != task) { + ctx->free_task(task); + } + } + } + if (OB_FAIL(ret)) { + has_error_ = true; } return ret; } @@ -527,7 +715,45 @@ int ObTableLoadMerger::get_next_merge_task(ObDirectLoadPartitionMergeTask *&merg LOG_WARN("fail to get next merger", KR(ret)); } } else { - OB_ASSERT(merging_list_.add_last(merge_task)); + abort_unless(merging_list_.add_last(merge_task)); + } + } + return ret; +} + +int ObTableLoadMerger::get_next_rescan_task(ObDirectLoadPartitionRescanTask *&rescan_task) +{ + int ret = OB_SUCCESS; + rescan_task = nullptr; + if (OB_UNLIKELY(is_stop_ || has_error_)) { + ret = OB_ITER_END; + } else { + ObMutexGuard guard(mutex_); + if (OB_FAIL(rescan_task_iter_.get_next_task(rescan_task))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("fail to get next task", KR(ret)); + } + } else { + abort_unless(rescan_list_.add_last(rescan_task)); + } + } + return ret; +} + +int ObTableLoadMerger::get_next_del_lob_task(ObDirectLoadPartitionDelLobTask *&del_lob_task) +{ + int ret = OB_SUCCESS; + del_lob_task = nullptr; + if (OB_UNLIKELY(is_stop_ || has_error_)) { + ret = OB_ITER_END; + } else { + ObMutexGuard guard(mutex_); + if (OB_FAIL(del_lob_task_iter_.get_next_task(del_lob_task))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("fail to get next task", KR(ret)); + } + } else { + abort_unless(del_lob_list_.add_last(del_lob_task)); } } return ret; @@ -536,13 +762,19 @@ int ObTableLoadMerger::get_next_merge_task(ObDirectLoadPartitionMergeTask *&merg void ObTableLoadMerger::handle_merge_task_finish(ObDirectLoadPartitionMergeTask *&merge_task) { ObMutexGuard guard(mutex_); - OB_ASSERT(OB_NOT_NULL(merging_list_.remove(merge_task))); + abort_unless(OB_NOT_NULL(merging_list_.remove(merge_task))); } void ObTableLoadMerger::handle_rescan_task_finish(ObDirectLoadPartitionRescanTask *&rescan_task) { ObMutexGuard guard(mutex_); - OB_ASSERT(OB_NOT_NULL(rescan_list_.remove(rescan_task))); + abort_unless(OB_NOT_NULL(rescan_list_.remove(rescan_task))); +} + +void ObTableLoadMerger::handle_del_lob_task_finish(ObDirectLoadPartitionDelLobTask *&del_lob_task) +{ + ObMutexGuard guard(mutex_); + abort_unless(OB_NOT_NULL(del_lob_list_.remove(del_lob_task))); } int ObTableLoadMerger::handle_merge_thread_finish(int ret_code) @@ -555,18 +787,24 @@ int ObTableLoadMerger::handle_merge_thread_finish(int ret_code) if (0 == running_thread_count) { if (OB_UNLIKELY(is_stop_ || has_error_)) { } else { - LOG_INFO("LOAD MERGE COMPLETED"); + FLOG_INFO("LOAD MERGE COMPLETED"); // release tmpfile // TODO(suzhi.yt) release all tables and merge tasks - if (!store_ctx_->is_fast_heap_table_) { - table_compact_ctx_.result_.release_all_table_data(); - } + table_compact_ctx_.result_.release_all_table_data(); if (store_ctx_->insert_table_ctx_->need_rescan()) { if (OB_FAIL(build_rescan_ctx())) { LOG_WARN("fail to build rescan ctx", KR(ret)); } else if (OB_FAIL(start_rescan())) { LOG_WARN("fail to start rescan", KR(ret)); } + } else if (store_ctx_->insert_table_ctx_->need_del_lob()) { + if (OB_FAIL(merge_ctx_.close_table_builder())) { + LOG_WARN("fail to close table builder", KR(ret)); + } else if (OB_FAIL(lob_id_compact_ctx_.init(store_ctx_, &lob_id_compact_config_))) { + LOG_WARN("fail to init lob compact", KR(ret)); + } else if (OB_FAIL(lob_id_compact_ctx_.start())) { + LOG_WARN("fail to start lob compact", KR(ret)); + } } else { if (OB_FAIL(store_ctx_->set_status_merged())) { LOG_WARN("fail to set store status merged", KR(ret)); @@ -587,7 +825,27 @@ int ObTableLoadMerger::handle_rescan_thread_finish(const int ret_code) if (0 == running_thread_count) { if (OB_UNLIKELY(is_stop_ || has_error_)) { } else { - LOG_INFO("LOAD RESCAN COMPLETED"); + FLOG_INFO("LOAD RESCAN COMPLETED"); + if (OB_FAIL(store_ctx_->set_status_merged())) { + LOG_WARN("fail to set store status merged", KR(ret)); + } + } + } + return ret; +} + +int ObTableLoadMerger::handle_del_lob_thread_finish(int ret_code) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ret_code)) { + has_error_ = true; + } + const int64_t running_thread_count = ATOMIC_SAF(&running_thread_count_, 1); + if (0 == running_thread_count) { + if (OB_UNLIKELY(is_stop_ || has_error_)) { + } else { + FLOG_INFO("LOAD DEL LOB COMPLETED"); + lob_id_compact_ctx_.result_.release_all_table_data(); if (OB_FAIL(store_ctx_->set_status_merged())) { LOG_WARN("fail to set store status merged", KR(ret)); } diff --git a/src/observer/table_load/ob_table_load_merger.h b/src/observer/table_load/ob_table_load_merger.h index a206b9bd47..ecb03fbdb5 100644 --- a/src/observer/table_load/ob_table_load_merger.h +++ b/src/observer/table_load/ob_table_load_merger.h @@ -18,6 +18,7 @@ #include "share/table/ob_table_load_define.h" #include "storage/direct_load/ob_direct_load_merge_ctx.h" #include "storage/direct_load/ob_direct_load_merge_task_iterator.h" +#include "storage/direct_load/ob_direct_load_partition_del_lob_task.h" #include "storage/direct_load/ob_direct_load_partition_rescan_task.h" #include "storage/direct_load/ob_direct_load_partition_merge_task.h" @@ -32,8 +33,10 @@ class ObTableLoadMerger { class MergeTaskProcessor; class RescanTaskProcessor; + class DelLobTaskProcessor; class MergeTaskCallback; class RescanTaskCallback; + class DelLobTaskCallback; public: ObTableLoadMerger(ObTableLoadStoreCtx *store_ctx); ~ObTableLoadMerger(); @@ -41,27 +44,39 @@ public: int start(); void stop(); int handle_table_compact_success(); + int handle_lob_id_compact_success(); + storage::ObDirectLoadMergeCtx& get_merge_ctx() { return merge_ctx_; } private: int build_merge_ctx(); int build_rescan_ctx(); + int build_del_lob_ctx(bool &need_del_lob); int start_merge(); int start_rescan(); + int start_del_lob(); int get_next_merge_task(storage::ObDirectLoadPartitionMergeTask *&merge_task); int get_next_rescan_task(ObDirectLoadPartitionRescanTask *&rescan_task); + int get_next_del_lob_task(ObDirectLoadPartitionDelLobTask *&del_lob_task); void handle_merge_task_finish(storage::ObDirectLoadPartitionMergeTask *&merge_task); int handle_merge_thread_finish(int ret_code); void handle_rescan_task_finish(ObDirectLoadPartitionRescanTask *&rescan_task); int handle_rescan_thread_finish(const int ret_code); + void handle_del_lob_task_finish(ObDirectLoadPartitionDelLobTask *&del_lob_task); + int handle_del_lob_thread_finish(int ret_code); private: ObTableLoadStoreCtx * const store_ctx_; const ObTableLoadParam ¶m_; + ObTableLoadTableCompactConfigMainTable table_compact_config_; + ObTableLoadTableCompactConfigLobIdTable lob_id_compact_config_; ObTableLoadTableCompactCtx table_compact_ctx_; + ObTableLoadTableCompactCtx lob_id_compact_ctx_; storage::ObDirectLoadMergeCtx merge_ctx_; mutable lib::ObMutex mutex_; ObDirectLoadMergeTaskIterator merge_task_iter_; ObDirectLoadRescanTaskIterator rescan_task_iter_; + ObDirectLoadDelLobTaskIterator del_lob_task_iter_; common::ObDList merging_list_; common::ObDList rescan_list_; + common::ObDList del_lob_list_; int64_t running_thread_count_ CACHE_ALIGNED; volatile bool has_error_; volatile bool is_stop_; diff --git a/src/observer/table_load/ob_table_load_schema.cpp b/src/observer/table_load/ob_table_load_schema.cpp index 5c4b7f50f6..7a26f23a16 100644 --- a/src/observer/table_load/ob_table_load_schema.cpp +++ b/src/observer/table_load/ob_table_load_schema.cpp @@ -16,6 +16,7 @@ #include "observer/table_load/ob_table_load_utils.h" #include "share/rc/ob_tenant_base.h" #include "share/schema/ob_multi_version_schema_service.h" +#include "storage/lob/ob_lob_meta.h" namespace oceanbase { @@ -27,6 +28,7 @@ using namespace share::schema; using namespace table; using namespace blocksstable; using namespace sql; +using namespace storage; int ObTableLoadSchema::get_schema_guard(uint64_t tenant_id, ObSchemaGetterGuard &schema_guard) { @@ -303,29 +305,6 @@ int ObTableLoadSchema::get_tenant_optimizer_gather_stats_on_load(const uint64_t return ret; } -int ObTableLoadSchema::get_lob_meta_tid( - const uint64_t tenant_id, - const uint64_t data_table_id, - uint64_t &lob_meta_table_id) -{ - int ret = OB_SUCCESS; - ObSchemaGetterGuard schema_guard; - const ObTableSchema *data_table_schema = nullptr; - lob_meta_table_id = OB_INVALID_ID; - if (OB_FAIL(ObTableLoadSchema::get_table_schema( - tenant_id, data_table_id, schema_guard, data_table_schema))) { - LOG_WARN("failed to get table schema", KR(ret), K(tenant_id), K(data_table_id)); - } else if (OB_ISNULL(data_table_schema)) { - ret = OB_TABLE_NOT_EXIST; - LOG_WARN("data table not exist", KR(ret), K(tenant_id), K(data_table_id)); - } else if (!data_table_schema->has_lob_aux_table()) { - // bypass - } else { - lob_meta_table_id = data_table_schema->get_aux_lob_meta_tid(); - } - return ret; -} - int ObTableLoadSchema::check_has_invisible_column(const ObTableSchema *table_schema, bool &bret) { int ret = OB_SUCCESS; @@ -385,15 +364,17 @@ ObTableLoadSchema::ObTableLoadSchema() has_identity_column_(false), rowkey_column_count_(0), store_column_count_(0), - lob_column_cnt_(0), collation_type_(CS_TYPE_INVALID), part_level_(PARTITION_LEVEL_ZERO), schema_version_(0), + lob_meta_table_id_(OB_INVALID_ID), is_inited_(false) { allocator_.set_tenant_id(MTL_ID()); + lob_column_idxs_.set_block_allocator(ModulePageAllocator(allocator_)); column_descs_.set_block_allocator(ModulePageAllocator(allocator_)); multi_version_column_descs_.set_block_allocator(ModulePageAllocator(allocator_)); + lob_meta_column_descs_.set_block_allocator(ModulePageAllocator(allocator_)); } ObTableLoadSchema::~ObTableLoadSchema() @@ -411,13 +392,16 @@ void ObTableLoadSchema::reset() has_identity_column_ = false; rowkey_column_count_ = 0; store_column_count_ = 0; - lob_column_cnt_ = 0; collation_type_ = CS_TYPE_INVALID; part_level_ = PARTITION_LEVEL_ZERO; schema_version_ = 0; + lob_meta_table_id_ = OB_INVALID_ID; + lob_column_idxs_.reset(); column_descs_.reset(); multi_version_column_descs_.reset(); datum_utils_.reset(); + lob_meta_column_descs_.reset(); + lob_meta_datum_utils_.reset(); cmp_funcs_.reset(); partition_ids_.reset(); allocator_.reset(); @@ -459,6 +443,9 @@ int ObTableLoadSchema::init_table_schema(const ObTableSchema *table_schema) collation_type_ = table_schema->get_collation_type(); part_level_ = table_schema->get_part_level(); schema_version_ = table_schema->get_schema_version(); + if (table_schema->has_lob_aux_table()) { + lob_meta_table_id_ = table_schema->get_aux_lob_meta_tid(); + } if (OB_FAIL(ObTableLoadUtils::deep_copy(table_schema->get_table_name_str(), table_name_, allocator_))) { LOG_WARN("fail to deep copy table name", KR(ret)); @@ -477,6 +464,8 @@ int ObTableLoadSchema::init_table_schema(const ObTableSchema *table_schema) LOG_WARN("fail to init datum utils", KR(ret)); } else if (OB_FAIL(init_lob_storage(column_descs_))) { LOG_WARN("fail to check lob storage", KR(ret)); + } else if (OB_FAIL(gen_lob_meta_datum_utils())) { + LOG_WARN("fail to gen lob meta datum utils", KR(ret)); } else if (OB_FAIL(init_cmp_funcs(column_descs_, lib::is_oracle_mode()))) { LOG_WARN("fail to init cmp funcs", KR(ret)); } @@ -517,14 +506,17 @@ int ObTableLoadSchema::init_table_schema(const ObTableSchema *table_schema) int ObTableLoadSchema::init_lob_storage(common::ObIArray &column_descs) { int ret = OB_SUCCESS; - lob_column_cnt_ = 0; for (int64_t i = 0; OB_SUCC(ret) && i < column_descs.count(); ++i) { const ObColDesc &col_desc = column_descs.at(i); if (col_desc.col_type_.is_lob_storage()) { column_descs.at(i).col_type_.set_has_lob_header(); - ++lob_column_cnt_; + if (OB_FAIL(lob_column_idxs_.push_back(i))) { + LOG_WARN("fail to push back", KR(ret)); + } } } + LOG_INFO("ObTableLoadSchema::init_lob_storage", K(lob_column_idxs_)); + return ret; } @@ -604,5 +596,26 @@ int ObTableLoadSchema::prepare_col_desc(const ObTableSchema *table_schema, commo return ret; } +int ObTableLoadSchema::gen_lob_meta_datum_utils() +{ + int ret = OB_SUCCESS; + ObColDesc col_desc1; + col_desc1.col_id_ = ObLobMetaUtil::LOB_ID_COL_ID; + col_desc1.col_type_.set_varbinary(); + col_desc1.col_order_ = ObOrderType::ASC; + ObColDesc col_desc2; + col_desc2.col_id_ = ObLobMetaUtil::SEQ_ID_COL_ID; + col_desc2.col_type_.set_varbinary(); + col_desc2.col_order_ = ObOrderType::ASC; + if (OB_FAIL(lob_meta_column_descs_.push_back(col_desc1))) { + LOG_WARN("fail to push back col_desc", KR(ret)); + } else if (OB_FAIL(lob_meta_column_descs_.push_back(col_desc2))) { + LOG_WARN("fail to push back col_desc", KR(ret)); + } else if (OB_FAIL(lob_meta_datum_utils_.init(lob_meta_column_descs_, ObLobMetaUtil::LOB_META_SCHEMA_ROWKEY_COL_CNT, false, allocator_))) { + LOG_WARN("fail to init", KR(ret)); + } + return ret; +} + } // namespace observer } // namespace oceanbase diff --git a/src/observer/table_load/ob_table_load_schema.h b/src/observer/table_load/ob_table_load_schema.h index 1c75a1e1d9..1cda10c530 100644 --- a/src/observer/table_load/ob_table_load_schema.h +++ b/src/observer/table_load/ob_table_load_schema.h @@ -59,9 +59,6 @@ public: bool contain_hidden_pk_column = false); static int check_has_udt_column(const share::schema::ObTableSchema *table_schema, bool &bret); static int get_tenant_optimizer_gather_stats_on_load(const uint64_t tenant_id, bool &value); - static int get_lob_meta_tid(const uint64_t tenant_id, - const uint64_t data_table_id, - uint64_t &lob_meta_table_id); static int check_has_invisible_column(const share::schema::ObTableSchema *table_schema, bool &bret); static int check_has_unused_column(const share::schema::ObTableSchema *table_schema, bool &bret); static int get_table_compressor_type(uint64_t tenant_id, uint64_t table_id, @@ -84,6 +81,7 @@ private: common::ObIArray &cols_desc); int prepare_col_desc(const ObTableSchema *table_schema, common::ObIArray &col_descs); + int gen_lob_meta_datum_utils(); public: common::ObArenaAllocator allocator_; common::ObString table_name_; @@ -95,15 +93,18 @@ public: int64_t rowkey_column_count_; // column count in store, does not contain virtual generated columns int64_t store_column_count_; - int64_t lob_column_cnt_; common::ObCollationType collation_type_; share::schema::ObPartitionLevel part_level_; int64_t schema_version_; + uint64_t lob_meta_table_id_; + common::ObArray lob_column_idxs_; // if it is a heap table, it contains hidden primary key column // does not contain virtual generated columns common::ObArray column_descs_; common::ObArray multi_version_column_descs_; blocksstable::ObStorageDatumUtils datum_utils_; + common::ObArray lob_meta_column_descs_; + blocksstable::ObStorageDatumUtils lob_meta_datum_utils_; blocksstable::ObStoreCmpFuncs cmp_funcs_; // for sql statistics table::ObTableLoadArray partition_ids_; bool is_inited_; diff --git a/src/observer/table_load/ob_table_load_service.cpp b/src/observer/table_load/ob_table_load_service.cpp index 892653f336..c84bf56638 100644 --- a/src/observer/table_load/ob_table_load_service.cpp +++ b/src/observer/table_load/ob_table_load_service.cpp @@ -576,10 +576,10 @@ int ObTableLoadService::check_support_direct_load(ObSchemaGetterGuard &schema_gu FORWARD_USER_ERROR_MSG(ret, "direct-load does not support table with materialized view log"); } else if (ObDirectLoadMethod::is_incremental(method)) { // incremental direct-load uint64_t compat_version = 0; - if (OB_UNLIKELY(ObDirectLoadInsertMode::INC_REPLACE != insert_mode)) { + if (!ObDirectLoadInsertMode::is_valid_for_incremental_method(insert_mode)) { ret = OB_NOT_SUPPORTED; - LOG_WARN("using incremental direct-load without inc_replace is not supported", KR(ret)); - FORWARD_USER_ERROR_MSG(ret, "using incremental direct-load without inc_replace is not supported"); + LOG_WARN("using incremental direct-load without inc_replace or normal is not supported", KR(ret)); + FORWARD_USER_ERROR_MSG(ret, "using incremental direct-load without inc_replace or normal is not supported"); } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) { LOG_WARN("fail to get data version", KR(ret), K(tenant_id)); } else if (compat_version < DATA_VERSION_4_3_1_0) { diff --git a/src/observer/table_load/ob_table_load_store_ctx.cpp b/src/observer/table_load/ob_table_load_store_ctx.cpp index f2b1fecd7e..0c5b5b3d1a 100644 --- a/src/observer/table_load/ob_table_load_store_ctx.cpp +++ b/src/observer/table_load/ob_table_load_store_ctx.cpp @@ -159,6 +159,12 @@ int ObTableLoadStoreCtx::init( ObDirectLoadSSTableScanMerge::MAX_SSTABLE_COUNT); table_data_desc_.max_mem_chunk_count_ = wa_mem_limit / ObDirectLoadExternalMultiPartitionRowChunk::MIN_MEMORY_LIMIT; } + if (OB_SUCC(ret)) { + lob_id_table_data_desc_ = table_data_desc_; + lob_id_table_data_desc_.rowkey_column_num_ = 1; + lob_id_table_data_desc_.column_count_ = 1; + lob_id_table_data_desc_.is_heap_table_ = false; + } } if (OB_FAIL(ret)) { } @@ -209,7 +215,7 @@ int ObTableLoadStoreCtx::init( insert_table_param.reserved_parallel_ = is_fast_heap_table_ ? ctx_->param_.session_count_ : 0; insert_table_param.rowkey_column_count_ = ctx_->schema_.rowkey_column_count_; insert_table_param.column_count_ = ctx_->schema_.store_column_count_; - insert_table_param.lob_column_count_ = ctx_->schema_.lob_column_cnt_; + insert_table_param.lob_column_count_ = ctx_->schema_.lob_column_idxs_.count(); insert_table_param.is_partitioned_table_ = ctx_->schema_.is_partitioned_table_; insert_table_param.is_heap_table_ = ctx_->schema_.is_heap_table_; insert_table_param.is_column_store_ = ctx_->schema_.is_column_store_; @@ -220,10 +226,6 @@ int ObTableLoadStoreCtx::init( insert_table_param.cmp_funcs_ = &(ctx_->schema_.cmp_funcs_); if (insert_table_param.is_incremental_ && OB_FAIL(init_trans_param(insert_table_param.trans_param_))) { LOG_WARN("fail to init trans param", KR(ret)); - } else if (OB_FAIL(ObTableLoadSchema::get_lob_meta_tid(ctx_->param_.tenant_id_, - insert_table_param.table_id_, insert_table_param.lob_meta_tid_))) { - LOG_WARN("fail to get lob meta tid", KR(ret), - K(ctx_->param_.tenant_id_), K(insert_table_param.table_id_)); } else if (OB_ISNULL(insert_table_ctx_ = OB_NEWx(ObDirectLoadInsertTableContext, (&allocator_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; diff --git a/src/observer/table_load/ob_table_load_store_ctx.h b/src/observer/table_load/ob_table_load_store_ctx.h index 2cacd09985..8362489e85 100644 --- a/src/observer/table_load/ob_table_load_store_ctx.h +++ b/src/observer/table_load/ob_table_load_store_ctx.h @@ -145,6 +145,7 @@ public: common::ObArray ls_partition_ids_; common::ObArray target_ls_partition_ids_; storage::ObDirectLoadTableDataDesc table_data_desc_; + storage::ObDirectLoadTableDataDesc lob_id_table_data_desc_; table::ObTableLoadResultInfo result_info_; ObITableLoadTaskScheduler *task_scheduler_; ObTableLoadMerger *merger_; diff --git a/src/observer/table_load/ob_table_load_table_compactor.cpp b/src/observer/table_load/ob_table_load_table_compactor.cpp index c291407f28..457a031919 100644 --- a/src/observer/table_load/ob_table_load_table_compactor.cpp +++ b/src/observer/table_load/ob_table_load_table_compactor.cpp @@ -21,6 +21,7 @@ #include "observer/table_load/ob_table_load_mem_compactor.h" #include "storage/direct_load/ob_direct_load_external_table.h" #include "observer/table_load/ob_table_load_multiple_heap_table_compactor.h" +#include "observer/table_load/ob_table_load_trans_store.h" #include "observer/table_load/ob_table_load_parallel_merge_table_compactor.h" namespace oceanbase @@ -107,12 +108,116 @@ void ObTableLoadTableCompactResult::release_all_table_data() } } +/** + * ObTableLoadTableCompactConfig + */ + +int ObTableLoadTableCompactConfigMainTable::handle_table_compact_success() +{ + // notify merger + return merger_->handle_table_compact_success(); +} + +int ObTableLoadTableCompactConfigMainTable::get_tables(common::ObIArray &table_array, + common::ObIAllocator &allocator) +{ + int ret = OB_SUCCESS; + ObArray trans_store_array; + trans_store_array.set_tenant_id(MTL_ID()); + if (OB_FAIL(store_ctx_->get_committed_trans_stores(trans_store_array))) { + LOG_WARN("fail to get committed trans stores", KR(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < trans_store_array.count(); ++i) { + ObTableLoadTransStore *trans_store = trans_store_array.at(i); + for (int64_t j = 0; OB_SUCC(ret) && j < trans_store->session_store_array_.count(); ++j) { + const ObTableLoadTransStore::SessionStore *session_store = trans_store->session_store_array_.at(j); + for (int64_t k = 0; OB_SUCC(ret) && k < session_store->partition_table_array_.count(); ++k) { + ObIDirectLoadPartitionTable *table = session_store->partition_table_array_.at(k); + ObDirectLoadExternalTable *external_table = nullptr; + ObDirectLoadExternalTable *copied_external_table = nullptr; + if (OB_ISNULL(external_table = dynamic_cast(table))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected table", KR(ret), K(i), KPC(table)); + } else if (OB_ISNULL(copied_external_table = + OB_NEWx(ObDirectLoadExternalTable, &allocator))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new external table", KR(ret)); + } else if (OB_FAIL(copied_external_table->copy(*external_table))) { + LOG_WARN("fail to copy external table", KR(ret)); + } else if (OB_FAIL(table_array.push_back(copied_external_table))) { + LOG_WARN("fail to add tablet table", KR(ret)); + } + if (OB_FAIL(ret)) { + if (nullptr != copied_external_table) { + copied_external_table->~ObDirectLoadExternalTable(); + copied_external_table = nullptr; + } + } + } + } + } + + if (OB_SUCC(ret)) { + store_ctx_->clear_committed_trans_stores(); + } + return ret; +} + +int ObTableLoadTableCompactConfigMainTable::init(ObTableLoadStoreCtx *store_ctx, ObTableLoadMerger &merger) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(nullptr == store_ctx)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(store_ctx)); + } else { + store_ctx_ = store_ctx; + merger_ = &merger; + } + return ret; +} + +ObTableLoadTableCompactConfigLobIdTable::ObTableLoadTableCompactConfigLobIdTable() : merger_(nullptr) +{ + +} + + +ObTableLoadTableCompactConfigLobIdTable::~ObTableLoadTableCompactConfigLobIdTable() +{ + +} +int ObTableLoadTableCompactConfigLobIdTable::init(ObTableLoadMerger &merger) +{ + int ret = OB_SUCCESS; + merger_ = &merger; + is_sort_lobid_ = true; + return ret; +} + +int ObTableLoadTableCompactConfigLobIdTable::handle_table_compact_success() +{ + // notify merger + return merger_->handle_lob_id_compact_success(); +} + +int ObTableLoadTableCompactConfigLobIdTable::get_tables(common::ObIArray &table_array, + common::ObIAllocator &allocator) +{ + int ret = OB_SUCCESS; + FOREACH_X(item, merger_->get_merge_ctx().get_table_builder_map(), OB_SUCC(ret)) { + if (OB_FAIL(item->second->get_tables(table_array, allocator))) { + LOG_WARN("fail to get tables", KR(ret)); + } + } + return ret; +} + /** * ObTableLoadTableCompactCtx */ ObTableLoadTableCompactCtx::ObTableLoadTableCompactCtx() - : store_ctx_(nullptr), merger_(nullptr) + : store_ctx_(nullptr), compact_config_(nullptr) { } @@ -120,18 +225,18 @@ ObTableLoadTableCompactCtx::~ObTableLoadTableCompactCtx() { } -int ObTableLoadTableCompactCtx::init(ObTableLoadStoreCtx *store_ctx, ObTableLoadMerger &merger) +int ObTableLoadTableCompactCtx::init(ObTableLoadStoreCtx *store_ctx, ObTableLoadTableCompactConfig *compact_config) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(nullptr == store_ctx)) { + if (OB_UNLIKELY(nullptr == store_ctx || nullptr == compact_config)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), KP(store_ctx)); + LOG_WARN("invalid args", KR(ret), KP(store_ctx), KP(compact_config)); } else { if (OB_FAIL(result_.init())) { LOG_WARN("fail to init result", KR(ret)); } else { store_ctx_ = store_ctx; - merger_ = &merger; + compact_config_ = compact_config; } } return ret; @@ -139,7 +244,7 @@ int ObTableLoadTableCompactCtx::init(ObTableLoadStoreCtx *store_ctx, ObTableLoad bool ObTableLoadTableCompactCtx::is_valid() const { - return nullptr != store_ctx_ && nullptr != merger_; + return nullptr != store_ctx_ && nullptr != compact_config_; } int ObTableLoadTableCompactCtx::new_compactor(ObTableLoadTableCompactorHandle &compactor_handle) @@ -148,20 +253,21 @@ int ObTableLoadTableCompactCtx::new_compactor(ObTableLoadTableCompactorHandle &c compactor_handle.reset(); ObTableLoadTableCompactor *compactor = nullptr; obsys::ObWLockGuard guard(rwlock_); - if (OB_UNLIKELY(compactor_handle_.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected not null compactor", KR(ret), K(compactor_handle_)); - } else { + { ObMemAttr attr(MTL_ID(), "TLD_Compactor"); - if (store_ctx_->is_multiple_mode_) { - if (store_ctx_->table_data_desc_.is_heap_table_) { - compactor = OB_NEW(ObTableLoadMultipleHeapTableCompactor, attr); - } else { - compactor = OB_NEW(ObTableLoadMemCompactor, attr); - } + if (compact_config_->is_sort_lobid_) { + compactor = OB_NEW(ObTableLoadMemCompactor, attr); } else { - // 有主键表不排序 - compactor = OB_NEW(ObTableLoadParallelMergeTableCompactor, attr); + if (store_ctx_->is_multiple_mode_) { + if (store_ctx_->table_data_desc_.is_heap_table_) { + compactor = OB_NEW(ObTableLoadMultipleHeapTableCompactor, attr); + } else { + compactor = OB_NEW(ObTableLoadMemCompactor, attr); + } + } else { + // 有主键表不排序 + compactor = OB_NEW(ObTableLoadParallelMergeTableCompactor, attr); + } } if (OB_ISNULL(compactor)) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -233,10 +339,17 @@ void ObTableLoadTableCompactCtx::stop() int ObTableLoadTableCompactCtx::handle_table_compact_success() { + int ret = OB_SUCCESS; // release compactor release_compactor(); - // notify merger - return merger_->handle_table_compact_success(); + + if (compact_config_ == nullptr) { + ret = OB_NOT_INIT; + LOG_WARN("compact_config_ is nullptr", KR(ret)); + } else if (OB_FAIL(compact_config_->handle_table_compact_success())) { + LOG_WARN("fail to handle_table_compact_success", KR(ret)); + } + return ret; } /** diff --git a/src/observer/table_load/ob_table_load_table_compactor.h b/src/observer/table_load/ob_table_load_table_compactor.h index 649790ccfa..2b879d767d 100644 --- a/src/observer/table_load/ob_table_load_table_compactor.h +++ b/src/observer/table_load/ob_table_load_table_compactor.h @@ -16,6 +16,7 @@ #include "lib/allocator/page_arena.h" #include "lib/hash/ob_link_hashmap.h" #include "storage/direct_load/ob_direct_load_i_table.h" +#include "storage/direct_load/ob_direct_load_external_multi_partition_table.h" namespace oceanbase { @@ -51,6 +52,45 @@ public: TabletResultMap tablet_result_map_; }; +class ObTableLoadTableCompactConfig +{ +public: + ObTableLoadTableCompactConfig() : is_sort_lobid_(false) {} + virtual int handle_table_compact_success() = 0; + virtual int get_tables(common::ObIArray &table_array, + common::ObIAllocator &allocator) = 0; + + bool is_sort_lobid_; +}; + +class ObTableLoadTableCompactConfigMainTable : public ObTableLoadTableCompactConfig +{ +public: + int init(ObTableLoadStoreCtx *store_ctx, ObTableLoadMerger &merger); + + int handle_table_compact_success() override; + int get_tables(common::ObIArray &table_array, + common::ObIAllocator &allocator) override; +private: + ObTableLoadStoreCtx *store_ctx_; + ObTableLoadMerger *merger_; +}; + +class ObTableLoadTableCompactConfigLobIdTable : public ObTableLoadTableCompactConfig +{ +public: + ObTableLoadTableCompactConfigLobIdTable(); + ~ObTableLoadTableCompactConfigLobIdTable(); + + int init(ObTableLoadMerger &merger); + + int handle_table_compact_success() override; + int get_tables(common::ObIArray &table_array, + common::ObIAllocator &allocator) override; +private: + ObTableLoadMerger *merger_; +}; + class ObTableLoadTableCompactor { public: @@ -104,12 +144,12 @@ class ObTableLoadTableCompactCtx public: ObTableLoadTableCompactCtx(); ~ObTableLoadTableCompactCtx(); - int init(ObTableLoadStoreCtx *store_ctx, ObTableLoadMerger &merger); + int init(ObTableLoadStoreCtx *store_ctx, ObTableLoadTableCompactConfig *compact_config); bool is_valid() const; int start(); void stop(); int handle_table_compact_success(); - TO_STRING_KV(KP_(store_ctx), KP_(merger), K_(compactor_handle)); + TO_STRING_KV(KP_(store_ctx), KP_(compact_config), K_(compactor_handle)); private: int new_compactor(ObTableLoadTableCompactorHandle &compactor_handle); void release_compactor(); @@ -117,7 +157,7 @@ private: public: ObTableLoadStoreCtx *store_ctx_; - ObTableLoadMerger *merger_; + ObTableLoadTableCompactConfig *compact_config_; mutable obsys::ObRWLock rwlock_; ObTableLoadTableCompactorHandle compactor_handle_; ObTableLoadTableCompactResult result_; diff --git a/src/observer/table_load/ob_table_load_trans_store.cpp b/src/observer/table_load/ob_table_load_trans_store.cpp index b24fb78282..7d1731b92a 100644 --- a/src/observer/table_load/ob_table_load_trans_store.cpp +++ b/src/observer/table_load/ob_table_load_trans_store.cpp @@ -121,8 +121,6 @@ ObTableLoadTransStoreWriter::ObTableLoadTransStoreWriter(ObTableLoadTransStore * table_data_desc_(nullptr), lob_inrow_threshold_(0), ref_count_(0), - is_incremental_(false), - is_inc_replace_(false), is_inited_(false) { allocator_.set_tenant_id(MTL_ID()); @@ -164,8 +162,6 @@ int ObTableLoadTransStoreWriter::init() } else if (OB_FAIL(init_column_schemas_and_lob_info())) { LOG_WARN("fail to init column schemas and lob info", KR(ret)); } else { - is_incremental_ = ObDirectLoadMethod::is_incremental(store_ctx_->ctx_->param_.method_); - is_inc_replace_ = (ObDirectLoadInsertMode::INC_REPLACE == store_ctx_->ctx_->param_.insert_mode_); is_inited_ = true; } } @@ -333,9 +329,7 @@ int ObTableLoadTransStoreWriter::px_write(const ObTabletID &tablet_id, const ObN for (int64_t i = 0; OB_SUCC(ret) && i < table_data_desc_->column_count_; ++i) { ObStorageDatum &datum = session_ctx.datum_row_.storage_datums_[i]; const ObObj &obj = row.cells_[i]; - if (OB_FAIL(check_support_obj(obj))) { - LOG_WARN("failed to check support obj", KR(ret), K(obj)); - } else if (OB_FAIL(datum.from_obj_enhance(obj))) { + if (OB_FAIL(datum.from_obj_enhance(obj))) { LOG_WARN("fail to from obj enhance", KR(ret), K(obj)); } } @@ -453,8 +447,6 @@ int ObTableLoadTransStoreWriter::cast_column( LOG_WARN("fail to cast obj and check", KR(ret), K(obj)); } if (OB_FAIL(ret)) { - } else if (OB_FAIL(check_support_obj(out_obj))) { - LOG_WARN("failed to check support obj", KR(ret), K(out_obj)); } else if (OB_FAIL(datum.from_obj_enhance(out_obj))) { LOG_WARN("fail to from obj enhance", KR(ret), K(out_obj)); } else if (column_schema->is_autoincrement()) { @@ -533,50 +525,5 @@ int ObTableLoadTransStoreWriter::write_row_to_table_store(ObDirectLoadTableStore return ret; } -static int check_lob_is_inrow(const ObObj &obj, const int64_t lob_inrow_threshold, bool &is_inrow) -{ - int ret = OB_SUCCESS; - is_inrow = false; - ObLobManager *lob_mngr = MTL(ObLobManager*); - if (OB_UNLIKELY(!obj.is_lob_storage() || lob_inrow_threshold < 0)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(obj), K(lob_inrow_threshold)); - } else if (OB_ISNULL(lob_mngr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to get lob manager handle.", K(ret)); - } else { - ObString data = obj.get_string(); - const bool set_has_lob_header = data.length() > 0; - ObLobLocatorV2 src(data, set_has_lob_header); - int64_t byte_len = 0; - if (OB_FAIL(src.get_lob_data_byte_len(byte_len))) { - LOG_WARN("fail to get lob data byte len", K(ret), K(src)); - } else if (src.has_inrow_data() && lob_mngr->can_write_inrow(byte_len, lob_inrow_threshold)) { - is_inrow = true; - } else { - is_inrow = false; - } - } - return ret; -} - -int ObTableLoadTransStoreWriter::check_support_obj(const ObObj &obj) -{ - int ret = OB_SUCCESS; - if (is_incremental_ && is_inc_replace_) { - if (obj.is_lob_storage()) { - bool is_inrow = false; - if (OB_FAIL(check_lob_is_inrow(obj, lob_inrow_threshold_, is_inrow))) { - LOG_WARN("fail to check lob is inrow", KR(ret)); - } else if (OB_UNLIKELY(!is_inrow)) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("incremental direct-load does not support outrow lob", KR(ret), K(obj)); - FORWARD_USER_ERROR_MSG(ret, "incremental direct-load does not support outrow lob"); - } - } - } - return ret; -} - } // namespace observer } // namespace oceanbase diff --git a/src/observer/table_load/ob_table_load_trans_store.h b/src/observer/table_load/ob_table_load_trans_store.h index f93ba92c81..86109184c1 100644 --- a/src/observer/table_load/ob_table_load_trans_store.h +++ b/src/observer/table_load/ob_table_load_trans_store.h @@ -105,7 +105,6 @@ private: const common::ObTabletID &tablet_id, const table::ObTableLoadSequenceNo &seq_no, const blocksstable::ObDatumRow &datum_row); - int check_support_obj(const common::ObObj &obj); private: ObTableLoadTransStore *const trans_store_; ObTableLoadTransCtx *const trans_ctx_; @@ -134,8 +133,6 @@ private: SessionContext *session_ctx_array_; int64_t lob_inrow_threshold_; // for incremental direct load int64_t ref_count_ CACHE_ALIGNED; - bool is_incremental_; - bool is_inc_replace_; bool is_inited_; ObSchemaGetterGuard schema_guard_; }; diff --git a/src/sql/engine/cmd/ob_load_data_direct_impl.cpp b/src/sql/engine/cmd/ob_load_data_direct_impl.cpp index d127e9a8e6..e8cdee16ab 100644 --- a/src/sql/engine/cmd/ob_load_data_direct_impl.cpp +++ b/src/sql/engine/cmd/ob_load_data_direct_impl.cpp @@ -1920,9 +1920,12 @@ int ObLoadDataDirectImpl::init_execute_param() : ObDirectLoadMethod::FULL); execute_param_.insert_mode_ = ObDirectLoadInsertMode::NORMAL; if (OB_UNLIKELY(direct_load_hint.is_inc_load_method())) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("inc load method not supported", KR(ret), K(direct_load_hint)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "inc load method in direct load is"); + if (OB_UNLIKELY(ObLoadDupActionType::LOAD_REPLACE == load_args.dupl_action_)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("replace for inc load method not supported", KR(ret), + K(direct_load_hint), K(load_args.dupl_action_)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "replace for inc load method in direct load is"); + } } else if (direct_load_hint.is_inc_replace_load_method()) { if (OB_UNLIKELY(ObLoadDupActionType::LOAD_STOP_ON_DUP != load_args.dupl_action_)) { ret = OB_NOT_SUPPORTED; diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 826dbac68f..64fa577468 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -726,8 +726,10 @@ ob_set_subtarget(ob_storage tablelock ob_set_subtarget(ob_storage direct_load direct_load/ob_direct_load_compare.cpp + direct_load/ob_direct_load_conflict_check.cpp direct_load/ob_direct_load_data_block.cpp direct_load/ob_direct_load_data_fuse.cpp + direct_load/ob_direct_load_data_insert.cpp direct_load/ob_direct_load_datum.cpp direct_load/ob_direct_load_external_fragment.cpp direct_load/ob_direct_load_external_multi_partition_row.cpp @@ -769,6 +771,7 @@ ob_set_subtarget(ob_storage direct_load direct_load/ob_direct_load_multiple_sstable_scan_merge_loser_tree.cpp direct_load/ob_direct_load_multiple_sstable_scanner.cpp direct_load/ob_direct_load_origin_table.cpp + direct_load/ob_direct_load_partition_del_lob_task.cpp direct_load/ob_direct_load_partition_merge_task.cpp direct_load/ob_direct_load_partition_rescan_task.cpp direct_load/ob_direct_load_range_splitter.cpp @@ -793,6 +796,7 @@ ob_set_subtarget(ob_storage direct_load direct_load/ob_direct_load_mem_context.cpp direct_load/ob_direct_load_multiple_heap_table_map.cpp direct_load/ob_direct_load_multiple_heap_table_sorter.cpp + direct_load/ob_direct_load_lob_meta_row_iter.cpp ) ob_set_subtarget(ob_storage lob diff --git a/src/storage/access/ob_multiple_merge.cpp b/src/storage/access/ob_multiple_merge.cpp index bd58e9f372..2411c3b4cd 100644 --- a/src/storage/access/ob_multiple_merge.cpp +++ b/src/storage/access/ob_multiple_merge.cpp @@ -1484,7 +1484,9 @@ int ObMultipleMerge::read_lob_columns_full_data(blocksstable::ObDatumRow &row) bool ObMultipleMerge::need_read_lob_columns(const blocksstable::ObDatumRow &row) { - return (access_param_->iter_param_.has_lob_column_out_ && row.row_flag_.is_exist()); + return (!access_ctx_->query_flag_.is_skip_read_lob() && + access_param_->iter_param_.has_lob_column_out_ && + row.row_flag_.is_exist()); } // handle lobs before process_fuse_row diff --git a/src/storage/ddl/ob_ddl_redo_log_row_iterator.cpp b/src/storage/ddl/ob_ddl_redo_log_row_iterator.cpp index e01d9cbce5..a8e5cfbafc 100644 --- a/src/storage/ddl/ob_ddl_redo_log_row_iterator.cpp +++ b/src/storage/ddl/ob_ddl_redo_log_row_iterator.cpp @@ -98,7 +98,7 @@ int ObDDLRedoLogRowIterator::get_next_row(const ObDatumRow *&row, LOG_WARN("fail to assign rowkey", KR(ret)); } else { rowkey = &rowkey_; - seq_no = ObTxSEQ::cast_from_int(datums[schema_rowkey_column_count_ + 1].get_int()); + seq_no = ObTxSEQ::cast_from_int(-datums[schema_rowkey_column_count_ + 1].get_int()); row_flag = row->row_flag_; } } diff --git a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp index 04574c7f33..3b01a1efeb 100644 --- a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp +++ b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp @@ -1551,7 +1551,8 @@ int ObTabletDirectLoadMgr::fill_lob_sstable_slice( ObDirectLoadSliceWriter *slice_writer = nullptr; const int64_t trans_version = is_full_direct_load(direct_load_type_) ? table_key_.get_snapshot_version() : INT64_MAX; ObBatchSliceWriteInfo info(tablet_id_, ls_id_, trans_version, direct_load_type_, sqc_build_ctx_.build_param_.runtime_only_param_.trans_id_, - sqc_build_ctx_.build_param_.runtime_only_param_.seq_no_, slice_info.src_tenant_id_); + sqc_build_ctx_.build_param_.runtime_only_param_.seq_no_, slice_info.src_tenant_id_, + sqc_build_ctx_.build_param_.runtime_only_param_.tx_desc_); if (OB_FAIL(lob_mgr_handle_.get_obj()->get_sqc_build_ctx().slice_mgr_map_.get_refactored(slice_info.slice_id_, slice_writer))) { LOG_WARN("get refactored failed", K(ret), K(slice_info), K(sqc_build_ctx_.slice_mgr_map_.size())); @@ -1611,7 +1612,8 @@ int ObTabletDirectLoadMgr::fill_lob_sstable_slice( ObDirectLoadSliceWriter *slice_writer = nullptr; const int64_t trans_version = is_full_direct_load(direct_load_type_) ? table_key_.get_snapshot_version() : INT64_MAX; ObBatchSliceWriteInfo info(tablet_id_, ls_id_, trans_version, direct_load_type_, sqc_build_ctx_.build_param_.runtime_only_param_.trans_id_, - sqc_build_ctx_.build_param_.runtime_only_param_.seq_no_, slice_info.src_tenant_id_); + sqc_build_ctx_.build_param_.runtime_only_param_.seq_no_, slice_info.src_tenant_id_, + sqc_build_ctx_.build_param_.runtime_only_param_.tx_desc_); if (OB_FAIL(lob_mgr_handle_.get_obj()->get_sqc_build_ctx().slice_mgr_map_.get_refactored(slice_info.slice_id_, slice_writer))) { LOG_WARN("get refactored failed", K(ret), K(slice_info), K(sqc_build_ctx_.slice_mgr_map_.size())); @@ -1640,6 +1642,56 @@ int ObTabletDirectLoadMgr::fill_lob_sstable_slice( return ret; } +int ObTabletDirectLoadMgr::fill_lob_meta_sstable_slice( + const ObDirectLoadSliceInfo &slice_info, + const share::SCN &start_scn, + ObIStoreRowIterator *iter, + int64_t &affected_rows) +{ + int ret = OB_SUCCESS; + share::SCN commit_scn; + affected_rows = 0; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_UNLIKELY(!slice_info.is_valid() || !sqc_build_ctx_.is_valid() || !start_scn.is_valid_and_not_min() || + !lob_mgr_handle_.is_valid() || !lob_mgr_handle_.get_obj()->get_sqc_build_ctx().is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(slice_info), "lob_direct_load_mgr is valid", lob_mgr_handle_.is_valid(), KPC(this), K(start_scn)); + } else if (OB_UNLIKELY(!is_incremental_direct_load(direct_load_type_))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected direct load type", K(ret), K(direct_load_type_)); + } else { + ObDirectLoadSliceWriter *slice_writer = nullptr; + if (OB_FAIL(lob_mgr_handle_.get_obj()->get_sqc_build_ctx().slice_mgr_map_.get_refactored(slice_info.slice_id_, slice_writer))) { + LOG_WARN("get refactored failed", K(ret), K(slice_info), K(sqc_build_ctx_.slice_mgr_map_.size())); + } else if (OB_ISNULL(slice_writer) || OB_UNLIKELY(!ATOMIC_LOAD(&(lob_mgr_handle_.get_obj()->is_schema_item_ready_)))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected err", K(ret), K(slice_info), K(lob_mgr_handle_.get_obj()->is_schema_item_ready_)); + } else if (OB_FAIL(slice_writer->fill_lob_meta_sstable_slice(start_scn, + lob_mgr_handle_.get_obj()->sqc_build_ctx_.build_param_.runtime_only_param_.table_id_, + lob_mgr_handle_.get_obj()->tablet_id_, + iter, + affected_rows))) { + LOG_WARN("fail to fill lob meta sstable slice", K(ret), K(start_scn), K(tablet_id_)); + } + } + if (OB_FAIL(ret) && lob_mgr_handle_.is_valid()) { + // cleanup when failed. + int tmp_ret = OB_SUCCESS; + ObDirectLoadSliceWriter *slice_writer = nullptr; + if (OB_TMP_FAIL(lob_mgr_handle_.get_obj()->get_sqc_build_ctx().slice_mgr_map_.erase_refactored(slice_info.slice_id_, &slice_writer))) { + LOG_ERROR("erase failed", K(ret), K(tmp_ret), K(slice_info)); + } else { + LOG_INFO("erase a slice writer", KP(slice_writer), "slice_id", slice_info.slice_id_, K(sqc_build_ctx_.slice_mgr_map_.size())); + slice_writer->~ObDirectLoadSliceWriter(); + lob_mgr_handle_.get_obj()->get_sqc_build_ctx().slice_writer_allocator_.free(slice_writer); + slice_writer = nullptr; + } + } + return ret; +} + int ObTabletDirectLoadMgr::wait_notify(const ObDirectLoadSliceWriter *slice_writer, const share::SCN &start_scn) { int ret = OB_SUCCESS; diff --git a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.h b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.h index 65cbc12462..c408383b42 100644 --- a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.h +++ b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.h @@ -344,6 +344,12 @@ public: const ObArray &lob_column_idxs, const ObArray &col_types, blocksstable::ObDatumRow &datum_row); + // for delete lob in incremental direct load only + virtual int fill_lob_meta_sstable_slice( + const ObDirectLoadSliceInfo &slice_info /*contains data_tablet_id, lob_slice_id, start_seq*/, + const share::SCN &start_scn, + ObIStoreRowIterator *iter, + int64_t &affected_rows); virtual int close_sstable_slice( const bool is_data_tablet_process_for_lob, const ObDirectLoadSliceInfo &slice_info, diff --git a/src/storage/ddl/ob_direct_load_struct.cpp b/src/storage/ddl/ob_direct_load_struct.cpp index 3c7b518fbe..6aa6c6f3d6 100644 --- a/src/storage/ddl/ob_direct_load_struct.cpp +++ b/src/storage/ddl/ob_direct_load_struct.cpp @@ -320,7 +320,7 @@ int ObDDLInsertRowIterator::switch_to_new_lob_slice() ObLobMetaRowIterator::ObLobMetaRowIterator() : is_inited_(false), iter_(nullptr), trans_id_(0), trans_version_(0), sql_no_(0), - tmp_row_(), lob_meta_write_result_() + tmp_row_(), lob_meta_write_result_(), direct_load_type_(DIRECT_LOAD_INVALID) { } @@ -332,7 +332,8 @@ ObLobMetaRowIterator::~ObLobMetaRowIterator() int ObLobMetaRowIterator::init(ObLobMetaWriteIter *iter, const transaction::ObTransID &trans_id, const int64_t trans_version, - const int64_t sql_no) + const int64_t sql_no, + const ObDirectLoadType direct_load_type) { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_inited_)) { @@ -348,6 +349,7 @@ int ObLobMetaRowIterator::init(ObLobMetaWriteIter *iter, trans_id_ = trans_id; trans_version_ = trans_version; sql_no_ = sql_no; + direct_load_type_ = direct_load_type; is_inited_ = true; } return ret; @@ -360,6 +362,7 @@ void ObLobMetaRowIterator::reset() trans_id_.reset(); trans_version_ = 0; sql_no_ = 0; + direct_load_type_ = DIRECT_LOAD_INVALID; tmp_row_.reset(); } @@ -392,9 +395,10 @@ int ObLobMetaRowIterator::get_next_row(const blocksstable::ObDatumRow *&row) LOG_WARN("transform failed", K(ret), K(lob_meta_write_result_.info_)); } else { tmp_row_.storage_datums_[ObLobMetaUtil::SEQ_ID_COL_ID + 1].set_int(-trans_version_); - tmp_row_.storage_datums_[ObLobMetaUtil::SEQ_ID_COL_ID + 2].set_int(-sql_no_); + tmp_row_.storage_datums_[ObLobMetaUtil::SEQ_ID_COL_ID + 2].set_int(-get_seq_no()); tmp_row_.set_trans_id(trans_id_); tmp_row_.row_flag_.set_flag(ObDmlFlag::DF_INSERT); + tmp_row_.mvcc_row_flag_.set_last_multi_version_row(true); tmp_row_.mvcc_row_flag_.set_uncommitted_row(trans_id_.is_valid()); row = &tmp_row_; } @@ -402,6 +406,11 @@ int ObLobMetaRowIterator::get_next_row(const blocksstable::ObDatumRow *&row) return ret; } +int64_t ObLobMetaRowIterator::get_seq_no() const +{ + return is_incremental_direct_load(direct_load_type_) ? lob_meta_write_result_.seq_no_ : sql_no_; +} + ObTabletDDLParam::ObTabletDDLParam() : direct_load_type_(ObDirectLoadType::DIRECT_LOAD_INVALID), ls_id_(), @@ -840,6 +849,8 @@ int ObDirectLoadSliceWriter::prepare_iters( const int64_t timeout_ts, const int64_t lob_inrow_threshold, const uint64_t src_tenant_id, + const ObDirectLoadType direct_load_type, + transaction::ObTxDesc* tx_desc, ObLobMetaRowIterator *&row_iter) { int ret = OB_SUCCESS; @@ -851,11 +862,11 @@ int ObDirectLoadSliceWriter::prepare_iters( ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc lob meta write iter failed", K(ret)); } else { - meta_write_iter_ = new (buf) ObLobMetaWriteIter(datum.get_string(), &iter_allocator, ObLobMetaUtil::LOB_OPER_PIECE_DATA_SIZE); + // keep allocator is same as insert_lob_column + meta_write_iter_ = new (buf) ObLobMetaWriteIter(&allocator, ObLobMetaUtil::LOB_OPER_PIECE_DATA_SIZE); } - } else { - meta_write_iter_->set_data(datum.get_string()); } + if (OB_SUCC(ret)) { if (OB_ISNULL(row_iterator_)) { void *buf = nullptr; @@ -872,13 +883,17 @@ int ObDirectLoadSliceWriter::prepare_iters( ObLobStorageParam lob_storage_param; lob_storage_param.inrow_threshold_ = lob_inrow_threshold; int64_t unused_affected_rows = 0; - if (OB_FAIL(ObInsertLobColumnHelper::insert_lob_column( - allocator, nullptr, ls_id, tablet_id, lob_id, obj_type, cs_type, lob_storage_param, datum, + if (is_incremental_direct_load(direct_load_type) && OB_ISNULL(tx_desc)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tx_desc should not be null if is incremental_direct_load", K(ret), K(direct_load_type), + K(ls_id), K(tablet_id), K(lob_id), K(trans_version), K(seq_no), K(obj_type), K(cs_type), K(trans_id)); + } else if (OB_FAIL(ObInsertLobColumnHelper::insert_lob_column( + allocator, tx_desc, ls_id, tablet_id, lob_id, obj_type, cs_type, lob_storage_param, datum, timeout_ts, true/*has_lob_header*/, src_tenant_id, *meta_write_iter_))) { LOG_WARN("fail to insert_lob_col", K(ret), K(ls_id), K(tablet_id), K(lob_id), K(src_tenant_id)); } else if (OB_FAIL(row_iterator_->init(meta_write_iter_, trans_id, - trans_version, seq_no))) { - LOG_WARN("fail to lob meta row iterator", K(ret), K(trans_id), K(trans_version), K(seq_no)); + trans_version, seq_no, direct_load_type))) { + LOG_WARN("fail to lob meta row iterator", K(ret), K(trans_id), K(trans_version), K(seq_no), K(direct_load_type)); } else { row_iter = row_iterator_; } @@ -970,7 +985,7 @@ int ObDirectLoadSliceWriter::fill_lob_into_macro_block( ObLobMetaRowIterator *row_iter = nullptr; if (OB_FAIL(prepare_iters(allocator, iter_allocator, datum, info.ls_id_, info.data_tablet_id_, info.trans_version_, col_types.at(i).get_type(), col_types.at(i).get_collation_type(), lob_id, - info.trans_id_, info.seq_no_, timeout_ts, lob_inrow_threshold, info.src_tenant_id_, row_iter))) { + info.trans_id_, info.seq_no_, timeout_ts, lob_inrow_threshold, info.src_tenant_id_, info.direct_load_type_, info.tx_desc_, row_iter))) { LOG_WARN("fail to prepare iters", K(ret), KP(row_iter), K(datum)); } else { while (OB_SUCC(ret)) { @@ -1018,6 +1033,59 @@ int ObDirectLoadSliceWriter::fill_lob_into_macro_block( return ret; } +int ObDirectLoadSliceWriter::fill_lob_meta_sstable_slice( + const share::SCN &start_scn, + const uint64_t table_id, + const ObTabletID &curr_tablet_id, + ObIStoreRowIterator *row_iter, + int64_t &affected_rows) +{ + int ret = OB_SUCCESS; + affected_rows = 0; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadSliceWriter not init", KR(ret), KP(this)); + } else { + const int64_t rowkey_column_count = ObLobMetaUtil::LOB_META_SCHEMA_ROWKEY_COL_CNT; + const int64_t column_count = ObLobMetaUtil::LOB_META_COLUMN_CNT + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); + while (OB_SUCC(ret)) { + const blocksstable::ObDatumRow *cur_row = nullptr; + if (OB_FAIL(THIS_WORKER.check_status())) { + LOG_WARN("check status failed", K(ret)); + } else if (ATOMIC_LOAD(&is_canceled_)) { + ret = OB_CANCELED; + LOG_WARN("fil sstable task canceled", K(ret), K(is_canceled_)); + } else if (OB_FAIL(row_iter->get_next_row(cur_row))) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + break; + } else { + LOG_WARN("get next row failed", K(ret)); + } + } else if (OB_ISNULL(cur_row) || !cur_row->is_valid() || cur_row->get_column_count() != column_count) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KPC(cur_row), K(column_count)); + } else if (OB_FAIL(check_null(false/*is_index_table*/, rowkey_column_count, *cur_row))) { + LOG_WARN("fail to check null value in row", KR(ret), KPC(cur_row)); + } else if (OB_FAIL(prepare_slice_store_if_need(rowkey_column_count, + false/*is_column_store*/, + 1L/*unsued*/, + 1L/*unused*/, + nullptr /*storage_schema*/, + start_scn))) { + LOG_WARN("prepare macro block writer failed", K(ret)); + } else if (OB_FAIL(slice_store_->append_row(*cur_row))) { + LOG_WARN("macro block writer append row failed", K(ret), KPC(cur_row)); + } + if (OB_SUCC(ret)) { + ++affected_rows; + LOG_DEBUG("sstable insert op append row", K(affected_rows), KPC(cur_row)); + } + } + } + return ret; +} + int ObDirectLoadSliceWriter::fill_sstable_slice( const SCN &start_scn, const uint64_t table_id, diff --git a/src/storage/ddl/ob_direct_load_struct.h b/src/storage/ddl/ob_direct_load_struct.h index b8cd786f2a..db0aeac8c7 100644 --- a/src/storage/ddl/ob_direct_load_struct.h +++ b/src/storage/ddl/ob_direct_load_struct.h @@ -55,22 +55,24 @@ public: direct_load_type_(), trans_id_(), seq_no_(0), - src_tenant_id_(0) + src_tenant_id_(0), + tx_desc_(nullptr) { } ObBatchSliceWriteInfo(const common::ObTabletID &tablet_id, const share::ObLSID &ls_id, const int64_t &trans_version, const ObDirectLoadType &direct_load_type, const transaction::ObTransID &trans_id, const int64_t &seq_no, - const uint64_t src_tenant_id) + const uint64_t src_tenant_id, transaction::ObTxDesc* tx_desc) : data_tablet_id_(tablet_id), ls_id_(ls_id), trans_version_(trans_version), direct_load_type_(direct_load_type), trans_id_(trans_id), seq_no_(seq_no), - src_tenant_id_(src_tenant_id) + src_tenant_id_(src_tenant_id), + tx_desc_(tx_desc) { } ~ObBatchSliceWriteInfo() = default; - TO_STRING_KV(K(ls_id_), K(data_tablet_id_), K(trans_version_), K(direct_load_type_), K(src_tenant_id_)); + TO_STRING_KV(K(ls_id_), K(data_tablet_id_), K(trans_version_), K(direct_load_type_), K(src_tenant_id_), KPC(tx_desc_)); public: common::ObTabletID data_tablet_id_; share::ObLSID ls_id_; @@ -79,6 +81,7 @@ public: transaction::ObTransID trans_id_; int64_t seq_no_; // uint64_t src_tenant_id_; + transaction::ObTxDesc* tx_desc_; }; struct ObTabletDirectLoadMgrKey final @@ -317,11 +320,15 @@ public: int init(ObLobMetaWriteIter *iter, const transaction::ObTransID &trans_id, const int64_t trans_version, - const int64_t sql_no); + const int64_t sql_no, + const ObDirectLoadType direct_load_type); void reset(); void reuse(); virtual int get_next_row(const blocksstable::ObDatumRow *&row) override; -// private: + +private: + int64_t get_seq_no() const; + public: bool is_inited_; ObLobMetaWriteIter *iter_; @@ -330,6 +337,7 @@ public: int64_t sql_no_; blocksstable::ObDatumRow tmp_row_; ObLobMetaWriteResult lob_meta_write_result_; + ObDirectLoadType direct_load_type_; }; struct ObTabletDDLParam final @@ -512,6 +520,13 @@ public: const ObArray &col_types, const int64_t lob_inrow_threshold, blocksstable::ObDatumRow &datum_row); + // fill lob meta row into macro block + int fill_lob_meta_sstable_slice( + const share::SCN &start_scn, + const uint64_t table_id, + const ObTabletID &curr_tablet_id, + ObIStoreRowIterator *row_iter, + int64_t &affected_rows); int close(); int fill_column_group( const ObStorageSchema *storage_schema, @@ -582,6 +597,8 @@ private: const int64_t timeout_ts, const int64_t lob_inrow_threshold, const uint64_t src_tenant_id, + const ObDirectLoadType direct_load_type, + transaction::ObTxDesc* tx_desc, ObLobMetaRowIterator *&row_iter); int mock_chunk_store(const int64_t row_cnt); private: diff --git a/src/storage/direct_load/ob_direct_load_conflict_check.cpp b/src/storage/direct_load/ob_direct_load_conflict_check.cpp new file mode 100644 index 0000000000..f2b4f3a19e --- /dev/null +++ b/src/storage/direct_load/ob_direct_load_conflict_check.cpp @@ -0,0 +1,365 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#define USING_LOG_PREFIX STORAGE + +#include "storage/direct_load/ob_direct_load_conflict_check.h" + +namespace oceanbase +{ +namespace storage +{ +using namespace common; +using namespace blocksstable; +using namespace share; +using namespace sql; + +/** + * ObDirectLoadConflictCheckParam + */ + +ObDirectLoadConflictCheckParam::ObDirectLoadConflictCheckParam() + : store_column_count_(0), + origin_table_(nullptr), + range_(nullptr), + col_descs_(nullptr), + lob_column_idxs_(nullptr), + builder_(nullptr), + datum_utils_(nullptr), + dml_row_handler_(nullptr) +{ +} + +ObDirectLoadConflictCheckParam::~ObDirectLoadConflictCheckParam() +{ +} + +bool ObDirectLoadConflictCheckParam::is_valid() const +{ + return tablet_id_.is_valid() && store_column_count_ > 0 && table_data_desc_.is_valid() && + nullptr != origin_table_ && nullptr != range_ && range_->is_valid() && + nullptr != col_descs_ && nullptr != lob_column_idxs_ && nullptr != builder_ && + nullptr != datum_utils_ && nullptr != dml_row_handler_; +} + +/** + * ObDirectLoadConflictCheck + */ + +ObDirectLoadConflictCheck::ObDirectLoadConflictCheck() + : allocator_("TLD_CfltCheck"), + range_allocator_("TLD_RCfltCheck"), + load_iter_(nullptr), + origin_iter_(nullptr), + origin_row_(nullptr), + origin_iter_is_end_(false), + is_inited_(false) +{ + allocator_.set_tenant_id(MTL_ID()); + range_allocator_.set_tenant_id(MTL_ID()); +} + +ObDirectLoadConflictCheck::~ObDirectLoadConflictCheck() +{ + if (origin_iter_ != nullptr) { + origin_iter_->~ObIStoreRowIterator(); + allocator_.free(origin_iter_); + origin_iter_ = nullptr; + } +} + +int ObDirectLoadConflictCheck::init( + const ObDirectLoadConflictCheckParam ¶m, + ObIStoreRowIterator *load_iter) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("already init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(!param.is_valid() || nullptr == load_iter)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid argument", K(ret), K(param), KP(load_iter)); + } else { + param_ = param; + load_iter_ = load_iter; + new_range_ = *param_.range_; + if (OB_FAIL(param_.origin_table_->scan(*param_.range_, allocator_, origin_iter_, true/*skip_read_lob*/))) { + LOG_WARN("fail to scan origin table", KR(ret)); + } else if (OB_FAIL(append_row_.init(1/*lobid*/))) { + LOG_WARN("fail to init append_rows_", KR(ret)); + } + if (OB_SUCC(ret)) { + is_inited_ = true; + } + } + + return ret; +} + +int ObDirectLoadConflictCheck::get_next_row(const ObDatumRow *&datum_row) +{ + int ret = OB_SUCCESS; + datum_row = nullptr; + const ObDatumRow *load_row = nullptr; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), KP(this)); + } else { + while (OB_SUCC(ret) && OB_ISNULL(datum_row)) { + if (OB_FAIL(load_iter_->get_next_row(load_row))) { + if (ret != OB_ITER_END) { + LOG_WARN("fail to get next row", KR(ret)); + } + } else if (OB_UNLIKELY(load_row->count_ != param_.store_column_count_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected column count", KR(ret), K(load_row->count_), K(param_.store_column_count_)); + } else if (OB_FAIL(handle_get_next_row_finish(load_row, datum_row))) { + LOG_WARN("fail to handle get next row finish", KR(ret), KP(load_row)); + } + } + } + + return ret; +} + +int ObDirectLoadConflictCheck::handle_get_next_row_finish( + const ObDatumRow *load_row, + const ObDatumRow *&datum_row) +{ + int ret = OB_SUCCESS; + int cmp_ret = 1; + int64_t skip_count = 0; + while (OB_SUCC(ret) && !origin_iter_is_end_) { + cmp_ret = 1; + if (origin_row_ == nullptr) { + if (OB_FAIL(origin_iter_->get_next_row(origin_row_))) { + if (ret == OB_ITER_END) { + origin_iter_is_end_ = true; + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to get next row", KR(ret)); + } + } else if (OB_UNLIKELY(origin_row_->count_ != param_.store_column_count_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected column count", KR(ret), K(origin_row_->count_), K(param_.store_column_count_)); + } + } + if (OB_SUCC(ret) && origin_row_ != nullptr) { + if (OB_FAIL(compare(*load_row, *origin_row_, cmp_ret))) { + LOG_WARN("fail to compare", KR(ret), K(skip_count)); + } else { + if (cmp_ret <= 0) { + break; + } else { + skip_count++; + origin_row_ = nullptr; + if (skip_count == SKIP_THESHOLD) { + skip_count = 0; + if (OB_FAIL(reopen_origin_iter(load_row))) { + LOG_WARN("fail to reopen origin_iter_", KR(ret)); + } + } + } + } + } + } + if (OB_SUCC(ret)) { + if (cmp_ret == 0) { + if (OB_FAIL(param_.dml_row_handler_->handle_update_row(*origin_row_, *load_row, datum_row))) { + LOG_WARN("fail to handle update row", KR(ret), KP(origin_row_), KP(load_row)); + } else { + if (datum_row == load_row) { + if (OB_FAIL(handle_old_row(origin_row_))) { + LOG_WARN("fail to handle old row", KR(ret), KP(origin_row_)); + } + } else { + datum_row = nullptr; + } + } + origin_row_ = nullptr; + } else { + datum_row = load_row; + if (OB_FAIL(param_.dml_row_handler_->handle_insert_row(*datum_row))) { + LOG_WARN("fail to handle insert row", KR(ret), KP(datum_row)); + } + } + } + + return ret; +} + +int ObDirectLoadConflictCheck::compare( + const ObDatumRow &first_row, + const ObDatumRow &second_row, + int &cmp_ret) +{ + int ret = OB_SUCCESS; + ObDatumRowkey first_key(first_row.storage_datums_, param_.table_data_desc_.rowkey_column_num_); + ObDatumRowkey second_key(second_row.storage_datums_, param_.table_data_desc_.rowkey_column_num_); + if (OB_FAIL(first_key.compare(second_key, *param_.datum_utils_, cmp_ret))) { + LOG_WARN("fail to compare", KR(ret)); + } + + return ret; +} + +int ObDirectLoadConflictCheck::reopen_origin_iter(const ObDatumRow *datum_row) +{ + int ret = OB_SUCCESS; + range_allocator_.reuse(); + ObDatumRowkey start_key(datum_row->storage_datums_, param_.table_data_desc_.rowkey_column_num_); + if (OB_FAIL(start_key.deep_copy(new_range_.start_key_, range_allocator_))) { + LOG_WARN("fail to copy start_key", KR(ret)); + } else if (OB_FAIL(new_range_.start_key_.prepare_memtable_readable(*param_.col_descs_, range_allocator_))) { + LOG_WARN("fail to prepare_memtable_readable", KR(ret)); + } else { + new_range_.set_left_closed(); + if (OB_FAIL(param_.origin_table_->rescan(new_range_, origin_iter_))) { + LOG_WARN("fail to rescan origin table", KR(ret), K(new_range_)); + } + } + + return ret; +} + +int ObDirectLoadConflictCheck::handle_old_row(const ObDatumRow *old_row) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < param_.lob_column_idxs_->count(); i++) { + int64_t lob_idx = param_.lob_column_idxs_->at(i); + ObStorageDatum &datum = old_row->storage_datums_[lob_idx]; + const ObLobCommon &lob_common = datum.get_lob_data(); + if (!lob_common.in_row_) { + const ObLobId &lob_id = reinterpret_cast(lob_common.buffer_)->id_; + append_row_.storage_datums_[0].set_string(reinterpret_cast(&lob_id), sizeof(ObLobId)); + if (OB_FAIL(param_.builder_->append_row(param_.tablet_id_, 0/*seq_no*/, append_row_))) { + LOG_WARN("fail to append row", KR(ret)); + } + } + } + + return ret; +} + +/** + * ObDirectLoadSSTableConflictCheck + */ + +ObDirectLoadSSTableConflictCheck::ObDirectLoadSSTableConflictCheck() + : is_inited_(false) +{ +} + +ObDirectLoadSSTableConflictCheck::~ObDirectLoadSSTableConflictCheck() +{ +} + +int ObDirectLoadSSTableConflictCheck::init( + const ObDirectLoadConflictCheckParam ¶m, + const ObIArray &sstable_array) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("already init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(!param.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid argument", K(ret), K(param)); + } else { + ObDirectLoadSSTableScanMergeParam scan_merge_param; + scan_merge_param.tablet_id_ = param.tablet_id_; + scan_merge_param.table_data_desc_ = param.table_data_desc_; + scan_merge_param.datum_utils_ = param.datum_utils_; + scan_merge_param.dml_row_handler_ = param.dml_row_handler_; + if (OB_FAIL(scan_merge_.init(scan_merge_param, sstable_array, *param.range_))) { + LOG_WARN("fail to init scan merge", KR(ret)); + } else if (OB_FAIL(conflict_check_.init(param, &scan_merge_))) { + LOG_WARN("fail to init conflict_check_", KR(ret)); + } else { + is_inited_ = true; + } + } + + return ret; +} + +int ObDirectLoadSSTableConflictCheck::get_next_row(const ObDatumRow *&datum_row) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), KP(this)); + } else { + ret = conflict_check_.get_next_row(datum_row); + } + return ret; +} + +/** + * ObDirectLoadMultipleSSTableConflictCheck + */ + +ObDirectLoadMultipleSSTableConflictCheck::ObDirectLoadMultipleSSTableConflictCheck() + : is_inited_(false) +{ +} + +ObDirectLoadMultipleSSTableConflictCheck::~ObDirectLoadMultipleSSTableConflictCheck() +{ +} + +int ObDirectLoadMultipleSSTableConflictCheck::init( + const ObDirectLoadConflictCheckParam ¶m, + const ObIArray &sstable_array) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("already init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(!param.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid argument", K(ret), K(param)); + } else if (OB_FAIL(range_.assign(param.tablet_id_, *param.range_))) { + LOG_WARN("fail to assign range", KR(ret)); + } else { + ObDirectLoadMultipleSSTableScanMergeParam scan_merge_param; + scan_merge_param.table_data_desc_ = param.table_data_desc_; + scan_merge_param.datum_utils_ = param.datum_utils_; + scan_merge_param.dml_row_handler_ = param.dml_row_handler_; + if (OB_FAIL(scan_merge_.init(scan_merge_param, sstable_array, range_))) { + LOG_WARN("fail to init scan merge", KR(ret)); + } else if (OB_FAIL(conflict_check_.init(param, &scan_merge_))) { + LOG_WARN("fail to init conflict_check_", KR(ret)); + } else { + is_inited_ = true; + } + } + + return ret; +} + +int ObDirectLoadMultipleSSTableConflictCheck::get_next_row(const ObDatumRow *&datum_row) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), KP(this)); + } else { + ret = conflict_check_.get_next_row(datum_row); + } + + return ret; +} + + + +} // namespace storage +} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_conflict_check.h b/src/storage/direct_load/ob_direct_load_conflict_check.h new file mode 100644 index 0000000000..1bfbd81341 --- /dev/null +++ b/src/storage/direct_load/ob_direct_load_conflict_check.h @@ -0,0 +1,116 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#pragma once + +#include "storage/blocksstable/ob_sstable.h" +#include "share/table/ob_table_load_define.h" +#include "storage/direct_load/ob_direct_load_dml_row_handler.h" +#include "storage/direct_load/ob_direct_load_multiple_datum_range.h" +#include "storage/direct_load/ob_direct_load_multiple_sstable_scan_merge.h" +#include "storage/direct_load/ob_direct_load_origin_table.h" +#include "storage/direct_load/ob_direct_load_sstable_scan_merge.h" +#include "storage/direct_load/ob_direct_load_external_multi_partition_table.h" + +namespace oceanbase +{ +namespace storage +{ +class ObDirectLoadSSTable; +class ObDirectLoadMultipleSSTable; + +struct ObDirectLoadConflictCheckParam +{ +public: + ObDirectLoadConflictCheckParam(); + ~ObDirectLoadConflictCheckParam(); + bool is_valid() const; + TO_STRING_KV( + K(tablet_id_), K(store_column_count_), K(table_data_desc_), KP(origin_table_), KP(range_), + KP(col_descs_), KP(lob_column_idxs_), KP(builder_), KP(datum_utils_), KP(dml_row_handler_)); +public: + common::ObTabletID tablet_id_; + int64_t store_column_count_; + ObDirectLoadTableDataDesc table_data_desc_; + ObDirectLoadOriginTable *origin_table_; + const blocksstable::ObDatumRange *range_; + const common::ObIArray *col_descs_; + const common::ObArray *lob_column_idxs_; + ObIDirectLoadPartitionTableBuilder *builder_; + const blocksstable::ObStorageDatumUtils *datum_utils_; + ObDirectLoadDMLRowHandler *dml_row_handler_; +}; + +class ObDirectLoadConflictCheck +{ +public: + static const int64_t SKIP_THESHOLD = 100; + ObDirectLoadConflictCheck(); + virtual ~ObDirectLoadConflictCheck(); + int init( + const ObDirectLoadConflictCheckParam ¶m, + ObIStoreRowIterator *load_iter); + int get_next_row(const blocksstable::ObDatumRow *&datum_row); +private: + int handle_get_next_row_finish( + const ObDatumRow *load_row, + const blocksstable::ObDatumRow *&datum_row); + int compare( + const blocksstable::ObDatumRow &first_row, + const blocksstable::ObDatumRow &second_row, + int &cmp_ret); + int reopen_origin_iter(const ObDatumRow *datum_row); + int handle_old_row(const ObDatumRow *old_row); +private: + common::ObArenaAllocator allocator_; + common::ObArenaAllocator range_allocator_; + ObDirectLoadConflictCheckParam param_; + ObIStoreRowIterator *load_iter_; + ObIStoreRowIterator *origin_iter_; + const blocksstable::ObDatumRow *origin_row_; + blocksstable::ObDatumRow append_row_; + ObDatumRange new_range_; + bool origin_iter_is_end_; + bool is_inited_; +}; + +class ObDirectLoadSSTableConflictCheck : public ObIStoreRowIterator +{ +public: + ObDirectLoadSSTableConflictCheck(); + ~ObDirectLoadSSTableConflictCheck(); + int init(const ObDirectLoadConflictCheckParam ¶m, + const common::ObIArray &sstable_array); + int get_next_row(const blocksstable::ObDatumRow *&datum_row) override; +private: + ObDirectLoadSSTableScanMerge scan_merge_; + ObDirectLoadConflictCheck conflict_check_; + bool is_inited_; +}; + +class ObDirectLoadMultipleSSTableConflictCheck : public ObIStoreRowIterator +{ +public: + ObDirectLoadMultipleSSTableConflictCheck(); + virtual ~ObDirectLoadMultipleSSTableConflictCheck(); + int init( + const ObDirectLoadConflictCheckParam ¶m, + const common::ObIArray &sstable_array); + int get_next_row(const blocksstable::ObDatumRow *&datum_row) override; +private: + ObDirectLoadMultipleDatumRange range_; + ObDirectLoadMultipleSSTableScanMerge scan_merge_; + ObDirectLoadConflictCheck conflict_check_; + bool is_inited_; +}; + +} // namespace storage +} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_data_block_writer.h b/src/storage/direct_load/ob_direct_load_data_block_writer.h index d3da5ad01b..ce73d3b8b6 100644 --- a/src/storage/direct_load/ob_direct_load_data_block_writer.h +++ b/src/storage/direct_load/ob_direct_load_data_block_writer.h @@ -74,6 +74,7 @@ ObDirectLoadDataBlockWriter::ObDirectLoadDataBlockWriter() offset_(0), block_count_(0), max_block_size_(0), + callback_(nullptr), is_opened_(false), is_inited_(false) { @@ -122,7 +123,7 @@ int ObDirectLoadDataBlockWriter::init( ObIDirectLoadDataBlockFlushCallback *callback) { int ret = common::OB_SUCCESS; - if (IS_INIT) { + if (OB_UNLIKELY(is_inited_)) { ret = common::OB_INIT_TWICE; STORAGE_LOG(WARN, "ObDirectLoadDataBlockWriter init twice", KR(ret), KP(this)); } else if (OB_UNLIKELY(data_block_size <= 0 || data_block_size % DIO_ALIGN_SIZE != 0 || diff --git a/src/storage/direct_load/ob_direct_load_data_fuse.cpp b/src/storage/direct_load/ob_direct_load_data_fuse.cpp index 0cb6956f59..8d5f224bf4 100644 --- a/src/storage/direct_load/ob_direct_load_data_fuse.cpp +++ b/src/storage/direct_load/ob_direct_load_data_fuse.cpp @@ -312,7 +312,7 @@ int ObDirectLoadSSTableDataFuse::init(const ObDirectLoadDataFuseParam ¶m, LOG_WARN("Invalid argument", K(ret), K(param), KP(origin_table), K(range)); } else { // construct iters - if (OB_FAIL(origin_table->scan(range, allocator_, origin_iter_))) { + if (OB_FAIL(origin_table->scan(range, allocator_, origin_iter_, false/*skip_read_lob*/))) { LOG_WARN("fail to scan origin table", KR(ret)); } else { ObDirectLoadSSTableScanMergeParam scan_merge_param; @@ -384,7 +384,7 @@ int ObDirectLoadMultipleSSTableDataFuse::init( LOG_WARN("fail to assign range", KR(ret)); } // construct iters - else if (OB_FAIL(origin_table->scan(range, allocator_, origin_iter_))) { + else if (OB_FAIL(origin_table->scan(range, allocator_, origin_iter_, false/*skip_read_lob*/))) { LOG_WARN("fail to scan origin table", KR(ret)); } else { ObDirectLoadMultipleSSTableScanMergeParam scan_merge_param; diff --git a/src/storage/direct_load/ob_direct_load_data_fuse.h b/src/storage/direct_load/ob_direct_load_data_fuse.h index b1cb4562a2..2185757964 100644 --- a/src/storage/direct_load/ob_direct_load_data_fuse.h +++ b/src/storage/direct_load/ob_direct_load_data_fuse.h @@ -19,6 +19,7 @@ #include "storage/direct_load/ob_direct_load_multiple_sstable_scan_merge.h" #include "storage/direct_load/ob_direct_load_origin_table.h" #include "storage/direct_load/ob_direct_load_sstable_scan_merge.h" +#include "storage/direct_load/ob_direct_load_struct.h" namespace oceanbase { @@ -97,7 +98,7 @@ protected: bool is_inited_; }; -class ObDirectLoadSSTableDataFuse +class ObDirectLoadSSTableDataFuse : public ObIStoreRowIterator { public: ObDirectLoadSSTableDataFuse(); @@ -105,7 +106,7 @@ public: int init(const ObDirectLoadDataFuseParam ¶m, ObDirectLoadOriginTable *origin_table, const common::ObIArray &sstable_array, const blocksstable::ObDatumRange &range); - int get_next_row(const blocksstable::ObDatumRow *&datum_row); + int get_next_row(const blocksstable::ObDatumRow *&datum_row) override; private: common::ObArenaAllocator allocator_; ObIStoreRowIterator *origin_iter_; @@ -114,7 +115,7 @@ private: bool is_inited_; }; -class ObDirectLoadMultipleSSTableDataFuse +class ObDirectLoadMultipleSSTableDataFuse : public ObIStoreRowIterator { public: ObDirectLoadMultipleSSTableDataFuse(); @@ -122,7 +123,7 @@ public: int init(const ObDirectLoadDataFuseParam ¶m, ObDirectLoadOriginTable *origin_table, const common::ObIArray &sstable_array, const blocksstable::ObDatumRange &range); - int get_next_row(const blocksstable::ObDatumRow *&datum_row); + int get_next_row(const blocksstable::ObDatumRow *&datum_row) override; private: common::ObArenaAllocator allocator_; ObDirectLoadMultipleDatumRange range_; diff --git a/src/storage/direct_load/ob_direct_load_data_insert.cpp b/src/storage/direct_load/ob_direct_load_data_insert.cpp new file mode 100644 index 0000000000..4018e1f29f --- /dev/null +++ b/src/storage/direct_load/ob_direct_load_data_insert.cpp @@ -0,0 +1,213 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#define USING_LOG_PREFIX STORAGE + +#include "storage/direct_load/ob_direct_load_data_insert.h" + +namespace oceanbase +{ +namespace storage +{ +using namespace common; +using namespace blocksstable; +using namespace share; +using namespace sql; + +/** + * ObDirectLoadSSTableScanMergeParam + */ + +ObDirectLoadDataInsertParam::ObDirectLoadDataInsertParam() + : store_column_count_(0), + datum_utils_(nullptr), + dml_row_handler_(nullptr) +{ +} + +ObDirectLoadDataInsertParam::~ObDirectLoadDataInsertParam() +{ +} + +bool ObDirectLoadDataInsertParam::is_valid() const +{ + return tablet_id_.is_valid() && store_column_count_ > 0 && table_data_desc_.is_valid() && + nullptr != datum_utils_ && nullptr != dml_row_handler_; +} + +/** + * ObDirectLoadDataInsert + */ + +ObDirectLoadDataInsert::ObDirectLoadDataInsert() + : load_iter_(nullptr), is_inited_(false) +{ +} + +ObDirectLoadDataInsert::~ObDirectLoadDataInsert() +{ +} + +int ObDirectLoadDataInsert::init( + const ObDirectLoadDataInsertParam ¶m, + ObIStoreRowIterator *load_iter) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObDirectLoadDataInsert init twice", KR(ret), KP(this)); + } else if (OB_UNLIKELY(!param.is_valid() || nullptr == load_iter)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid argument", K(ret), K(param), KP(load_iter)); + } else { + param_ = param; + load_iter_ = load_iter; + is_inited_ = true; + } + + return ret; +} + +int ObDirectLoadDataInsert::get_next_row(const ObDatumRow *&datum_row) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), KP(this)); + } else if (OB_FAIL(load_iter_->get_next_row(datum_row))) { + if (ret != OB_ITER_END) { + LOG_WARN("fail to get next row", KR(ret)); + } + } else if (OB_UNLIKELY(datum_row->count_ != param_.store_column_count_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected column count", KR(ret), K(datum_row->count_), K(param_.store_column_count_)); + } else if (OB_FAIL(param_.dml_row_handler_->handle_insert_row(*datum_row))) { + LOG_WARN("fail to handle insert row", KR(ret), KPC(datum_row)); + } + + return ret; +} + +/** + * ObDirectLoadSSTableDataInsert + */ + +ObDirectLoadSSTableDataInsert::ObDirectLoadSSTableDataInsert() + : is_inited_(false) +{ +} + +ObDirectLoadSSTableDataInsert::~ObDirectLoadSSTableDataInsert() +{ +} + +int ObDirectLoadSSTableDataInsert::init( + const ObDirectLoadDataInsertParam ¶m, + const ObIArray &sstable_array, + const ObDatumRange &range) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", KR(ret), KP(this)); + } else if (OB_UNLIKELY(!param.is_valid()) || !range.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid argument", K(ret), K(param), K(range)); + } else { + ObDirectLoadSSTableScanMergeParam scan_merge_param; + scan_merge_param.tablet_id_ = param.tablet_id_; + scan_merge_param.table_data_desc_ = param.table_data_desc_; + scan_merge_param.datum_utils_ = param.datum_utils_; + scan_merge_param.dml_row_handler_ = param.dml_row_handler_; + if (OB_FAIL(scan_merge_.init(scan_merge_param, sstable_array, range))) { + LOG_WARN("fail to init scan merge", KR(ret)); + } else if (OB_FAIL(data_insert_.init(param, &scan_merge_))) { + LOG_WARN("fail to init data insert", KR(ret)); + } else { + is_inited_ = true; + } + } + + return ret; +} + +int ObDirectLoadSSTableDataInsert::get_next_row(const ObDatumRow *&datum_row) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), KP(this)); + } else { + ret = data_insert_.get_next_row(datum_row); + } + + return ret; +} + +/** + * ObDirectLoadMultipleSSTableDataInsert + */ + +ObDirectLoadMultipleSSTableDataInsert::ObDirectLoadMultipleSSTableDataInsert() + : is_inited_(false) +{ +} + +ObDirectLoadMultipleSSTableDataInsert::~ObDirectLoadMultipleSSTableDataInsert() +{ +} + +int ObDirectLoadMultipleSSTableDataInsert::init( + const ObDirectLoadDataInsertParam ¶m, + const ObIArray &sstable_array, + const ObDatumRange &range) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", KR(ret), KP(this)); + } else if (OB_UNLIKELY(!param.is_valid()) || !range.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid argument", K(ret), K(param), K(range)); + } else { + ObDirectLoadMultipleSSTableScanMergeParam scan_merge_param; + scan_merge_param.table_data_desc_ = param.table_data_desc_; + scan_merge_param.datum_utils_ = param.datum_utils_; + scan_merge_param.dml_row_handler_ = param.dml_row_handler_; + if (OB_FAIL(range_.assign(param.tablet_id_, range))) { + LOG_WARN("fail to assign range", KR(ret)); + } else if (OB_FAIL(scan_merge_.init(scan_merge_param, sstable_array, range_))) { + LOG_WARN("fail to init scan merge", KR(ret)); + } else if (OB_FAIL(data_insert_.init(param, &scan_merge_))) { + LOG_WARN("fail to init data insert", KR(ret)); + } else { + is_inited_ = true; + } + } + + return ret; +} + +int ObDirectLoadMultipleSSTableDataInsert::get_next_row(const ObDatumRow *&datum_row) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), KP(this)); + } else { + ret = data_insert_.get_next_row(datum_row); + } + + return ret; +} + +} // namespace storage +} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_data_insert.h b/src/storage/direct_load/ob_direct_load_data_insert.h new file mode 100644 index 0000000000..21f38fe428 --- /dev/null +++ b/src/storage/direct_load/ob_direct_load_data_insert.h @@ -0,0 +1,95 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#pragma once + +#include "storage/blocksstable/ob_sstable.h" +#include "share/table/ob_table_load_define.h" +#include "sql/resolver/cmd/ob_load_data_stmt.h" +#include "storage/direct_load/ob_direct_load_dml_row_handler.h" +#include "storage/direct_load/ob_direct_load_multiple_datum_range.h" +#include "storage/direct_load/ob_direct_load_multiple_sstable_scan_merge.h" +#include "storage/direct_load/ob_direct_load_sstable_scan_merge.h" +#include "storage/direct_load/ob_direct_load_struct.h" + +namespace oceanbase +{ +namespace storage +{ +class ObDirectLoadSSTable; +class ObDirectLoadMultipleSSTable; + +struct ObDirectLoadDataInsertParam +{ +public: + ObDirectLoadDataInsertParam(); + ~ObDirectLoadDataInsertParam(); + bool is_valid() const; + TO_STRING_KV(K_(tablet_id), K_(store_column_count), K_(table_data_desc), KP_(datum_utils), + KP_(dml_row_handler)); +public: + common::ObTabletID tablet_id_; + int64_t store_column_count_; + ObDirectLoadTableDataDesc table_data_desc_; + const blocksstable::ObStorageDatumUtils *datum_utils_; + ObDirectLoadDMLRowHandler *dml_row_handler_; +}; + +class ObDirectLoadDataInsert +{ +public: + ObDirectLoadDataInsert(); + ~ObDirectLoadDataInsert(); + int init( + const ObDirectLoadDataInsertParam ¶m, + ObIStoreRowIterator *load_iter); + int get_next_row(const blocksstable::ObDatumRow *&datum_row); +private: + ObDirectLoadDataInsertParam param_; + ObIStoreRowIterator *load_iter_; + bool is_inited_; +}; + +class ObDirectLoadSSTableDataInsert : public ObIStoreRowIterator +{ +public: + ObDirectLoadSSTableDataInsert(); + ~ObDirectLoadSSTableDataInsert(); + int init( + const ObDirectLoadDataInsertParam ¶m, + const common::ObIArray &sstable_array, + const blocksstable::ObDatumRange &range); + int get_next_row(const blocksstable::ObDatumRow *&datum_row); +private: + ObDirectLoadSSTableScanMerge scan_merge_; + ObDirectLoadDataInsert data_insert_; + bool is_inited_; +}; + +class ObDirectLoadMultipleSSTableDataInsert : public ObIStoreRowIterator +{ +public: + ObDirectLoadMultipleSSTableDataInsert(); + ~ObDirectLoadMultipleSSTableDataInsert(); + int init( + const ObDirectLoadDataInsertParam ¶m, + const common::ObIArray &sstable_array, + const blocksstable::ObDatumRange &range); + int get_next_row(const blocksstable::ObDatumRow *&datum_row); +private: + ObDirectLoadMultipleDatumRange range_; + ObDirectLoadMultipleSSTableScanMerge scan_merge_; + ObDirectLoadDataInsert data_insert_; + bool is_inited_; +}; + +} // namespace storage +} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_external_multi_partition_table.cpp b/src/storage/direct_load/ob_direct_load_external_multi_partition_table.cpp index 1676bd38ac..d7165e9990 100644 --- a/src/storage/direct_load/ob_direct_load_external_multi_partition_table.cpp +++ b/src/storage/direct_load/ob_direct_load_external_multi_partition_table.cpp @@ -210,6 +210,8 @@ int ObDirectLoadExternalMultiPartitionTableBuilder::get_tables( } else if (OB_UNLIKELY(!is_closed_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("direct load external table not closed", KR(ret)); + } else if (total_row_count_ == 0) { + // do nothing } else { ObDirectLoadExternalTableCreateParam create_param; create_param.tablet_id_ = 0; //因为包含了所有的tablet_id，设置为一个无效值 diff --git a/src/storage/direct_load/ob_direct_load_external_table_builder.cpp b/src/storage/direct_load/ob_direct_load_external_table_builder.cpp index a3ef003640..dbef108e6c 100644 --- a/src/storage/direct_load/ob_direct_load_external_table_builder.cpp +++ b/src/storage/direct_load/ob_direct_load_external_table_builder.cpp @@ -145,6 +145,8 @@ int ObDirectLoadExternalTableBuilder::get_tables( } else if (OB_UNLIKELY(!is_closed_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("direct load external table not closed", KR(ret)); + } else if (row_count_ == 0) { + // do nothing } else { ObDirectLoadExternalTableCreateParam create_param; create_param.tablet_id_ = build_param_.tablet_id_; diff --git a/src/storage/direct_load/ob_direct_load_fast_heap_table_builder.cpp b/src/storage/direct_load/ob_direct_load_fast_heap_table_builder.cpp index 7c4d27eed4..ba6766825c 100644 --- a/src/storage/direct_load/ob_direct_load_fast_heap_table_builder.cpp +++ b/src/storage/direct_load/ob_direct_load_fast_heap_table_builder.cpp @@ -261,6 +261,8 @@ int ObDirectLoadFastHeapTableBuilder::get_tables( } else if (OB_UNLIKELY(!is_closed_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("fast heap table builder not closed", KR(ret)); + } else if (row_count_ == 0) { + // do nothing } else { ObDirectLoadFastHeapTableCreateParam create_param; create_param.tablet_id_ = param_.tablet_id_; diff --git a/src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp b/src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp index 8c296f5d92..664c3f7ab0 100644 --- a/src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp +++ b/src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp @@ -19,6 +19,7 @@ #include "share/table/ob_table_load_sql_statistics.h" #include "share/stat/ob_stat_item.h" #include "storage/direct_load/ob_direct_load_origin_table.h" +#include "storage/lob/ob_lob_meta.h" namespace oceanbase { @@ -37,7 +38,6 @@ using namespace share; ObDirectLoadInsertTableParam::ObDirectLoadInsertTableParam() : table_id_(OB_INVALID_ID), - lob_meta_tid_(OB_INVALID_ID), schema_version_(OB_INVALID_VERSION), snapshot_version_(0), ddl_task_id_(0), @@ -147,56 +147,10 @@ int ObDirectLoadInsertTabletContext::init(ObDirectLoadInsertTableContext *table_ tablet_id_ = tablet_id; lob_tablet_id_ = ddl_data.lob_meta_tablet_id_; start_seq_.set_parallel_degree(param_->reserved_parallel_); - if (param_->is_incremental_ && OB_FAIL(check_lob_meta_empty())) { - LOG_WARN("failed to check lob meta empty", KR(ret)); - } else { - is_inited_ = true; + if (need_del_lob()) { + lob_start_seq_.set_parallel_degree(param_->parallel_); } - } - } - return ret; -} - -int ObDirectLoadInsertTabletContext::check_lob_meta_empty() -{ - int ret = OB_SUCCESS; - uint64_t lob_meta_tid = param_->lob_meta_tid_; - if (OB_INVALID_ID == lob_meta_tid) { - // bypass - } else { - ObArenaAllocator iter_allocator("TLD_Iter"); - ObDirectLoadOriginTable origin_table; - ObIStoreRowIterator *lob_row_iter = nullptr; - const blocksstable::ObDatumRow *row = nullptr; - blocksstable::ObDatumRange range; - ObDirectLoadOriginTableCreateParam table_param; - table_param.table_id_ = lob_meta_tid; - table_param.tablet_id_ = lob_tablet_id_; - table_param.ls_id_ = ls_id_; - table_param.insert_mode_ = ObDirectLoadInsertMode::NORMAL; - iter_allocator.set_tenant_id(MTL_ID()); - range.set_whole_range(); - - if (OB_FAIL(origin_table.init(table_param))) { - LOG_WARN("failed to init origin table", KR(ret)); - } else if (OB_FAIL(origin_table.scan(range, iter_allocator, lob_row_iter))) { - LOG_WARN("fail to scan origin table", KR(ret)); - } else if (OB_FAIL(lob_row_iter->get_next_row(row))) { - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; // empty table - } else { - LOG_WARN("failed to get next row", KR(ret)); - } - } else { - ret = OB_NOT_SUPPORTED; - LOG_WARN("incremental direct-load to table with outrow lob data is not supported", KR(ret)); - FORWARD_USER_ERROR(ret, "incremental direct-load to table with outrow lob data is not supported"); - } - // release row iter - if (OB_NOT_NULL(lob_row_iter)) { - lob_row_iter->~ObIStoreRowIterator(); - iter_allocator.free(lob_row_iter); - lob_row_iter = nullptr; + is_inited_ = true; } } return ret; @@ -408,6 +362,35 @@ int ObDirectLoadInsertTabletContext::init_datum_row(ObDatumRow &datum_row) return ret; } +int ObDirectLoadInsertTabletContext::init_lob_datum_row(blocksstable::ObDatumRow &datum_row, const bool is_delete) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadInsertTableContext not init", K(ret), KP(this)); + } else { + const int64_t rowkey_column_count = ObLobMetaUtil::LOB_META_SCHEMA_ROWKEY_COL_CNT; + const int64_t real_column_count = + ObLobMetaUtil::LOB_META_COLUMN_CNT + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); + if (OB_FAIL(datum_row.init(real_column_count))) { + LOG_WARN("fail to init datum row", KR(ret), K(real_column_count)); + } else { + const int64_t trans_version = + !param_->is_incremental_ ? param_->snapshot_version_ : INT64_MAX; + datum_row.trans_id_ = param_->trans_param_.tx_id_; + datum_row.row_flag_.set_flag(is_delete ? ObDmlFlag::DF_DELETE : ObDmlFlag::DF_INSERT); + datum_row.mvcc_row_flag_.set_last_multi_version_row(true); + datum_row.mvcc_row_flag_.set_uncommitted_row(param_->is_incremental_); + // fill trans_version + datum_row.storage_datums_[rowkey_column_count].set_int(-trans_version); + // fill sql_no + datum_row.storage_datums_[rowkey_column_count + 1].set_int( + -param_->trans_param_.tx_seq_.cast_to_int()); + } + } + return ret; +} + int ObDirectLoadInsertTabletContext::fill_sstable_slice(const int64_t &slice_id, ObIStoreRowIterator &iter, int64_t &affected_rows) @@ -436,6 +419,40 @@ int ObDirectLoadInsertTabletContext::fill_sstable_slice(const int64_t &slice_id, return ret; } +int ObDirectLoadInsertTabletContext::fill_lob_meta_sstable_slice(const int64_t &lob_slice_id, + ObIStoreRowIterator &iter, + int64_t &affected_rows) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadInsertTableContext not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(is_cancel_)) { + ret = OB_CANCELED; + LOG_WARN("task is cancel", KR(ret)); + } else { + ObDirectLoadInsertTabletContext *tablet_ctx = nullptr; + ObTenantDirectLoadMgr *sstable_insert_mgr = MTL(ObTenantDirectLoadMgr *); + ObDirectLoadSliceInfo slice_info; + slice_info.is_full_direct_load_ = !param_->is_incremental_; + slice_info.is_lob_slice_ = true; + slice_info.ls_id_ = ls_id_; + slice_info.data_tablet_id_ = tablet_id_; + slice_info.slice_id_ = lob_slice_id; + slice_info.context_id_ = context_id_; + if (OB_UNLIKELY(!handle_.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected invalid handle", KR(ret)); + } else if (OB_FAIL(handle_.get_obj()->fill_lob_meta_sstable_slice(slice_info, + start_scn_, + &iter, + affected_rows))) { + LOG_WARN("fail to fill lob meta sstable slice", KR(ret), K(slice_info)); + } + } + return ret; +} + int ObDirectLoadInsertTabletContext::fill_lob_sstable_slice(ObIAllocator &allocator, const int64_t &lob_slice_id, ObTabletCacheInterval &pk_interval, diff --git a/src/storage/direct_load/ob_direct_load_insert_table_ctx.h b/src/storage/direct_load/ob_direct_load_insert_table_ctx.h index 8b8f5ac65d..5a843233a6 100644 --- a/src/storage/direct_load/ob_direct_load_insert_table_ctx.h +++ b/src/storage/direct_load/ob_direct_load_insert_table_ctx.h @@ -59,7 +59,6 @@ public: ~ObDirectLoadInsertTableParam(); bool is_valid() const; TO_STRING_KV(K_(table_id), - K_(lob_meta_tid), K_(schema_version), K_(snapshot_version), K_(ddl_task_id), @@ -81,7 +80,6 @@ public: public: uint64_t table_id_; // dest_table_id - uint64_t lob_meta_tid_; // dest_table_lob_meta_tid int64_t schema_version_; int64_t snapshot_version_; int64_t ddl_task_id_; @@ -153,6 +151,7 @@ public: OB_INLINE bool has_lob_storage() const { return get_lob_column_count() > 0; } OB_INLINE bool need_rescan() const { return nullptr != param_ ? (!param_->is_incremental_ && param_->is_column_store_) : false; } + OB_INLINE bool need_del_lob() const { return nullptr != param_ ? (param_->is_incremental_ && param_->lob_column_count_ > 0) : false; } public: int init(ObDirectLoadInsertTableContext *table_ctx, @@ -165,6 +164,7 @@ public: int get_write_ctx(ObDirectLoadInsertTabletWriteCtx &write_ctx); int get_lob_write_ctx(ObDirectLoadInsertTabletWriteCtx &write_ctx); int init_datum_row(blocksstable::ObDatumRow &datum_row); + int init_lob_datum_row(blocksstable::ObDatumRow &datum_row, const bool is_delete = true); int open_sstable_slice(const blocksstable::ObMacroDataSeq &start_seq, int64_t &slice_id); int close_sstable_slice(const int64_t slice_id); int fill_sstable_slice(const int64_t &slice_id, ObIStoreRowIterator &iter, @@ -174,6 +174,9 @@ public: int fill_lob_sstable_slice(ObIAllocator &allocator, const int64_t &lob_slice_id, share::ObTabletCacheInterval &pk_interval, blocksstable::ObDatumRow &datum_row); + int fill_lob_meta_sstable_slice(const int64_t &lob_slice_id, + ObIStoreRowIterator &iter, + int64_t &affected_rows); int calc_range(const int64_t thread_cnt); int fill_column_group(const int64_t thread_cnt, const int64_t thread_id); @@ -195,7 +198,6 @@ private: int get_pk_interval(uint64_t count, share::ObTabletCacheInterval &pk_interval); int get_lob_pk_interval(uint64_t count, share::ObTabletCacheInterval &pk_interval); int refresh_pk_cache(const common::ObTabletID &tablet_id, share::ObTabletCacheInterval &pk_cache); - int check_lob_meta_empty(); private: ObDirectLoadInsertTableContext *table_ctx_; const ObDirectLoadInsertTableParam *param_; @@ -241,6 +243,7 @@ public: OB_INLINE int64_t get_context_id() const { return ddl_ctrl_.context_id_; } OB_INLINE bool need_rescan() const { return (!param_.is_incremental_ && param_.is_column_store_); } + OB_INLINE bool need_del_lob() const { return (param_.is_incremental_ && param_.lob_column_count_ > 0); } int64_t get_sql_stat_column_count() const; int get_sql_statistics(table::ObTableLoadSqlStatistics *&sql_statistics); diff --git a/src/storage/direct_load/ob_direct_load_lob_meta_row_iter.cpp b/src/storage/direct_load/ob_direct_load_lob_meta_row_iter.cpp new file mode 100644 index 0000000000..c766c10a45 --- /dev/null +++ b/src/storage/direct_load/ob_direct_load_lob_meta_row_iter.cpp @@ -0,0 +1,183 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "storage/direct_load/ob_direct_load_lob_meta_row_iter.h" +#include "storage/lob/ob_lob_meta.h" + +namespace oceanbase +{ +namespace storage +{ +using namespace common; +using namespace blocksstable; +using namespace share; + +ObDirectLoadLobMetaIterParam::ObDirectLoadLobMetaIterParam() + : datum_utils_(nullptr), col_descs_(nullptr) +{ +} + +ObDirectLoadLobMetaIterParam::~ObDirectLoadLobMetaIterParam() {} + +bool ObDirectLoadLobMetaIterParam::is_valid() const +{ + return tablet_id_.is_valid() && + table_data_desc_.is_valid() && + OB_NOT_NULL(datum_utils_) && datum_utils_->is_valid() && + OB_NOT_NULL(col_descs_); +} + +ObDirectLoadLobMetaRowIter::ObDirectLoadLobMetaRowIter() + : allocator_("TLD_LobRowIter"), + origin_table_(nullptr), + origin_iter_(nullptr), + range_allocator_("TLD_LobRange"), + is_inited_(false) +{ + allocator_.set_tenant_id(MTL_ID()); + range_allocator_.set_tenant_id(MTL_ID()); +} + +ObDirectLoadLobMetaRowIter::~ObDirectLoadLobMetaRowIter() +{ + if (nullptr != origin_iter_) { + origin_iter_->~ObIStoreRowIterator(); + allocator_.free(origin_iter_); + origin_iter_ = nullptr; + } +} + +int ObDirectLoadLobMetaRowIter::init(const ObDirectLoadLobMetaIterParam ¶m, + ObDirectLoadOriginTable *origin_table, + const ObIArray &sstable_array, + const ObDatumRange &range) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("get not inited error", K(ret)); + } else { + ObDirectLoadMultipleSSTableScanMergeParam scan_merge_param; + scan_merge_param.table_data_desc_ = param.table_data_desc_; + scan_merge_param.datum_utils_ = param.datum_utils_; + scan_merge_param.dml_row_handler_ = &conflict_handler_; + if (OB_FAIL(init_range())) { + LOG_WARN("fail to init range", KR(ret)); + } else if (OB_FAIL(scan_range_.assign(param.tablet_id_, range))) { + LOG_WARN("fail to assign range", KR(ret)); + } else if (OB_FAIL(scan_merge_.init(scan_merge_param, sstable_array, scan_range_))) { + LOG_WARN("fail to init scan merge ", KR(ret)); + } else { + param_ = param; + origin_table_ = origin_table; + is_inited_ = true; + } + } + return ret; +} + +int ObDirectLoadLobMetaRowIter::init_range() +{ + int ret = OB_SUCCESS; + ObStorageDatum *datums = nullptr; + const int64_t count = ObLobMetaUtil::LOB_META_SCHEMA_ROWKEY_COL_CNT * 2; + void *buf = nullptr; + if (OB_ISNULL(buf = allocator_.alloc(sizeof(ObStorageDatum) * count))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc buf", KR(ret)); + } else { + datums = new (buf) ObStorageDatum[count]; + range_.start_key_.assign(datums, ObLobMetaUtil::LOB_META_SCHEMA_ROWKEY_COL_CNT); + range_.end_key_.assign(datums + ObLobMetaUtil::LOB_META_SCHEMA_ROWKEY_COL_CNT, + ObLobMetaUtil::LOB_META_SCHEMA_ROWKEY_COL_CNT); + range_.start_key_.datums_[ObLobMetaUtil::SEQ_ID_COL_ID].set_min(); + range_.end_key_.datums_[ObLobMetaUtil::SEQ_ID_COL_ID].set_max(); + range_.set_left_open(); + range_.set_right_open(); + } + return ret; +} + +int ObDirectLoadLobMetaRowIter::get_next_row(const ObDatumRow *&datum_row) +{ + int ret = OB_SUCCESS; + datum_row = nullptr; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("get not inited error", K(ret)); + } else { + if (OB_ISNULL(origin_iter_)) { + if (OB_FAIL(switch_next_lob_id())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("fail to switch next lob id", KR(ret)); + } + } + } + while (OB_SUCC(ret) && nullptr == datum_row) { + if (OB_FAIL(origin_iter_->get_next_row(datum_row))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("fail to get next row", KR(ret)); + } else { + ret = OB_SUCCESS; + if (OB_FAIL(switch_next_lob_id())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("fail to switch next lob id", KR(ret)); + } + } + } + } + } + } + return ret; +} + +int ObDirectLoadLobMetaRowIter::switch_next_lob_id() +{ + int ret = OB_SUCCESS; + const ObDatumRow *row = nullptr; + if (OB_FAIL(scan_merge_.get_next_row(row))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("fail to get next lob", K(ret)); + } + } else if (OB_UNLIKELY(row->count_ != 1)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected datum row count", KR(ret), KPC(row)); + } else { + range_allocator_.reuse(); + range_.start_key_.datums_[ObLobMetaUtil::LOB_ID_COL_ID] = row->storage_datums_[ObLobMetaUtil::LOB_ID_COL_ID]; + range_.end_key_.datums_[ObLobMetaUtil::LOB_ID_COL_ID] = row->storage_datums_[ObLobMetaUtil::LOB_ID_COL_ID]; + if (OB_FAIL( + range_.start_key_.prepare_memtable_readable(*param_.col_descs_, range_allocator_))) { + LOG_WARN("fail to prepare memtable readable", KR(ret), K(range_)); + } else if (OB_FAIL( + range_.end_key_.prepare_memtable_readable(*param_.col_descs_, range_allocator_))) { + LOG_WARN("fail to prepare memtable readable", KR(ret), K(range_)); + } else { + if (nullptr == origin_iter_) { + if (OB_FAIL( + origin_table_->scan(range_, allocator_, origin_iter_, true /*skip_read_lob*/))) { + LOG_WARN("fail to scan origin table", KR(ret)); + } + } else { + if (OB_FAIL(origin_table_->rescan(range_, origin_iter_))) { + LOG_WARN("fail to scan origin table", KR(ret)); + } + } + } + } + return ret; +} + +} // namespace storage +} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_lob_meta_row_iter.h b/src/storage/direct_load/ob_direct_load_lob_meta_row_iter.h new file mode 100644 index 0000000000..0ff1690215 --- /dev/null +++ b/src/storage/direct_load/ob_direct_load_lob_meta_row_iter.h @@ -0,0 +1,94 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#pragma once + +#include "storage/blocksstable/ob_datum_range.h" +#include "storage/direct_load/ob_direct_load_dml_row_handler.h" +#include "storage/direct_load/ob_direct_load_multiple_datum_range.h" +#include "storage/direct_load/ob_direct_load_multiple_sstable_scan_merge.h" +#include "storage/direct_load/ob_direct_load_origin_table.h" + +namespace oceanbase +{ +namespace storage +{ +class ObDirectLoadMultipleSSTable; + +class ObDirectLoadLobIdConflictHandler : public ObDirectLoadDMLRowHandler +{ +public: + ObDirectLoadLobIdConflictHandler() {} + virtual ~ObDirectLoadLobIdConflictHandler() {} + int handle_insert_row(const blocksstable::ObDatumRow &row) override { return OB_SUCCESS; } + int handle_update_row(const blocksstable::ObDatumRow &row) override { return OB_ERR_UNEXPECTED; } + int handle_update_row(common::ObArray &rows, + const ObDirectLoadExternalRow *&row) override + { + return OB_ERR_UNEXPECTED; + } + int handle_update_row(common::ObArray &rows, + const ObDirectLoadMultipleDatumRow *&row) override + { + return OB_ERR_UNEXPECTED; + } + int handle_update_row(const blocksstable::ObDatumRow &old_row, + const blocksstable::ObDatumRow &new_row, + const blocksstable::ObDatumRow *&result_row) override + { + return OB_ERR_UNEXPECTED; + } + TO_STRING_EMPTY(); +}; + +struct ObDirectLoadLobMetaIterParam +{ +public: + ObDirectLoadLobMetaIterParam(); + ~ObDirectLoadLobMetaIterParam(); + bool is_valid() const; + +public: + ObTabletID tablet_id_; + ObDirectLoadTableDataDesc table_data_desc_; + const blocksstable::ObStorageDatumUtils *datum_utils_; + const common::ObIArray *col_descs_; +}; + +class ObDirectLoadLobMetaRowIter +{ +public: + ObDirectLoadLobMetaRowIter(); + ~ObDirectLoadLobMetaRowIter(); + int init(const ObDirectLoadLobMetaIterParam ¶m, ObDirectLoadOriginTable *origin_table, + const common::ObIArray &sstable_array, + const blocksstable::ObDatumRange &range); + int get_next_row(const blocksstable::ObDatumRow *&datum_row); + +private: + int init_range(); + int switch_next_lob_id(); + +private: + common::ObArenaAllocator allocator_; + ObDirectLoadLobMetaIterParam param_; + ObDirectLoadMultipleDatumRange scan_range_; + ObDirectLoadMultipleSSTableScanMerge scan_merge_; + ObDirectLoadOriginTable *origin_table_; + ObIStoreRowIterator *origin_iter_; + common::ObArenaAllocator range_allocator_; + blocksstable::ObDatumRange range_; + ObDirectLoadLobIdConflictHandler conflict_handler_; + bool is_inited_; +}; + +} // namespace storage +} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_merge_ctx.cpp b/src/storage/direct_load/ob_direct_load_merge_ctx.cpp index 2aefcedc7a..8b458abd10 100644 --- a/src/storage/direct_load/ob_direct_load_merge_ctx.cpp +++ b/src/storage/direct_load/ob_direct_load_merge_ctx.cpp @@ -18,10 +18,14 @@ #include "storage/direct_load/ob_direct_load_multiple_heap_table.h" #include "storage/direct_load/ob_direct_load_multiple_heap_table_index_block.h" #include "storage/direct_load/ob_direct_load_multiple_sstable.h" -#include "storage/direct_load/ob_direct_load_partition_merge_task.h" #include "storage/direct_load/ob_direct_load_range_splitter.h" +#include "storage/direct_load/ob_direct_load_partition_del_lob_task.h" +#include "storage/direct_load/ob_direct_load_partition_merge_task.h" #include "storage/direct_load/ob_direct_load_partition_rescan_task.h" +#include "storage/direct_load/ob_direct_load_tmp_file.h" +#include "storage/direct_load/ob_direct_load_external_multi_partition_table.h" #include "observer/table_load/ob_table_load_schema.h" +#include "storage/tx_storage/ob_ls_service.h" namespace oceanbase { @@ -40,18 +44,23 @@ using namespace observer; ObDirectLoadMergeParam::ObDirectLoadMergeParam() : table_id_(OB_INVALID_ID), + lob_meta_table_id_(OB_INVALID_ID), target_table_id_(OB_INVALID_ID), rowkey_column_num_(0), store_column_count_(0), fill_cg_thread_cnt_(0), + lob_column_idxs_(nullptr), datum_utils_(nullptr), col_descs_(nullptr), + lob_meta_datum_utils_(nullptr), + lob_meta_col_descs_(nullptr), is_heap_table_(false), is_fast_heap_table_(false), is_incremental_(false), insert_mode_(ObDirectLoadInsertMode::INVALID_INSERT_MODE), insert_table_ctx_(nullptr), - dml_row_handler_(nullptr) + dml_row_handler_(nullptr), + file_mgr_(nullptr) { } @@ -62,12 +71,13 @@ ObDirectLoadMergeParam::~ObDirectLoadMergeParam() bool ObDirectLoadMergeParam::is_valid() const { return OB_INVALID_ID != table_id_ && OB_INVALID_ID != target_table_id_ && - 0 < rowkey_column_num_ && 0 < store_column_count_ && table_data_desc_.is_valid() && - nullptr != datum_utils_ && nullptr != col_descs_ && + 0 < rowkey_column_num_ && 0 < store_column_count_ && nullptr != lob_column_idxs_ && + table_data_desc_.is_valid() && nullptr != datum_utils_ && nullptr != col_descs_ && + lob_id_table_data_desc_.is_valid() && nullptr != lob_meta_datum_utils_ && nullptr != lob_meta_col_descs_ && ObDirectLoadInsertMode::is_type_valid(insert_mode_) && (ObDirectLoadInsertMode::INC_REPLACE == insert_mode_ ? is_incremental_ : true) && (ObDirectLoadInsertMode::OVERWRITE == insert_mode_ ? !is_incremental_ : true) && - nullptr != insert_table_ctx_ && nullptr != dml_row_handler_; + nullptr != insert_table_ctx_ && nullptr != dml_row_handler_ && nullptr != file_mgr_; } /** @@ -89,12 +99,18 @@ ObDirectLoadMergeCtx::~ObDirectLoadMergeCtx() allocator_.free(tablet_ctx); } tablet_merge_ctx_array_.reset(); + FOREACH(iter, table_builder_map_) + { + ObIDirectLoadPartitionTableBuilder *table_builder = iter->second; + table_builder->~ObIDirectLoadPartitionTableBuilder(); + allocator_.free(table_builder); + } + table_builder_map_.destroy(); } int ObDirectLoadMergeCtx::init(ObTableLoadTableCtx *ctx, const ObDirectLoadMergeParam ¶m, - const ObIArray &ls_partition_ids, - const ObIArray &target_ls_partition_ids) + const ObIArray &ls_partition_ids) { int ret = OB_SUCCESS; if (IS_INIT) { @@ -102,15 +118,15 @@ int ObDirectLoadMergeCtx::init(ObTableLoadTableCtx *ctx, LOG_WARN("ObDirectLoadMerger init twice", KR(ret), KP(this)); } else if (OB_UNLIKELY(nullptr == ctx || !param.is_valid() - || ls_partition_ids.empty() - || target_ls_partition_ids.empty() - || (ls_partition_ids.count() != target_ls_partition_ids.count()))) { + || ls_partition_ids.empty())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(param), K(ls_partition_ids), K(target_ls_partition_ids)); + LOG_WARN("invalid args", KR(ret), KP(ctx), K(param), K(ls_partition_ids)); + } else if (OB_FAIL(table_builder_map_.create(1024, "TLD_TblBd", "TLD_TblBd", MTL_ID()))) { + LOG_WARN("fail to create table_builder_map_", KR(ret)); } else { ctx_ = ctx; param_ = param; - if (OB_FAIL(create_all_tablet_ctxs(ls_partition_ids, target_ls_partition_ids))) { + if (OB_FAIL(create_all_tablet_ctxs(ls_partition_ids))) { LOG_WARN("fail to create all tablet ctxs", KR(ret)); } else { std::sort(tablet_merge_ctx_array_.begin(), tablet_merge_ctx_array_.end(), @@ -123,20 +139,83 @@ int ObDirectLoadMergeCtx::init(ObTableLoadTableCtx *ctx, return ret; } +int ObDirectLoadMergeCtx::get_table_builder(ObIDirectLoadPartitionTableBuilder *&table_builder) +{ + int ret = OB_SUCCESS; + table_builder = nullptr; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), KP(this)); + } else { + const int64_t part_id = get_tid_cache(); + if (OB_FAIL(table_builder_map_.get_refactored(part_id, table_builder))) { + if (OB_UNLIKELY(OB_HASH_NOT_EXIST != ret)) { + LOG_WARN("fail to get table builder", KR(ret), K(part_id)); + } else { + ret = OB_SUCCESS; + ObDirectLoadExternalMultiPartitionTableBuildParam builder_param; + builder_param.table_data_desc_ = param_.table_data_desc_; + builder_param.datum_utils_ = param_.datum_utils_; + builder_param.file_mgr_ = param_.file_mgr_; + builder_param.extra_buf_ = reinterpret_cast(1); // unuse, delete in future + builder_param.extra_buf_size_ = 4096; // unuse, delete in future + builder_param.table_data_desc_.rowkey_column_num_ = 1; // only a column of lobid + builder_param.table_data_desc_.column_count_ = 1; // only a column of lobid + { + ObMutexGuard guard(mutex_); + // use mutext to avoid thread unsafe allocator; + ObDirectLoadExternalMultiPartitionTableBuilder *new_builder = nullptr; + if (OB_ISNULL(new_builder = OB_NEWx( + ObDirectLoadExternalMultiPartitionTableBuilder, &allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObDirectLoadExternalMultiPartitionTableBuilder", KR(ret)); + } else if (OB_FAIL(new_builder->init(builder_param))) { + LOG_WARN("fail to init new_builder", KR(ret)); + } else if (OB_FAIL(table_builder_map_.set_refactored(part_id, new_builder))) { + LOG_WARN("fail to set table_builder_map_", KR(ret), K(part_id)); + } else { + table_builder = new_builder; + } + if (OB_FAIL(ret)) { + if (nullptr != new_builder) { + new_builder->~ObDirectLoadExternalMultiPartitionTableBuilder(); + allocator_.free(new_builder); + new_builder = nullptr; + } + } + } + } + } + } + + return ret; +} + +int ObDirectLoadMergeCtx::close_table_builder() +{ + int ret = OB_SUCCESS; + FOREACH_X(item, table_builder_map_, OB_SUCC(ret)) { + if (item->second != nullptr) { + if (OB_FAIL(item->second->close())) { + LOG_WARN("fail to close table", KR(ret)); + } + } + } + return ret; +} + int ObDirectLoadMergeCtx::create_all_tablet_ctxs( - const ObIArray &ls_partition_ids, - const ObIArray &target_ls_partition_ids) + const ObIArray &ls_partition_ids) { int ret = OB_SUCCESS; for (int64_t i = 0; OB_SUCC(ret) && i < ls_partition_ids.count(); ++i) { const ObTableLoadLSIdAndPartitionId &ls_partition_id = ls_partition_ids.at(i); - const ObTableLoadLSIdAndPartitionId &target_ls_partition_id = target_ls_partition_ids.at(i); ObDirectLoadTabletMergeCtx *partition_ctx = nullptr; if (OB_ISNULL(partition_ctx = OB_NEWx(ObDirectLoadTabletMergeCtx, (&allocator_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to new ObDirectLoadTabletMergeCtx", KR(ret)); - } else if (OB_FAIL(partition_ctx->init(ctx_, param_, ls_partition_id, target_ls_partition_id))) { - LOG_WARN("fail to init tablet ctx", KR(ret), K(param_), K(ls_partition_id), K(target_ls_partition_id)); + } else if (OB_FAIL(partition_ctx->init(this, ctx_, param_, ls_partition_id))) { + LOG_WARN("fail to init tablet ctx", KR(ret), K(param_), K(ls_partition_id)); } else if (OB_FAIL(tablet_merge_ctx_array_.push_back(partition_ctx))) { LOG_WARN("fail to push back", KR(ret)); } @@ -156,19 +235,30 @@ int ObDirectLoadMergeCtx::create_all_tablet_ctxs( */ ObDirectLoadTabletMergeCtx::ObDirectLoadTabletMergeCtx() - : allocator_("TLD_MegTbtCtx"), task_finish_count_(0), rescan_task_finish_count_(0), is_inited_(false) + : allocator_("TLD_MegTbtCtx"), + merge_ctx_(nullptr), + ctx_(nullptr), + task_finish_count_(0), + rescan_task_finish_count_(0), + del_lob_task_finish_count_(0), + is_inited_(false) { allocator_.set_tenant_id(MTL_ID()); sstable_array_.set_tenant_id(MTL_ID()); multiple_sstable_array_.set_tenant_id(MTL_ID()); multiple_heap_table_array_.set_tenant_id(MTL_ID()); + del_lob_multiple_sstable_array_.set_tenant_id(MTL_ID()); range_array_.set_tenant_id(MTL_ID()); + del_lob_range_array_.set_tenant_id(MTL_ID()); task_array_.set_tenant_id(MTL_ID()); rescan_task_array_.set_tenant_id(MTL_ID()); + del_lob_task_array_.set_tenant_id(MTL_ID()); } ObDirectLoadTabletMergeCtx::~ObDirectLoadTabletMergeCtx() { + merge_ctx_ = nullptr; + ctx_ = nullptr; for (int64_t i = 0; i < task_array_.count(); ++i) { ObDirectLoadPartitionMergeTask *task = task_array_.at(i); task->~ObDirectLoadPartitionMergeTask(); @@ -179,40 +269,76 @@ ObDirectLoadTabletMergeCtx::~ObDirectLoadTabletMergeCtx() task->~ObDirectLoadPartitionRescanTask(); allocator_.free(task); } + for (int64_t i = 0; i < del_lob_task_array_.count(); ++i) { + ObDirectLoadPartitionDelLobTask *task = del_lob_task_array_.at(i); + task->~ObDirectLoadPartitionDelLobTask(); + allocator_.free(task); + } task_array_.reset(); rescan_task_array_.reset(); + del_lob_task_array_.reset(); } -int ObDirectLoadTabletMergeCtx::init(ObTableLoadTableCtx *ctx, +int ObDirectLoadTabletMergeCtx::init(ObDirectLoadMergeCtx *merge_ctx, + ObTableLoadTableCtx *ctx, const ObDirectLoadMergeParam ¶m, - const ObTableLoadLSIdAndPartitionId &ls_partition_id, - const ObTableLoadLSIdAndPartitionId &target_ls_partition_id) + const ObTableLoadLSIdAndPartitionId &ls_partition_id) { int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("ObDirectLoadTabletMergeCtx init twice", KR(ret), KP(this)); - } else if (OB_UNLIKELY(nullptr == ctx - || !param.is_valid() - || !ls_partition_id.is_valid() - || !target_ls_partition_id.is_valid())) { + } else if (OB_UNLIKELY(nullptr == merge_ctx || nullptr == ctx || !param.is_valid() || + !ls_partition_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(param), K(ls_partition_id)); + LOG_WARN("invalid args", KR(ret), KP(merge_ctx), KP(ctx), K(param), K(ls_partition_id)); } else { - ObDirectLoadOriginTableCreateParam origin_table_param; - origin_table_param.table_id_ = param.table_id_; - origin_table_param.tablet_id_ = ls_partition_id.part_tablet_id_.tablet_id_; - origin_table_param.ls_id_ = ls_partition_id.ls_id_; - origin_table_param.insert_mode_ = param.insert_mode_; - if (OB_FAIL(origin_table_.init(origin_table_param))) { - LOG_WARN("fail to init origin sstable", KR(ret)); - } else { + const ObLSID &ls_id = ls_partition_id.ls_id_; + const ObTabletID &tablet_id = ls_partition_id.part_tablet_id_.tablet_id_; + ObTabletID lob_meta_tablet_id; + if (OB_INVALID_ID != param.lob_meta_table_id_) { + ObLSService *ls_service = nullptr; + ObLSHandle ls_handle; + ObTabletHandle tablet_handle; + ObTabletBindingMdsUserData ddl_data; + if (OB_ISNULL(ls_service = MTL(ObLSService *))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected err", K(ret), K(MTL_ID())); + } else if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { + LOG_WARN("failed to get log stream", K(ret)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, tablet_id, tablet_handle, + ObMDSGetTabletMode::READ_ALL_COMMITED))) { + LOG_WARN("get tablet handle failed", K(ret)); + } else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_data(SCN::max_scn(), ddl_data))) { + LOG_WARN("get ddl data failed", K(ret)); + } else { + lob_meta_tablet_id = ddl_data.lob_meta_tablet_id_; + } + } + if (OB_SUCC(ret)) { + ObDirectLoadOriginTableCreateParam origin_table_param; + origin_table_param.table_id_ = param.table_id_; + origin_table_param.ls_id_ = ls_id; + origin_table_param.tablet_id_ = tablet_id; + if (OB_FAIL(origin_table_.init(origin_table_param))) { + LOG_WARN("fail to init origin table", KR(ret)); + } + } + if (OB_SUCC(ret) && OB_INVALID_ID != param.lob_meta_table_id_) { + ObDirectLoadOriginTableCreateParam lob_meta_origin_table_param; + lob_meta_origin_table_param.table_id_ = param.lob_meta_table_id_; + lob_meta_origin_table_param.ls_id_ = ls_id; + lob_meta_origin_table_param.tablet_id_ = lob_meta_tablet_id; + if (OB_FAIL(lob_meta_origin_table_.init(lob_meta_origin_table_param))) { + LOG_WARN("fail to init lob meta origin table", KR(ret)); + } + } + if (OB_SUCC(ret)) { + merge_ctx_ = merge_ctx; ctx_ = ctx; param_ = param; - target_partition_id_ = target_ls_partition_id.part_tablet_id_.partition_id_; - tablet_id_ = ls_partition_id.part_tablet_id_.tablet_id_; - target_tablet_id_ = target_ls_partition_id.part_tablet_id_.tablet_id_; + tablet_id_ = tablet_id; is_inited_ = true; } } @@ -286,21 +412,22 @@ int ObDirectLoadTabletMergeCtx::build_merge_task( LOG_WARN("fail to build empty data merge task", KR(ret)); } } else if (!param_.is_heap_table_) { - // 有主键表排序和不排序都写成multiple sstable - // if (!is_multiple_mode) { - // if (OB_FAIL(build_pk_table_merge_task(table_array, col_descs, max_parallel_degree))) { - // LOG_WARN("fail to build pk table merge task", KR(ret)); - // } - // } else { - if (OB_FAIL( - build_pk_table_multiple_merge_task(table_array, col_descs, max_parallel_degree))) { - LOG_WARN("fail to build pk table multiple merge task", KR(ret)); + if (!is_multiple_mode) { + // 有主键表不排序也写成multiple sstable + if (OB_FAIL(build_pk_table_multiple_merge_task(table_array, col_descs, max_parallel_degree))) { + LOG_WARN("fail to build pk table merge task", KR(ret)); } - // } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("This should never be reached", KR(ret)); + } } else { if (!is_multiple_mode) { - if (OB_FAIL(build_heap_table_merge_task(table_array, col_descs, max_parallel_degree))) { - LOG_WARN("fail to build heap table merge task", KR(ret)); + if (!param_.is_fast_heap_table_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("This should never be reached", KR(ret)); + } else if (OB_FAIL(build_empty_data_merge_task(col_descs, max_parallel_degree))) { + LOG_WARN("fail to build empty table merge task", KR(ret)); } } else { if (OB_FAIL( @@ -318,17 +445,18 @@ int ObDirectLoadTabletMergeCtx::build_empty_data_merge_task(const ObIArray max_parallel_degree)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected range count", KR(ret), K(max_parallel_degree), K(range_array_.count())); - } + ObDirectLoadMergeRangeSplitter range_splitter; + if (OB_FAIL(range_splitter.init( + (merge_with_origin_data() ? &origin_table_ : nullptr), + sstable_array_, + param_.datum_utils_, + col_descs))) { + LOG_WARN("fail to init range splitter", KR(ret)); + } else if (OB_FAIL(range_splitter.split_range(range_array_, max_parallel_degree, allocator_))) { + LOG_WARN("fail to split range", KR(ret)); + } else if (OB_UNLIKELY(range_array_.count() > max_parallel_degree)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected range count", KR(ret), K(max_parallel_degree), K(range_array_.count())); } for (int64_t i = 0; OB_SUCC(ret) && i < range_array_.count(); ++i) { const ObDatumRange &range = range_array_.at(i); @@ -349,51 +477,7 @@ int ObDirectLoadTabletMergeCtx::build_empty_data_merge_task(const ObIArray &table_array, - const ObIArray &col_descs, - int64_t max_parallel_degree) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(init_sstable_array(table_array))) { - LOG_WARN("fail to init sstable array", KR(ret)); - } - // split range - if (OB_SUCC(ret)) { - ObDirectLoadMergeRangeSplitter range_splitter; - if (OB_FAIL( - range_splitter.init(&origin_table_, sstable_array_, param_.datum_utils_, col_descs))) { - LOG_WARN("fail to init range splitter", KR(ret)); - } else if (OB_FAIL(range_splitter.split_range(range_array_, max_parallel_degree, allocator_))) { - LOG_WARN("fail to split range", KR(ret)); - } else if (OB_UNLIKELY(range_array_.count() > max_parallel_degree)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected range count", KR(ret), K(max_parallel_degree), K(range_array_.count())); - } - } - // construct task per range - for (int64_t i = 0; OB_SUCC(ret) && i < range_array_.count(); ++i) { - const ObDatumRange &range = range_array_.at(i); - ObDirectLoadPartitionRangeMergeTask *merge_task = nullptr; - if (OB_ISNULL(merge_task = OB_NEWx(ObDirectLoadPartitionRangeMergeTask, (&allocator_)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to new ObDirectLoadPartitionRangeMergeTask", KR(ret)); - } else if (OB_FAIL(merge_task->init(ctx_, param_, this, &origin_table_, sstable_array_, range, i))) { - LOG_WARN("fail to init merge task", KR(ret)); - } else if (OB_FAIL(task_array_.push_back(merge_task))) { - LOG_WARN("fail to push back merge task", KR(ret)); - } - if (OB_FAIL(ret)) { - if (nullptr != merge_task) { - merge_task->~ObDirectLoadPartitionRangeMergeTask(); - allocator_.free(merge_task); - merge_task = nullptr; - } - } - } return ret; } @@ -409,8 +493,13 @@ int ObDirectLoadTabletMergeCtx::build_pk_table_multiple_merge_task( // split range if (OB_SUCC(ret)) { ObDirectLoadMultipleMergeTabletRangeSplitter range_splitter; - if (OB_FAIL(range_splitter.init(tablet_id_, &origin_table_, multiple_sstable_array_, - param_.table_data_desc_, param_.datum_utils_, col_descs))) { + if (OB_FAIL(range_splitter.init( + tablet_id_, + (merge_with_origin_data() ? &origin_table_ : nullptr), + multiple_sstable_array_, + param_.table_data_desc_, + param_.datum_utils_, + col_descs))) { LOG_WARN("fail to init range splitter", KR(ret)); } else if (OB_FAIL(range_splitter.split_range(range_array_, max_parallel_degree, allocator_))) { LOG_WARN("fail to split range", KR(ret)); @@ -459,8 +548,12 @@ int ObDirectLoadTabletMergeCtx::build_merge_task_for_multiple_pk_table( } else { if (OB_FAIL(multiple_sstable_array_.assign(multiple_sstable_array))) { LOG_WARN("fail to assign multiple sstable array", KR(ret)); - } else if (OB_FAIL(range_splitter.split_range(tablet_id_, &origin_table_, max_parallel_degree, - range_array_, allocator_))) { + } else if (OB_FAIL(range_splitter.split_range( + tablet_id_, + (merge_with_origin_data() ? &origin_table_ : nullptr), + max_parallel_degree, + range_array_, + allocator_))) { LOG_WARN("fail to split range", KR(ret)); } else if (OB_UNLIKELY(range_array_.count() > max_parallel_degree)) { ret = OB_ERR_UNEXPECTED; @@ -492,77 +585,6 @@ int ObDirectLoadTabletMergeCtx::build_merge_task_for_multiple_pk_table( return ret; } -int ObDirectLoadTabletMergeCtx::build_heap_table_merge_task( - const ObIArray &table_array, - const ObIArray &col_descs, - int64_t max_parallel_degree) -{ - int ret = OB_SUCCESS; - int64_t parallel_idx = 0; - // for existing data, construct task by split range - if (OB_SUCC(ret)) { - ObDirectLoadMergeRangeSplitter range_splitter; - if (OB_FAIL( - range_splitter.init(&origin_table_, sstable_array_, param_.datum_utils_, col_descs))) { - LOG_WARN("fail to init range splitter", KR(ret)); - } else if (OB_FAIL(range_splitter.split_range(range_array_, max_parallel_degree, allocator_))) { - LOG_WARN("fail to split range", KR(ret)); - } else if (OB_UNLIKELY(range_array_.count() > max_parallel_degree)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected range count", KR(ret), K(max_parallel_degree), K(range_array_.count())); - } - } - for (int64_t i = 0; OB_SUCC(ret) && i < range_array_.count(); ++i) { - const ObDatumRange &range = range_array_.at(i); - ObDirectLoadPartitionRangeMergeTask *merge_task = nullptr; - if (OB_ISNULL(merge_task = OB_NEWx(ObDirectLoadPartitionRangeMergeTask, (&allocator_)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to new ObDirectLoadPartitionRangeMergeTask", KR(ret)); - } else if (OB_FAIL(merge_task->init(ctx_, param_, this, &origin_table_, sstable_array_, range, - parallel_idx++))) { - LOG_WARN("fail to init merge task", KR(ret)); - } else if (OB_FAIL(task_array_.push_back(merge_task))) { - LOG_WARN("fail to push back merge task", KR(ret)); - } - if (OB_FAIL(ret)) { - if (nullptr != merge_task) { - merge_task->~ObDirectLoadPartitionRangeMergeTask(); - allocator_.free(merge_task); - merge_task = nullptr; - } - } - } - // for imported data, construct task per external table - for (int64_t i = 0; OB_SUCC(ret) && !param_.is_fast_heap_table_ && i < table_array.count(); ++i) { - ObDirectLoadExternalTable *external_table = nullptr; - ObDirectLoadPartitionHeapTableMergeTask *merge_task = nullptr; - ObTabletCacheInterval pk_interval; - if (OB_ISNULL(external_table = dynamic_cast(table_array.at(i)))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected table", KR(ret), K(i), K(table_array)); - } else if (OB_FAIL(get_autoincrement_value(external_table->get_row_count(), pk_interval))) { - LOG_WARN("fail to get autoincrement value", KR(ret), K(external_table->get_row_count())); - } else if (OB_ISNULL(merge_task = - OB_NEWx(ObDirectLoadPartitionHeapTableMergeTask, (&allocator_)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to new ObDirectLoadPartitionHeapTableMergeTask", KR(ret)); - } else if (OB_FAIL(merge_task->init(ctx_, param_, this, external_table, pk_interval, - parallel_idx++))) { - LOG_WARN("fail to init merge task", KR(ret)); - } else if (OB_FAIL(task_array_.push_back(merge_task))) { - LOG_WARN("fail to push back merge task", KR(ret)); - } - if (OB_FAIL(ret)) { - if (nullptr != merge_task) { - merge_task->~ObDirectLoadPartitionHeapTableMergeTask(); - allocator_.free(merge_task); - merge_task = nullptr; - } - } - } - return ret; -} - int ObDirectLoadTabletMergeCtx::build_heap_table_multiple_merge_task( const ObIArray &table_array, const ObIArray &col_descs, @@ -576,8 +598,11 @@ int ObDirectLoadTabletMergeCtx::build_heap_table_multiple_merge_task( // for existing data, construct task by split range if (OB_SUCC(ret)) { ObDirectLoadMergeRangeSplitter range_splitter; - if (OB_FAIL( - range_splitter.init(&origin_table_, sstable_array_, param_.datum_utils_, col_descs))) { + if (OB_FAIL(range_splitter.init( + (merge_with_origin_data() ? &origin_table_ : nullptr), + sstable_array_, + param_.datum_utils_, + col_descs))) { LOG_WARN("fail to init range splitter", KR(ret)); } else if (OB_FAIL(range_splitter.split_range(range_array_, max_parallel_degree, allocator_))) { LOG_WARN("fail to split range", KR(ret)); @@ -605,36 +630,38 @@ int ObDirectLoadTabletMergeCtx::build_heap_table_multiple_merge_task( merge_task = nullptr; } } - } - // for imported data, construct task by multiple heap table - for (int64_t i = 0; OB_SUCC(ret) && !param_.is_fast_heap_table_ && i < multiple_heap_table_array_.count(); ++i) { - ObDirectLoadMultipleHeapTable *heap_table = multiple_heap_table_array_.at(i); - ObDirectLoadPartitionHeapTableMultipleMergeTask *merge_task = nullptr; - int64_t row_count = 0; - ObTabletCacheInterval pk_interval; - if (OB_FAIL(heap_table->get_tablet_row_count(tablet_id_, param_.table_data_desc_, row_count))) { - LOG_WARN("fail to get tablet row count", KR(ret), K(tablet_id_)); - } else if (0 == row_count) { - // ignore - } else if (OB_FAIL(get_autoincrement_value(row_count, pk_interval))) { - LOG_WARN("fail to get autoincrement value", KR(ret), K(row_count)); - } else if (OB_ISNULL(merge_task = OB_NEWx(ObDirectLoadPartitionHeapTableMultipleMergeTask, - (&allocator_)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to new ObDirectLoadPartitionHeapTableMultipleMergeTask", KR(ret)); - } else if (OB_FAIL(merge_task->init(ctx_, param_, this, heap_table, pk_interval, parallel_idx++))) { - LOG_WARN("fail to init merge task", KR(ret)); - } else if (OB_FAIL(task_array_.push_back(merge_task))) { - LOG_WARN("fail to push back merge task", KR(ret)); - } - if (OB_FAIL(ret)) { - if (nullptr != merge_task) { - merge_task->~ObDirectLoadPartitionHeapTableMultipleMergeTask(); - allocator_.free(merge_task); - merge_task = nullptr; + if (OB_SUCC(ret)) { + // for imported data, construct task by multiple heap table + for (int64_t i = 0; OB_SUCC(ret) && !param_.is_fast_heap_table_ && i < multiple_heap_table_array_.count(); ++i) { + ObDirectLoadMultipleHeapTable *heap_table = multiple_heap_table_array_.at(i); + ObDirectLoadPartitionHeapTableMultipleMergeTask *merge_task = nullptr; + int64_t row_count = 0; + ObTabletCacheInterval pk_interval; + if (OB_FAIL(heap_table->get_tablet_row_count(tablet_id_, param_.table_data_desc_, row_count))) { + LOG_WARN("fail to get tablet row count", KR(ret), K(tablet_id_)); + } else if (0 == row_count) { + // ignore + } else if (OB_FAIL(get_autoincrement_value(row_count, pk_interval))) { + LOG_WARN("fail to get autoincrement value", KR(ret), K(row_count)); + } else if (OB_ISNULL(merge_task = OB_NEWx(ObDirectLoadPartitionHeapTableMultipleMergeTask, (&allocator_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObDirectLoadPartitionHeapTableMultipleMergeTask", KR(ret)); + } else if (OB_FAIL(merge_task->init(ctx_, param_, this, heap_table, pk_interval, parallel_idx++))) { + LOG_WARN("fail to init merge task", KR(ret)); + } else if (OB_FAIL(task_array_.push_back(merge_task))) { + LOG_WARN("fail to push back merge task", KR(ret)); + } + if (OB_FAIL(ret)) { + if (nullptr != merge_task) { + merge_task->~ObDirectLoadPartitionHeapTableMultipleMergeTask(); + allocator_.free(merge_task); + merge_task = nullptr; + } + } } } } + return ret; } @@ -707,6 +734,61 @@ int ObDirectLoadTabletMergeCtx::build_rescan_task(int64_t thread_count) return ret; } +int ObDirectLoadTabletMergeCtx::build_del_lob_task( + const common::ObIArray &multiple_sstable_array, + ObDirectLoadMultipleMergeRangeSplitter &range_splitter, + const int64_t max_parallel_degree) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadTabletMergeCtx not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(max_parallel_degree <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(max_parallel_degree)); + } else { + if (OB_FAIL(del_lob_multiple_sstable_array_.assign(multiple_sstable_array))) { + LOG_WARN("fail to assign multiple sstable array", KR(ret)); + } else if (OB_FAIL(range_splitter.split_range(tablet_id_, + nullptr /*origin_table*/, + max_parallel_degree, + del_lob_range_array_, + allocator_))) { + LOG_WARN("fail to split range", KR(ret)); + } else if (OB_UNLIKELY(del_lob_range_array_.count() > max_parallel_degree)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected range count", KR(ret), K(max_parallel_degree), K(del_lob_range_array_.count())); + } + // construct task per range + for (int64_t i = 0; OB_SUCC(ret) && i < del_lob_range_array_.count(); ++i) { + const ObDatumRange &range = del_lob_range_array_.at(i); + ObDirectLoadPartitionDelLobTask *del_lob_task = nullptr; + if (OB_ISNULL(del_lob_task = + OB_NEWx(ObDirectLoadPartitionDelLobTask, (&allocator_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObDirectLoadPartitionDelLobTask", KR(ret)); + } else if (OB_FAIL(del_lob_task->init(param_, + this, + &lob_meta_origin_table_, + del_lob_multiple_sstable_array_, + range, + i))) { + LOG_WARN("fail to init del lob task", KR(ret)); + } else if (OB_FAIL(del_lob_task_array_.push_back(del_lob_task))) { + LOG_WARN("fail to push back del lob task", KR(ret)); + } + if (OB_FAIL(ret)) { + if (nullptr != del_lob_task) { + del_lob_task->~ObDirectLoadPartitionDelLobTask(); + allocator_.free(del_lob_task); + del_lob_task = nullptr; + } + } + } + } + return ret; +} + int ObDirectLoadTabletMergeCtx::get_autoincrement_value(uint64_t count, ObTabletCacheInterval &interval) { @@ -755,5 +837,31 @@ int ObDirectLoadTabletMergeCtx::inc_rescan_finish_count(bool &is_ready) return ret; } +int ObDirectLoadTabletMergeCtx::inc_del_lob_finish_count(bool &is_ready) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadTabletMergeCtx not init", KR(ret), KP(this)); + } else { + const int64_t finish_count = ATOMIC_AAF(&del_lob_task_finish_count_, 1); + is_ready = (finish_count >= del_lob_task_array_.count()); + } + return ret; +} + +int ObDirectLoadTabletMergeCtx::get_table_builder(ObIDirectLoadPartitionTableBuilder *&table_builder) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadTabletMergeCtx not init", KR(ret), KP(this)); + } else if (OB_FAIL(merge_ctx_->get_table_builder(table_builder))) { + LOG_WARN("fail to get table builder", KR(ret)); + } + + return ret; +} + } // namespace storage } // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_merge_ctx.h b/src/storage/direct_load/ob_direct_load_merge_ctx.h index 4e9b7e59bb..55ef8b5fde 100644 --- a/src/storage/direct_load/ob_direct_load_merge_ctx.h +++ b/src/storage/direct_load/ob_direct_load_merge_ctx.h @@ -32,6 +32,7 @@ namespace storage class ObDirectLoadInsertTableContext; class ObDirectLoadPartitionMergeTask; class ObDirectLoadPartitionRescanTask; +class ObDirectLoadPartitionDelLobTask; class ObDirectLoadTabletMergeCtx; class ObIDirectLoadPartitionTable; class ObDirectLoadSSTable; @@ -39,6 +40,8 @@ class ObDirectLoadMultipleSSTable; class ObDirectLoadMultipleHeapTable; class ObDirectLoadMultipleMergeRangeSplitter; class ObDirectLoadDMLRowHandler; +class ObIDirectLoadPartitionTableBuilder; +class ObDirectLoadTmpFileManager; struct ObDirectLoadMergeParam { @@ -47,57 +50,75 @@ public: ~ObDirectLoadMergeParam(); bool is_valid() const; TO_STRING_KV(K_(table_id), + K_(lob_meta_table_id), K_(target_table_id), K_(rowkey_column_num), K_(store_column_count), K_(fill_cg_thread_cnt), + KP_(lob_column_idxs), K_(table_data_desc), KP_(datum_utils), + KP_(lob_column_idxs), KP_(col_descs), + K_(lob_id_table_data_desc), + KP_(lob_meta_datum_utils), + KP_(lob_meta_col_descs), K_(is_heap_table), K_(is_fast_heap_table), K_(is_incremental), "insert_mode", ObDirectLoadInsertMode::get_type_string(insert_mode_), KP_(insert_table_ctx), - KP_(dml_row_handler)); + KP_(dml_row_handler), + KP_(file_mgr)); public: uint64_t table_id_; + uint64_t lob_meta_table_id_; uint64_t target_table_id_; int64_t rowkey_column_num_; int64_t store_column_count_; int64_t fill_cg_thread_cnt_; + const common::ObArray *lob_column_idxs_; storage::ObDirectLoadTableDataDesc table_data_desc_; const blocksstable::ObStorageDatumUtils *datum_utils_; const common::ObIArray *col_descs_; + storage::ObDirectLoadTableDataDesc lob_id_table_data_desc_; + const blocksstable::ObStorageDatumUtils *lob_meta_datum_utils_; + const common::ObIArray *lob_meta_col_descs_; bool is_heap_table_; bool is_fast_heap_table_; bool is_incremental_; ObDirectLoadInsertMode::Type insert_mode_; ObDirectLoadInsertTableContext *insert_table_ctx_; ObDirectLoadDMLRowHandler *dml_row_handler_; + ObDirectLoadTmpFileManager *file_mgr_; }; class ObDirectLoadMergeCtx { +private: + typedef common::hash::ObHashMap TABLE_BUILDER_MAP; public: ObDirectLoadMergeCtx(); ~ObDirectLoadMergeCtx(); int init(observer::ObTableLoadTableCtx *ctx, const ObDirectLoadMergeParam ¶m, - const common::ObIArray &ls_partition_ids, - const common::ObIArray &target_ls_partition_ids); + const common::ObIArray &ls_partition_ids); + int get_table_builder(ObIDirectLoadPartitionTableBuilder *&table_builder); + int close_table_builder(); + TABLE_BUILDER_MAP& get_table_builder_map() { return table_builder_map_; } const common::ObIArray &get_tablet_merge_ctxs() const { return tablet_merge_ctx_array_; } private: - int create_all_tablet_ctxs(const common::ObIArray &ls_partition_ids, - const common::ObIArray &target_ls_partition_ids); + int create_all_tablet_ctxs(const common::ObIArray &ls_partition_ids); private: common::ObArenaAllocator allocator_; observer::ObTableLoadTableCtx *ctx_; ObDirectLoadMergeParam param_; common::ObArray tablet_merge_ctx_array_; + TABLE_BUILDER_MAP table_builder_map_; + lib::ObMutex mutex_; bool is_inited_; }; @@ -106,11 +127,10 @@ class ObDirectLoadTabletMergeCtx public: ObDirectLoadTabletMergeCtx(); ~ObDirectLoadTabletMergeCtx(); - int init(observer::ObTableLoadTableCtx *ctx, + int init(ObDirectLoadMergeCtx *merge_ctx, + observer::ObTableLoadTableCtx *ctx, const ObDirectLoadMergeParam ¶m, - const table::ObTableLoadLSIdAndPartitionId &ls_partition_id, - const table::ObTableLoadLSIdAndPartitionId &target_ls_partition_id); - int build_rescan_task(int64_t thread_count); + const table::ObTableLoadLSIdAndPartitionId &ls_partition_id); int build_merge_task(const common::ObIArray &table_array, const common::ObIArray &col_descs, int64_t max_parallel_degree, bool is_multiple_mode); @@ -120,11 +140,27 @@ public: int64_t max_parallel_degree); int build_aggregate_merge_task_for_multiple_heap_table( const common::ObIArray &table_array); + int build_rescan_task(int64_t thread_count); + int build_del_lob_task( + const common::ObIArray &multiple_sstable_array, + ObDirectLoadMultipleMergeRangeSplitter &range_splitter, + const int64_t max_parallel_degree); int inc_finish_count(bool &is_ready); int inc_rescan_finish_count(bool &is_ready); + int inc_del_lob_finish_count(bool &is_ready); + int get_table_builder(ObIDirectLoadPartitionTableBuilder *&table_builder); + bool merge_with_origin_data() + { + return !param_.is_incremental_ && param_.insert_mode_ == ObDirectLoadInsertMode::NORMAL; + } + bool merge_with_conflict_check() + { + return param_.is_incremental_ && + (param_.insert_mode_ == ObDirectLoadInsertMode::NORMAL || + (param_.insert_mode_ == ObDirectLoadInsertMode::INC_REPLACE && param_.lob_column_idxs_->count() > 0)); + } const ObDirectLoadMergeParam &get_param() const { return param_; } const common::ObTabletID &get_tablet_id() const { return tablet_id_; } - const common::ObTabletID &get_target_tablet_id() const { return target_tablet_id_; } const common::ObIArray &get_tasks() const { return task_array_; @@ -133,7 +169,11 @@ public: { return rescan_task_array_; } - TO_STRING_KV(K_(param), K_(target_partition_id), K_(tablet_id), K_(target_tablet_id)); + const common::ObIArray &get_del_lob_tasks() const + { + return del_lob_task_array_; + } + TO_STRING_KV(K_(param), K_(tablet_id)); private: int init_sstable_array(const common::ObIArray &table_array); int init_multiple_sstable_array( @@ -142,38 +182,36 @@ private: const common::ObIArray &table_array); int build_empty_data_merge_task(const common::ObIArray &col_descs, int64_t max_parallel_degree); - int build_pk_table_merge_task(const common::ObIArray &table_array, - const common::ObIArray &col_descs, - int64_t max_parallel_degree); int build_pk_table_multiple_merge_task( const common::ObIArray &table_array, const common::ObIArray &col_descs, int64_t max_parallel_degree); - int build_heap_table_merge_task( - const common::ObIArray &table_array, - const common::ObIArray &col_descs, - int64_t max_parallel_degree); int build_heap_table_multiple_merge_task( const common::ObIArray &table_array, const common::ObIArray &col_descs, int64_t max_parallel_degree); int get_autoincrement_value(uint64_t count, share::ObTabletCacheInterval &interval); + private: common::ObArenaAllocator allocator_; + ObDirectLoadMergeCtx *merge_ctx_; observer::ObTableLoadTableCtx *ctx_; ObDirectLoadMergeParam param_; - uint64_t target_partition_id_; common::ObTabletID tablet_id_; - common::ObTabletID target_tablet_id_; ObDirectLoadOriginTable origin_table_; + ObDirectLoadOriginTable lob_meta_origin_table_; common::ObArray sstable_array_; common::ObArray multiple_sstable_array_; common::ObArray multiple_heap_table_array_; + common::ObArray del_lob_multiple_sstable_array_; common::ObArray range_array_; + common::ObArray del_lob_range_array_; common::ObArray task_array_; common::ObArray rescan_task_array_; + common::ObArray del_lob_task_array_; int64_t task_finish_count_ CACHE_ALIGNED; int64_t rescan_task_finish_count_ CACHE_ALIGNED; + int64_t del_lob_task_finish_count_ CACHE_ALIGNED; bool is_inited_; }; diff --git a/src/storage/direct_load/ob_direct_load_multiple_datum_row.cpp b/src/storage/direct_load/ob_direct_load_multiple_datum_row.cpp index d3a0f8117d..8d69fa5d79 100644 --- a/src/storage/direct_load/ob_direct_load_multiple_datum_row.cpp +++ b/src/storage/direct_load/ob_direct_load_multiple_datum_row.cpp @@ -97,25 +97,35 @@ int ObDirectLoadMultipleDatumRow::from_datums(const ObTabletID &tablet_id, ObSto K(rowkey_column_count)); } else { reuse(); + seq_no_ = seq_no; ObDirectLoadDatumArray serialize_datum_array; if (OB_FAIL(rowkey_.assign(tablet_id, datums, rowkey_column_count))) { LOG_WARN("fail to assign rowkey", KR(ret)); - } else if (OB_FAIL(serialize_datum_array.assign(datums + rowkey_column_count, - column_count - rowkey_column_count))) { - LOG_WARN("fail to assign datum array", KR(ret)); - } else { - const int64_t buf_size = serialize_datum_array.get_serialize_size(); - char *buf = nullptr; - int64_t pos = 0; - if (OB_ISNULL(buf = static_cast(allocator_.alloc(buf_size)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc memory", KR(ret), K(buf_size)); - } else if (OB_FAIL(serialize_datum_array.serialize(buf, buf_size, pos))) { - LOG_WARN("fail to serialize datum array", KR(ret)); + } else if (column_count > rowkey_column_count) { + // to deserialize datum array + if (OB_FAIL(serialize_datum_array.assign(datums + rowkey_column_count, + column_count - rowkey_column_count))) { + LOG_WARN("fail to assign datum array", KR(ret)); } else { - buf_ = buf; - buf_size_ = buf_size; - seq_no_ = seq_no; + const int64_t buf_size = serialize_datum_array.get_serialize_size(); + char *buf = nullptr; + int64_t pos = 0; + if (OB_ISNULL(buf = static_cast(allocator_.alloc(buf_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc memory", KR(ret), K(buf_size)); + } else if (OB_FAIL(serialize_datum_array.serialize(buf, buf_size, pos))) { + LOG_WARN("fail to serialize datum array", KR(ret)); + } else { + if (OB_NOT_NULL(buf_)) { + allocator_.free(buf); + buf_ = nullptr; + } + buf_ = buf; + buf_size_ = buf_size; + } + if (OB_FAIL(ret) && OB_NOT_NULL(buf)) { + allocator_.free(buf); + } } } } @@ -137,19 +147,21 @@ int ObDirectLoadMultipleDatumRow::to_datums(ObStorageDatum *datums, int64_t colu for (int64_t i = 0; i < rowkey_.datum_array_.count_; ++i) { datums[i] = rowkey_.datum_array_.datums_[i]; } - // from deserialize datum array - ObDirectLoadDatumArray deserialize_datum_array; - int64_t pos = 0; - if (OB_FAIL(deserialize_datum_array.assign(datums + rowkey_.datum_array_.count_, - column_count - rowkey_.datum_array_.count_))) { - LOG_WARN("fail to assign datum array", KR(ret)); - } else if (OB_FAIL(deserialize_datum_array.deserialize(buf_, buf_size_, pos))) { - LOG_WARN("fail to deserialize datum array", KR(ret)); - } else if (OB_UNLIKELY(rowkey_.datum_array_.count_ + deserialize_datum_array.count_ != - column_count)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected column count", KR(ret), K(rowkey_.datum_array_.count_), - K(deserialize_datum_array.count_), K(column_count)); + if (column_count > rowkey_.datum_array_.count_) { + // from deserialize datum array + ObDirectLoadDatumArray deserialize_datum_array; + int64_t pos = 0; + if (OB_FAIL(deserialize_datum_array.assign(datums + rowkey_.datum_array_.count_, + column_count - rowkey_.datum_array_.count_))) { + LOG_WARN("fail to assign datum array", KR(ret)); + } else if (OB_FAIL(deserialize_datum_array.deserialize(buf_, buf_size_, pos))) { + LOG_WARN("fail to deserialize datum array", KR(ret)); + } else if (OB_UNLIKELY(rowkey_.datum_array_.count_ + deserialize_datum_array.count_ != + column_count)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected column count", KR(ret), K(rowkey_.datum_array_.count_), + K(deserialize_datum_array.count_), K(column_count)); + } } } return ret; diff --git a/src/storage/direct_load/ob_direct_load_multiple_datum_row.h b/src/storage/direct_load/ob_direct_load_multiple_datum_row.h index ba4f7e8ec6..ada48ac2ea 100644 --- a/src/storage/direct_load/ob_direct_load_multiple_datum_row.h +++ b/src/storage/direct_load/ob_direct_load_multiple_datum_row.h @@ -37,7 +37,7 @@ public: int to_datums(blocksstable::ObStorageDatum *datums, int64_t column_count) const; OB_INLINE bool is_valid() const { - return rowkey_.is_valid() && seq_no_.is_valid() && buf_size_ > 0 && nullptr != buf_; + return rowkey_.is_valid() && seq_no_.is_valid() && (buf_size_ == 0 || nullptr != buf_); } OB_INLINE int64_t get_raw_size() const { return buf_size_; } TO_STRING_KV(K_(rowkey), K_(seq_no), K_(buf_size), KP_(buf)); diff --git a/src/storage/direct_load/ob_direct_load_multiple_heap_table_builder.cpp b/src/storage/direct_load/ob_direct_load_multiple_heap_table_builder.cpp index 0a5202f2fe..ffeadb9645 100644 --- a/src/storage/direct_load/ob_direct_load_multiple_heap_table_builder.cpp +++ b/src/storage/direct_load/ob_direct_load_multiple_heap_table_builder.cpp @@ -203,6 +203,8 @@ int ObDirectLoadMultipleHeapTableBuilder::get_tables( } else if (OB_UNLIKELY(!is_closed_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("multiple heap table builder is not closed", KR(ret)); + } else if (row_count_ == 0) { + // do nothing } else { ObDirectLoadMultipleHeapTableDataFragment data_fragment; ObDirectLoadMultipleHeapTableCreateParam create_param; diff --git a/src/storage/direct_load/ob_direct_load_multiple_sstable_builder.cpp b/src/storage/direct_load/ob_direct_load_multiple_sstable_builder.cpp index a390023a57..33498744c1 100644 --- a/src/storage/direct_load/ob_direct_load_multiple_sstable_builder.cpp +++ b/src/storage/direct_load/ob_direct_load_multiple_sstable_builder.cpp @@ -261,6 +261,8 @@ int ObDirectLoadMultipleSSTableBuilder::get_tables( } else if (OB_UNLIKELY(!is_closed_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("multiple sstable builder is not closed", KR(ret)); + } else if (row_count_ == 0) { + // do nothing } else { ObDirectLoadMultipleSSTableFragment fragment; ObDirectLoadMultipleSSTableCreateParam create_param; diff --git a/src/storage/direct_load/ob_direct_load_origin_table.cpp b/src/storage/direct_load/ob_direct_load_origin_table.cpp index b1a420a438..8bebb01850 100644 --- a/src/storage/direct_load/ob_direct_load_origin_table.cpp +++ b/src/storage/direct_load/ob_direct_load_origin_table.cpp @@ -31,7 +31,7 @@ using namespace share::schema; */ ObDirectLoadOriginTableCreateParam::ObDirectLoadOriginTableCreateParam() - : table_id_(OB_INVALID_ID), insert_mode_(ObDirectLoadInsertMode::INVALID_INSERT_MODE) + : table_id_(OB_INVALID_ID) { } @@ -41,10 +41,7 @@ ObDirectLoadOriginTableCreateParam::~ObDirectLoadOriginTableCreateParam() bool ObDirectLoadOriginTableCreateParam::is_valid() const { - return OB_INVALID_ID != table_id_ && - tablet_id_.is_valid() && - ls_id_.is_valid() && - ObDirectLoadInsertMode::is_type_valid(insert_mode_); + return OB_INVALID_ID != table_id_ && tablet_id_.is_valid() && ls_id_.is_valid(); } /** @@ -52,7 +49,7 @@ bool ObDirectLoadOriginTableCreateParam::is_valid() const */ ObDirectLoadOriginTableMeta::ObDirectLoadOriginTableMeta() - : table_id_(OB_INVALID_ID), insert_mode_(ObDirectLoadInsertMode::INVALID_INSERT_MODE) + : table_id_(OB_INVALID_ID) { } @@ -99,13 +96,12 @@ int ObDirectLoadOriginTable::init(const ObDirectLoadOriginTableCreateParam ¶ LOG_WARN("unexpected ls is nullptr", KR(ret)); } else if (OB_FAIL(ls->get_tablet(tablet_id, tablet_handle_))) { LOG_WARN("fail to get tablet", KR(ret), K(tablet_id)); - } else if (ObDirectLoadInsertMode::need_origin_data(param.insert_mode_) && OB_FAIL(prepare_tables())) { + } else if (OB_FAIL(prepare_tables())) { LOG_WARN("fail to prepare tables", KR(ret)); } else { meta_.ls_id_ = param.ls_id_; meta_.table_id_ = param.table_id_; meta_.tablet_id_ = param.tablet_id_; - meta_.insert_mode_ = param.insert_mode_; is_inited_ = true; } } @@ -162,8 +158,11 @@ int ObDirectLoadOriginTable::prepare_tables() return ret; } -int ObDirectLoadOriginTable::scan(const ObDatumRange &key_range, - ObIAllocator &allocator, ObIStoreRowIterator *&row_iter) +int ObDirectLoadOriginTable::scan( + const ObDatumRange &key_range, + ObIAllocator &allocator, + ObIStoreRowIterator *&row_iter, + bool skip_read_lob) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -177,8 +176,10 @@ int ObDirectLoadOriginTable::scan(const ObDatumRange &key_range, if (OB_ISNULL(row_scanner = OB_NEWx(ObDirectLoadOriginTableScanner, (&allocator)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to new ObDirectLoadOriginTableScanner", KR(ret)); - } else if (OB_FAIL(row_scanner->init(this, key_range))) { - LOG_WARN("Fail to open row scanner", KR(ret), K(key_range), K(*this)); + } else if (OB_FAIL(row_scanner->init(this, skip_read_lob))) { + LOG_WARN("Fail to init row scanner", KR(ret), K(*this)); + } else if (OB_FAIL(row_scanner->open(key_range))) { + LOG_WARN("Fail to open row scanner", KR(ret), K(key_range)); } else { row_iter = row_scanner; } @@ -193,14 +194,41 @@ int ObDirectLoadOriginTable::scan(const ObDatumRange &key_range, return ret; } +int ObDirectLoadOriginTable::rescan( + const ObDatumRange &key_range, + ObIStoreRowIterator *row_iter) +{ + int ret = OB_SUCCESS; + ObDirectLoadOriginTableScanner * row_iter_ptr = nullptr; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadOriginTable not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(!key_range.is_valid() || nullptr == row_iter)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid argument", KR(ret), K(key_range), KP(row_iter)); + } else if (OB_ISNULL(row_iter_ptr = static_cast(row_iter))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", KR(ret)); + } else if (OB_FAIL(row_iter_ptr->open(key_range))) { + LOG_WARN("fail to open ObDirectLoadOriginTableScanner", KR(ret), K(key_range)); + } + + return ret; +} + /** * ObDirectLoadOriginTableScanner */ ObDirectLoadOriginTableScanner::ObDirectLoadOriginTableScanner() - : allocator_("TLD_OriSSTScan"), origin_table_(nullptr), schema_param_(allocator_), is_inited_(false) + : allocator_("TLD_OriSSTScan"), + stmt_allocator_("TLD_SOriSSTScan"), + origin_table_(nullptr), + schema_param_(stmt_allocator_), + is_inited_(false) { allocator_.set_tenant_id(MTL_ID()); + stmt_allocator_.set_tenant_id(MTL_ID()); col_ids_.set_tenant_id(MTL_ID()); } @@ -208,40 +236,52 @@ ObDirectLoadOriginTableScanner::~ObDirectLoadOriginTableScanner() { } -int ObDirectLoadOriginTableScanner::init(ObDirectLoadOriginTable *origin_table, - const ObDatumRange &query_range) +int ObDirectLoadOriginTableScanner::init(ObDirectLoadOriginTable *origin_table, bool skip_read_lob) { int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("ObDirectLoadOriginIterator init twice", KR(ret), KP(this)); - } else if (OB_UNLIKELY(nullptr == origin_table || !origin_table->is_valid() || - !query_range.is_valid() || !query_range.is_memtable_valid())) { + } else if (OB_UNLIKELY(nullptr == origin_table || !origin_table->is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("Invalid argument", KR(ret), KPC(origin_table), K(query_range)); + LOG_WARN("Invalid argument", KR(ret), KPC(origin_table)); } else { origin_table_ = origin_table; - if (!ObDirectLoadInsertMode::need_origin_data(origin_table_->get_meta().insert_mode_)) { - // do nothing - } else if (OB_FAIL((init_table_access_param()))) { + if (OB_FAIL((init_table_access_param()))) { LOG_WARN("fail to init query range", KR(ret)); - } else if (OB_FAIL(init_table_access_ctx())) { + } else if (OB_FAIL(init_table_access_ctx(skip_read_lob))) { LOG_WARN("fail to init table access param", KR(ret)); } else if (OB_FAIL(init_get_table_param())) { LOG_WARN("fail to init get table param", KR(ret)); } else if (OB_FAIL( scan_merge_.init(table_access_param_, table_access_ctx_, get_table_param_))) { LOG_WARN("fail to init multi merge", KR(ret)); - } else if (OB_FAIL(scan_merge_.open(query_range))) { - LOG_WARN("fail to open multi merge", KR(ret), K(query_range)); - } - if (OB_SUCC(ret)) { + } else { is_inited_ = true; } } return ret; } +int ObDirectLoadOriginTableScanner::open(const ObDatumRange &query_range) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(!query_range.is_valid() || !query_range.is_memtable_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid argument", KR(ret), K(query_range)); + } else { + allocator_.reuse(); + scan_merge_.reuse(); + if (OB_FAIL(scan_merge_.open(query_range))) { + LOG_WARN("fail to open multi merge", KR(ret), K(query_range)); + } + } + return ret; +} + int ObDirectLoadOriginTableScanner::init_table_access_param() { int ret = OB_SUCCESS; @@ -284,7 +324,7 @@ int ObDirectLoadOriginTableScanner::init_table_access_param() return ret; } -int ObDirectLoadOriginTableScanner::init_table_access_ctx() +int ObDirectLoadOriginTableScanner::init_table_access_ctx(bool skip_read_lob) { int ret = OB_SUCCESS; const uint64_t table_id = origin_table_->get_meta().table_id_; @@ -299,6 +339,9 @@ int ObDirectLoadOriginTableScanner::init_table_access_ctx() false /*query_stat*/); //whole_macro_scan use false，otherwise query range is not overlap with sstable range will report error ObVersionRange trans_version_range; query_flag.multi_version_minor_merge_ = false; + if (skip_read_lob) { + query_flag.skip_read_lob_ = ObQueryFlag::OBSF_MASK_SKIP_READ_LOB; + } trans_version_range.multi_version_start_ = 0; trans_version_range.base_version_ = 0; trans_version_range.snapshot_version_ = snapshot_version; @@ -308,7 +351,7 @@ int ObDirectLoadOriginTableScanner::init_table_access_ctx() } else if (OB_FAIL(store_ctx_.init_for_read(origin_table_->get_meta().ls_id_, tablet_id, INT64_MAX, -1, snapshot_scn))) { LOG_WARN("fail to init for read", KR(ret)); - } else if (OB_FAIL(table_access_ctx_.init(query_flag, store_ctx_, allocator_, allocator_, + } else if (OB_FAIL(table_access_ctx_.init(query_flag, store_ctx_, allocator_, stmt_allocator_, trans_version_range))) { LOG_WARN("fail to init table access context", KR(ret)); } else { @@ -334,8 +377,6 @@ int ObDirectLoadOriginTableScanner::get_next_row(const ObDatumRow *&datum_row) if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObDirectLoadOriginTableScanner not init", KR(ret), KP(this)); - } else if (!ObDirectLoadInsertMode::need_origin_data(origin_table_->get_meta().insert_mode_)) { - ret = OB_ITER_END; } else { ObDatumRow *result_row = nullptr; if (OB_FAIL(scan_merge_.get_next_row(result_row))) { diff --git a/src/storage/direct_load/ob_direct_load_origin_table.h b/src/storage/direct_load/ob_direct_load_origin_table.h index a7a057a733..51d4a083aa 100644 --- a/src/storage/direct_load/ob_direct_load_origin_table.h +++ b/src/storage/direct_load/ob_direct_load_origin_table.h @@ -27,15 +27,11 @@ public: ObDirectLoadOriginTableCreateParam(); ~ObDirectLoadOriginTableCreateParam(); bool is_valid() const; - TO_STRING_KV(K_(table_id), - K_(tablet_id), - K_(ls_id), - "insert_mode", ObDirectLoadInsertMode::get_type_string(insert_mode_)); + TO_STRING_KV(K_(table_id), K_(tablet_id), K_(ls_id)); public: uint64_t table_id_; common::ObTabletID tablet_id_; share::ObLSID ls_id_; - ObDirectLoadInsertMode::Type insert_mode_; }; struct ObDirectLoadOriginTableMeta @@ -43,15 +39,11 @@ struct ObDirectLoadOriginTableMeta public: ObDirectLoadOriginTableMeta(); ~ObDirectLoadOriginTableMeta(); - TO_STRING_KV(K_(table_id), - K_(tablet_id), - K_(ls_id), - "insert_mode", ObDirectLoadInsertMode::get_type_string(insert_mode_)); + TO_STRING_KV(K_(table_id), K_(tablet_id), K_(ls_id)); public: uint64_t table_id_; common::ObTabletID tablet_id_; share::ObLSID ls_id_; - ObDirectLoadInsertMode::Type insert_mode_; }; class ObDirectLoadOriginTable @@ -60,7 +52,12 @@ public: ObDirectLoadOriginTable(); virtual ~ObDirectLoadOriginTable(); int init(const ObDirectLoadOriginTableCreateParam ¶m); - int scan(const blocksstable::ObDatumRange &key_range, common::ObIAllocator &allocator, ObIStoreRowIterator *&row_iter); + int scan( + const blocksstable::ObDatumRange &key_range, + common::ObIAllocator &allocator, + ObIStoreRowIterator *&row_iter, + bool skip_read_lob); + int rescan(const blocksstable::ObDatumRange &key_range, ObIStoreRowIterator *row_iter); bool is_valid() const { return is_inited_; } const ObDirectLoadOriginTableMeta &get_meta() const {return meta_; } const ObTabletHandle &get_tablet_handle() const { return tablet_handle_; } @@ -85,15 +82,16 @@ class ObDirectLoadOriginTableScanner : public ObIStoreRowIterator public: ObDirectLoadOriginTableScanner(); virtual ~ObDirectLoadOriginTableScanner(); - int init(ObDirectLoadOriginTable *table, - const blocksstable::ObDatumRange &query_range); + int init(ObDirectLoadOriginTable *table, bool skip_read_lob); + int open(const blocksstable::ObDatumRange &query_range); int get_next_row(const blocksstable::ObDatumRow *&datum_row) override; private: int init_table_access_param(); - int init_table_access_ctx(); + int init_table_access_ctx(bool skip_read_lob); int init_get_table_param(); private: common::ObArenaAllocator allocator_; + common::ObArenaAllocator stmt_allocator_; ObDirectLoadOriginTable *origin_table_; ObArray col_ids_; share::schema::ObTableSchemaParam schema_param_; diff --git a/src/storage/direct_load/ob_direct_load_partition_del_lob_task.cpp b/src/storage/direct_load/ob_direct_load_partition_del_lob_task.cpp new file mode 100644 index 0000000000..7bf612aefe --- /dev/null +++ b/src/storage/direct_load/ob_direct_load_partition_del_lob_task.cpp @@ -0,0 +1,245 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "storage/direct_load/ob_direct_load_partition_del_lob_task.h" +#include "storage/direct_load/ob_direct_load_insert_table_ctx.h" +#include "storage/direct_load/ob_direct_load_lob_meta_row_iter.h" +#include "storage/lob/ob_lob_meta.h" + +namespace oceanbase +{ +namespace storage +{ +using namespace common; +using namespace blocksstable; +using namespace share; + +ObDirectLoadPartitionDelLobTask::RowIterator::RowIterator() : is_inited_(false) {} + +ObDirectLoadPartitionDelLobTask::RowIterator::~RowIterator() {} + +int ObDirectLoadPartitionDelLobTask::RowIterator::init( + const ObDirectLoadMergeParam &merge_param, const ObDirectLoadTabletMergeCtx *merge_ctx, + ObDirectLoadOriginTable *origin_table, + const ObIArray &sstable_array, const ObDatumRange &range, + int64_t parallel_idx, ObDirectLoadInsertTabletContext *insert_tablet_ctx) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObDirectLoadPartitionRangeMultipleMergeTask::RowIterator init twice", KR(ret), + KP(this)); + } else if (OB_UNLIKELY(!merge_param.is_valid() || nullptr == merge_ctx || + nullptr == origin_table || !range.is_valid() || + nullptr == insert_tablet_ctx)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(merge_param), KP(merge_ctx), K(sstable_array), K(range), + KP(insert_tablet_ctx)); + } else { + ObDirectLoadLobMetaIterParam iter_param; + iter_param.tablet_id_ = merge_ctx->get_tablet_id(); + iter_param.table_data_desc_ = merge_param.lob_id_table_data_desc_; + iter_param.datum_utils_ = merge_param.lob_meta_datum_utils_; + iter_param.col_descs_ = merge_param.lob_meta_col_descs_; + if (OB_FAIL(lob_iter_.init(iter_param, origin_table, sstable_array, range))) { + LOG_WARN("fail to init lob meta row iter", KR(ret)); + } else if (OB_FAIL(insert_tablet_ctx->init_lob_datum_row(datum_row_))) { + LOG_WARN("fail to init lob datum row", KR(ret)); + } else { + is_inited_ = true; + } + } + return ret; +} + +int ObDirectLoadPartitionDelLobTask::RowIterator::get_next_row(const ObDatumRow *&result_row) +{ + int ret = OB_SUCCESS; + result_row = nullptr; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadPartitionRangeMultipleMergeTask::RowIterator not init", KR(ret), + KP(this)); + } else { + const ObDatumRow *datum_row = nullptr; + if (OB_FAIL(lob_iter_.get_next_row(datum_row))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("fail to get next row", KR(ret)); + } + } else if (OB_UNLIKELY(datum_row->count_ < ObLobMetaUtil::LOB_META_SCHEMA_ROWKEY_COL_CNT)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected datum row count", KR(ret), KPC(datum_row)); + } else { + for (int64_t i = 0; i < ObLobMetaUtil::LOB_META_SCHEMA_ROWKEY_COL_CNT; ++i) { + datum_row_.storage_datums_[i] = datum_row->storage_datums_[i]; + } + result_row = &datum_row_; + } + } + return ret; +} + +ObDirectLoadPartitionDelLobTask::ObDirectLoadPartitionDelLobTask() + : merge_param_(nullptr), + merge_ctx_(nullptr), + origin_table_(nullptr), + sstable_array_(nullptr), + range_(nullptr), + parallel_idx_(-1), + is_stop_(false), + is_inited_(false) +{ +} + +ObDirectLoadPartitionDelLobTask::~ObDirectLoadPartitionDelLobTask() {} + +int ObDirectLoadPartitionDelLobTask::init( + const ObDirectLoadMergeParam &merge_param, ObDirectLoadTabletMergeCtx *merge_ctx, + ObDirectLoadOriginTable *origin_table, + const ObIArray &sstable_array, const ObDatumRange &range, + int64_t parallel_idx) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObDirectLoadPartitionDelLobTask init twice", KR(ret), KP(this)); + } else if (OB_UNLIKELY(!merge_param.is_valid() || nullptr == merge_ctx || + nullptr == origin_table || !range.is_valid() || parallel_idx < 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(merge_param), KP(merge_ctx), K(sstable_array), K(range), + K(parallel_idx)); + } else { + merge_param_ = &merge_param; + merge_ctx_ = merge_ctx; + origin_table_ = origin_table; + sstable_array_ = &sstable_array; + range_ = ⦥ + parallel_idx_ = parallel_idx; + is_inited_ = true; + } + return ret; +} + +int ObDirectLoadPartitionDelLobTask::process() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadPartitionDelLobTask not init", KR(ret), KP(this)); + } else { + int64_t slice_id = 0; + const ObTabletID &tablet_id = merge_ctx_->get_tablet_id(); + ObDirectLoadInsertTabletContext *insert_tablet_ctx = nullptr; + RowIterator row_iter; + ObMacroDataSeq block_start_seq; + int64_t affected_rows = 0; + block_start_seq.set_parallel_degree(parallel_idx_); + if (OB_UNLIKELY(is_stop_)) { + ret = OB_CANCELED; + LOG_WARN("merge task canceled", KR(ret)); + } else if (OB_FAIL(merge_param_->insert_table_ctx_->get_tablet_context(tablet_id, + insert_tablet_ctx))) { + LOG_WARN("fail to get tablet context ", KR(ret), K(tablet_id)); + } else if (OB_FAIL(row_iter.init(*merge_param_, merge_ctx_, origin_table_, *sstable_array_, + *range_, parallel_idx_, insert_tablet_ctx))) { + LOG_WARN("fail to init row iter", KR(ret)); + } else if (OB_FAIL(insert_tablet_ctx->open_lob_sstable_slice(block_start_seq, slice_id))) { + LOG_WARN("fail to construct lob sstable slice ", KR(ret), K(slice_id), K(block_start_seq)); + } else { + LOG_INFO("add lob meta sstable slice begin", K(tablet_id), K(parallel_idx_), K(slice_id)); + if (OB_FAIL( + insert_tablet_ctx->fill_lob_meta_sstable_slice(slice_id, row_iter, affected_rows))) { + LOG_WARN("fail to fill lob meta sstable slice", KR(ret)); + } else if (OB_FAIL(insert_tablet_ctx->close_lob_sstable_slice(slice_id))) { + LOG_WARN("fail to close writer", KR(ret)); + } + LOG_INFO("add lob meta sstable slice end", KR(ret), K(tablet_id), K(parallel_idx_), + K(affected_rows)); + } + if (OB_SUCC(ret)) { + bool is_ready = false; + if (OB_FAIL(merge_ctx_->inc_del_lob_finish_count(is_ready))) { + LOG_WARN("fail to inc del lob finish count", KR(ret)); + } else if (is_ready) { + if (OB_FAIL(insert_tablet_ctx->close())) { + LOG_WARN("fail to close", KR(ret)); + } + } + } + } + return ret; +} + +void ObDirectLoadPartitionDelLobTask::stop() { is_stop_ = true; } + +ObDirectLoadDelLobTaskIterator::ObDirectLoadDelLobTaskIterator() + : merge_ctx_(nullptr), tablet_merge_ctx_(nullptr), tablet_pos_(0), task_pos_(0), is_inited_(false) +{ +} + +ObDirectLoadDelLobTaskIterator::~ObDirectLoadDelLobTaskIterator() {} + +int ObDirectLoadDelLobTaskIterator::init(ObDirectLoadMergeCtx *merge_ctx) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObDirectLoadDelLobTaskIterator init twice", KR(ret), KP(this)); + } else if (OB_UNLIKELY(nullptr == merge_ctx)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(merge_ctx)); + } else { + merge_ctx_ = merge_ctx; + is_inited_ = true; + } + return ret; +} + +int ObDirectLoadDelLobTaskIterator::get_next_task(ObDirectLoadPartitionDelLobTask *&task) +{ + int ret = OB_SUCCESS; + task = nullptr; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadDelLobTaskIterator not init", KR(ret), KP(this)); + } else { + while (OB_SUCC(ret) && nullptr == task) { + if (nullptr == tablet_merge_ctx_) { + // get next partition merge ctx + const ObIArray &tablet_merge_ctxs = + merge_ctx_->get_tablet_merge_ctxs(); + if (tablet_pos_ >= tablet_merge_ctxs.count()) { + ret = OB_ITER_END; + } else { + tablet_merge_ctx_ = tablet_merge_ctxs.at(tablet_pos_++); + task_pos_ = 0; + } + } + if (OB_SUCC(ret)) { + const ObIArray &tasks = + tablet_merge_ctx_->get_del_lob_tasks(); + if (task_pos_ >= tasks.count()) { + // try next partition + tablet_merge_ctx_ = nullptr; + } else { + task = tasks.at(task_pos_++); + } + } + } + } + return ret; +} + +} // namespace storage +} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_partition_del_lob_task.h b/src/storage/direct_load/ob_direct_load_partition_del_lob_task.h new file mode 100644 index 0000000000..7b1a7bc5a6 --- /dev/null +++ b/src/storage/direct_load/ob_direct_load_partition_del_lob_task.h @@ -0,0 +1,85 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#pragma once + +#include "lib/list/ob_dlist.h" +#include "storage/direct_load/ob_direct_load_lob_meta_row_iter.h" +#include "storage/direct_load/ob_direct_load_merge_ctx.h" + +namespace oceanbase +{ +namespace storage +{ +class ObDirectLoadInsertTabletContext; + +class ObDirectLoadPartitionDelLobTask : public common::ObDLinkBase +{ +public: + ObDirectLoadPartitionDelLobTask(); + ~ObDirectLoadPartitionDelLobTask(); + int init(const ObDirectLoadMergeParam &merge_param, ObDirectLoadTabletMergeCtx *merge_ctx, + ObDirectLoadOriginTable *origin_table, + const common::ObIArray &sstable_array, + const blocksstable::ObDatumRange &range, int64_t parallel_idx); + int process(); + void stop(); + TO_STRING_KV(K_(parallel_idx), K_(is_stop)); + +private: + class RowIterator : public ObIStoreRowIterator + { + public: + RowIterator(); + virtual ~RowIterator(); + int init(const ObDirectLoadMergeParam &merge_param, const ObDirectLoadTabletMergeCtx *merge_ctx, + ObDirectLoadOriginTable *origin_table, + const common::ObIArray &sstable_array, + const blocksstable::ObDatumRange &range, int64_t parallel_idx, + ObDirectLoadInsertTabletContext *insert_tablet_ctx); + int get_next_row(const blocksstable::ObDatumRow *&result_row) override; + + private: + ObDirectLoadLobMetaRowIter lob_iter_; + blocksstable::ObDatumRow datum_row_; + bool is_inited_; + }; + +private: + const ObDirectLoadMergeParam *merge_param_; + ObDirectLoadTabletMergeCtx *merge_ctx_; + ObDirectLoadOriginTable *origin_table_; + const ObIArray *sstable_array_; + const blocksstable::ObDatumRange *range_; + int64_t parallel_idx_; + bool is_stop_; + bool is_inited_; +}; + +class ObDirectLoadDelLobTaskIterator +{ +public: + ObDirectLoadDelLobTaskIterator(); + ~ObDirectLoadDelLobTaskIterator(); + int init(storage::ObDirectLoadMergeCtx *merge_ctx); + int get_next_task(ObDirectLoadPartitionDelLobTask *&task); + +private: + storage::ObDirectLoadMergeCtx *merge_ctx_; + storage::ObDirectLoadTabletMergeCtx *tablet_merge_ctx_; + int64_t tablet_pos_; + int64_t task_pos_; + bool is_inited_; +}; + +} // namespace storage +} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_partition_merge_task.cpp b/src/storage/direct_load/ob_direct_load_partition_merge_task.cpp index 606fb33326..ff79928807 100644 --- a/src/storage/direct_load/ob_direct_load_partition_merge_task.cpp +++ b/src/storage/direct_load/ob_direct_load_partition_merge_task.cpp @@ -17,6 +17,8 @@ #include "storage/direct_load/ob_direct_load_merge_ctx.h" #include "storage/direct_load/ob_direct_load_multiple_heap_table.h" #include "storage/direct_load/ob_direct_load_origin_table.h" +#include "storage/direct_load/ob_direct_load_conflict_check.h" +#include "storage/direct_load/ob_direct_load_data_insert.h" namespace oceanbase { @@ -127,6 +129,8 @@ int ObDirectLoadPartitionMergeTask::process() if (OB_FAIL(insert_tablet_ctx_->calc_range(merge_param_->fill_cg_thread_cnt_))) { LOG_WARN("fail to calc range", KR(ret)); } + } else if (insert_tablet_ctx_->need_del_lob()) { + // do nothing } else if (OB_FAIL(insert_tablet_ctx_->close())) { LOG_WARN("fail to close", KR(ret)); } @@ -161,15 +165,20 @@ void ObDirectLoadPartitionMergeTask::stop() */ ObDirectLoadPartitionRangeMergeTask::RowIterator::RowIterator() - : rowkey_column_num_(0) + : data_iter_(nullptr), rowkey_column_num_(0) { } -ObDirectLoadPartitionRangeMergeTask::RowIterator::~RowIterator() {} +ObDirectLoadPartitionRangeMergeTask::RowIterator::~RowIterator() +{ + if (data_iter_ != nullptr) { + ob_delete(data_iter_); + } +} int ObDirectLoadPartitionRangeMergeTask::RowIterator::init( const ObDirectLoadMergeParam &merge_param, - const ObTabletID &tablet_id, + ObDirectLoadTabletMergeCtx *merge_ctx, ObDirectLoadOriginTable *origin_table, ObTableLoadSqlStatistics *sql_statistics, ObDirectLoadLobBuilder &lob_builder, @@ -181,33 +190,78 @@ int ObDirectLoadPartitionRangeMergeTask::RowIterator::init( if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("ObDirectLoadPartitionRangeMergeTask::RowIterator init twice", KR(ret), KP(this)); - } else if (OB_UNLIKELY(!merge_param.is_valid() || !tablet_id.is_valid() || + } else if (OB_UNLIKELY(!merge_param.is_valid() || nullptr == merge_ctx || nullptr == origin_table || !range.is_valid() || nullptr == insert_tablet_ctx)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(merge_param), K(tablet_id), K(sstable_array), K(range), + LOG_WARN("invalid args", KR(ret), K(merge_param), KP(merge_ctx), K(sstable_array), K(range), KP(insert_tablet_ctx)); } else { - // init data_fuse_ - ObDirectLoadDataFuseParam data_fuse_param; - data_fuse_param.tablet_id_ = tablet_id; - data_fuse_param.store_column_count_ = merge_param.store_column_count_; - data_fuse_param.table_data_desc_ = merge_param.table_data_desc_; - data_fuse_param.datum_utils_ = merge_param.datum_utils_; - data_fuse_param.dml_row_handler_ = merge_param.dml_row_handler_; if (OB_FAIL(inner_init(insert_tablet_ctx, sql_statistics, lob_builder))) { LOG_WARN("fail to inner init", KR(ret)); - } else if (OB_FAIL(data_fuse_.init(data_fuse_param, origin_table, sstable_array, range))) { - LOG_WARN("fail to init data fuse", KR(ret)); - } - // init datum_row_ - else if (OB_FAIL(insert_tablet_ctx->init_datum_row(datum_row_))) { + } else if (OB_FAIL(insert_tablet_ctx->init_datum_row(datum_row_))) { LOG_WARN("fail to init datum row", KR(ret)); } else { + if (merge_ctx->merge_with_origin_data()) { + // init data_fuse_ + ObDirectLoadDataFuseParam data_fuse_param; + data_fuse_param.tablet_id_ = merge_ctx->get_tablet_id(); + data_fuse_param.store_column_count_ = merge_param.store_column_count_; + data_fuse_param.table_data_desc_ = merge_param.table_data_desc_; + data_fuse_param.datum_utils_ = merge_param.datum_utils_; + data_fuse_param.dml_row_handler_ = merge_param.dml_row_handler_; + ObDirectLoadSSTableDataFuse *data_fuse = nullptr; + if (OB_ISNULL(data_fuse = OB_NEW(ObDirectLoadSSTableDataFuse, ObMemAttr(MTL_ID(), "TLD_PRMT")))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory", K(ret)); + } else if (FALSE_IT(data_iter_ = data_fuse)) { + } else if (OB_FAIL(data_fuse->init(data_fuse_param, origin_table, sstable_array, range))) { + LOG_WARN("fail to init ObDirectLoadSSTableDataFuse", K(ret)); + } + } else if (merge_ctx->merge_with_conflict_check()) { + ObDirectLoadConflictCheckParam conflict_check_param; + conflict_check_param.tablet_id_ = merge_ctx->get_tablet_id(); + conflict_check_param.table_data_desc_ = merge_param.table_data_desc_; + conflict_check_param.store_column_count_ = merge_param.store_column_count_; + conflict_check_param.origin_table_ = origin_table; + conflict_check_param.range_ = ⦥ + conflict_check_param.col_descs_ = merge_param.col_descs_; + conflict_check_param.lob_column_idxs_ = merge_param.lob_column_idxs_; + conflict_check_param.datum_utils_ = merge_param.datum_utils_; + conflict_check_param.dml_row_handler_ = merge_param.dml_row_handler_; + ObDirectLoadSSTableConflictCheck *conflict_check = nullptr; + if (OB_FAIL(merge_ctx->get_table_builder(conflict_check_param.builder_))) { + LOG_WARN("fail to get table builder", K(ret)); + } else if (OB_ISNULL(conflict_check = OB_NEW(ObDirectLoadSSTableConflictCheck, ObMemAttr(MTL_ID(), "TLD_PRMT")))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc memory", K(ret)); + } else if (FALSE_IT(data_iter_ = conflict_check)) { + } else if (OB_FAIL(conflict_check->init(conflict_check_param, sstable_array))) { + LOG_WARN("fail to init ObDirectLoadSSTableConflictCheck", K(ret)); + } + } else { + ObDirectLoadDataInsertParam data_insert_param; + data_insert_param.tablet_id_ = merge_ctx->get_tablet_id(); + data_insert_param.store_column_count_ = merge_param.store_column_count_; + data_insert_param.table_data_desc_ = merge_param.table_data_desc_; + data_insert_param.datum_utils_ = merge_param.datum_utils_; + data_insert_param.dml_row_handler_ = merge_param.dml_row_handler_; + ObDirectLoadSSTableDataInsert *data_insert = nullptr; + if (OB_ISNULL(data_insert = OB_NEW(ObDirectLoadSSTableDataInsert, ObMemAttr(MTL_ID(), "TLD_PRMT")))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory", K(ret)); + } else if (FALSE_IT(data_iter_ = data_insert)) { + } else if (OB_FAIL(data_insert->init(data_insert_param, sstable_array, range))) { + LOG_WARN("fail to init ObDirectLoadSSTableDataInsert", K(ret)); + } + } + } + if (OB_SUCC(ret)) { rowkey_column_num_ = merge_param.rowkey_column_num_; is_inited_ = true; } } + return ret; } @@ -220,7 +274,7 @@ int ObDirectLoadPartitionRangeMergeTask::RowIterator::inner_get_next_row(ObDatum LOG_WARN("ObDirectLoadPartitionRangeMergeTask::RowIterator not init", KR(ret), KP(this)); } else { const ObDatumRow *datum_row = nullptr; - if (OB_FAIL(data_fuse_.get_next_row(datum_row))) { + if (OB_FAIL(data_iter_->get_next_row(datum_row))) { if (OB_UNLIKELY(OB_ITER_END != ret)) { LOG_WARN("fail to get next row", KR(ret)); } @@ -291,7 +345,7 @@ int ObDirectLoadPartitionRangeMergeTask::construct_row_iter(ObIAllocator &alloca if (OB_ISNULL(row_iter = OB_NEWx(RowIterator, (&allocator)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to new RowIterator", KR(ret)); - } else if (OB_FAIL(row_iter->init(*merge_param_, merge_ctx_->get_tablet_id(), origin_table_, + } else if (OB_FAIL(row_iter->init(*merge_param_, merge_ctx_, origin_table_, sql_statistics_, lob_builder_, *sstable_array_, *range_, insert_tablet_ctx_))) { LOG_WARN("fail to init row iter", KR(ret)); @@ -314,15 +368,20 @@ int ObDirectLoadPartitionRangeMergeTask::construct_row_iter(ObIAllocator &alloca */ ObDirectLoadPartitionRangeMultipleMergeTask::RowIterator::RowIterator() - : rowkey_column_num_(0) + : data_iter_(nullptr), rowkey_column_num_(0) { } -ObDirectLoadPartitionRangeMultipleMergeTask::RowIterator::~RowIterator() {} +ObDirectLoadPartitionRangeMultipleMergeTask::RowIterator::~RowIterator() +{ + if (data_iter_ != nullptr) { + ob_delete(data_iter_); + } +} int ObDirectLoadPartitionRangeMultipleMergeTask::RowIterator::init( const ObDirectLoadMergeParam &merge_param, - const ObTabletID &tablet_id, + ObDirectLoadTabletMergeCtx *merge_ctx, ObDirectLoadOriginTable *origin_table, ObTableLoadSqlStatistics *sql_statistics, ObDirectLoadLobBuilder &lob_builder, @@ -335,33 +394,78 @@ int ObDirectLoadPartitionRangeMultipleMergeTask::RowIterator::init( ret = OB_INIT_TWICE; LOG_WARN("ObDirectLoadPartitionRangeMultipleMergeTask::RowIterator init twice", KR(ret), KP(this)); - } else if (OB_UNLIKELY(!merge_param.is_valid() || !tablet_id.is_valid() || + } else if (OB_UNLIKELY(!merge_param.is_valid() || nullptr == merge_ctx || nullptr == origin_table || !range.is_valid() || nullptr == insert_tablet_ctx)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(merge_param), K(tablet_id), K(sstable_array), K(range), + LOG_WARN("invalid args", KR(ret), K(merge_param), KP(merge_ctx), K(sstable_array), K(range), KP(insert_tablet_ctx)); } else { - // init data_fuse_ - ObDirectLoadDataFuseParam data_fuse_param; - data_fuse_param.tablet_id_ = tablet_id; - data_fuse_param.store_column_count_ = merge_param.store_column_count_; - data_fuse_param.table_data_desc_ = merge_param.table_data_desc_; - data_fuse_param.datum_utils_ = merge_param.datum_utils_; - data_fuse_param.dml_row_handler_ = merge_param.dml_row_handler_; if (OB_FAIL(inner_init(insert_tablet_ctx, sql_statistics, lob_builder))) { LOG_WARN("fail to inner init", KR(ret)); - } else if (OB_FAIL(data_fuse_.init(data_fuse_param, origin_table, sstable_array, range))) { - LOG_WARN("fail to init data fuse", KR(ret)); - } - // init datum_row_ - else if (OB_FAIL(insert_tablet_ctx->init_datum_row(datum_row_))) { + } else if (OB_FAIL(insert_tablet_ctx->init_datum_row(datum_row_))) { LOG_WARN("fail to init datum row", KR(ret)); } else { + if (merge_ctx->merge_with_origin_data()) { + // init data_fuse_ + ObDirectLoadDataFuseParam data_fuse_param; + data_fuse_param.tablet_id_ = merge_ctx->get_tablet_id(); + data_fuse_param.store_column_count_ = merge_param.store_column_count_; + data_fuse_param.table_data_desc_ = merge_param.table_data_desc_; + data_fuse_param.datum_utils_ = merge_param.datum_utils_; + data_fuse_param.dml_row_handler_ = merge_param.dml_row_handler_; + ObDirectLoadMultipleSSTableDataFuse *data_fuse = nullptr; + if (OB_ISNULL(data_fuse = OB_NEW(ObDirectLoadMultipleSSTableDataFuse, ObMemAttr(MTL_ID(), "TLD_PRMMT")))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory", K(ret)); + } else if (FALSE_IT(data_iter_ = data_fuse)) { + } else if (OB_FAIL(data_fuse->init(data_fuse_param, origin_table, sstable_array, range))) { + LOG_WARN("fail to init ObDirectLoadMultipleSSTableDataFuse", K(ret)); + } + } else if (merge_ctx->merge_with_conflict_check()) { + ObDirectLoadConflictCheckParam conflict_check_param; + conflict_check_param.tablet_id_ = merge_ctx->get_tablet_id(); + conflict_check_param.table_data_desc_ = merge_param.table_data_desc_; + conflict_check_param.store_column_count_ = merge_param.store_column_count_; + conflict_check_param.origin_table_ = origin_table; + conflict_check_param.range_ = ⦥ + conflict_check_param.col_descs_ = merge_param.col_descs_; + conflict_check_param.lob_column_idxs_ = merge_param.lob_column_idxs_; + conflict_check_param.datum_utils_ = merge_param.datum_utils_; + conflict_check_param.dml_row_handler_ = merge_param.dml_row_handler_; + ObDirectLoadMultipleSSTableConflictCheck *conflict_check = nullptr; + if (OB_FAIL(merge_ctx->get_table_builder(conflict_check_param.builder_))) { + LOG_WARN("fail to get table builder", K(ret)); + } else if (OB_ISNULL(conflict_check = OB_NEW(ObDirectLoadMultipleSSTableConflictCheck, ObMemAttr(MTL_ID(), "TLD_PRMMT")))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc memory", K(ret)); + } else if (FALSE_IT(data_iter_ = conflict_check)) { + } else if (OB_FAIL(conflict_check->init(conflict_check_param, sstable_array))) { + LOG_WARN("fail to init ObDirectLoadMultipleSSTableConflictCheck", K(ret)); + } + } else { + ObDirectLoadDataInsertParam data_insert_param; + data_insert_param.tablet_id_ = merge_ctx->get_tablet_id(); + data_insert_param.store_column_count_ = merge_param.store_column_count_; + data_insert_param.table_data_desc_ = merge_param.table_data_desc_; + data_insert_param.datum_utils_ = merge_param.datum_utils_; + data_insert_param.dml_row_handler_ = merge_param.dml_row_handler_; + ObDirectLoadMultipleSSTableDataInsert *data_insert = nullptr; + if (OB_ISNULL(data_insert = OB_NEW(ObDirectLoadMultipleSSTableDataInsert, ObMemAttr(MTL_ID(), "TLD_PRMT")))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory", K(ret)); + } else if (FALSE_IT(data_iter_ = data_insert)) { + } else if (OB_FAIL(data_insert->init(data_insert_param, sstable_array, range))) { + LOG_WARN("fail to init ObDirectLoadMultipleSSTableDataInsert", K(ret)); + } + } + } + if (OB_SUCC(ret)) { rowkey_column_num_ = merge_param.rowkey_column_num_; is_inited_ = true; } } + return ret; } @@ -376,7 +480,7 @@ int ObDirectLoadPartitionRangeMultipleMergeTask::RowIterator::inner_get_next_row KP(this)); } else { const ObDatumRow *datum_row = nullptr; - if (OB_FAIL(data_fuse_.get_next_row(datum_row))) { + if (OB_FAIL(data_iter_->get_next_row(datum_row))) { if (OB_UNLIKELY(OB_ITER_END != ret)) { LOG_WARN("fail to get next row", KR(ret)); } @@ -448,7 +552,7 @@ int ObDirectLoadPartitionRangeMultipleMergeTask::construct_row_iter( if (OB_ISNULL(row_iter = OB_NEWx(RowIterator, (&allocator)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to new RowIterator", KR(ret)); - } else if (OB_FAIL(row_iter->init(*merge_param_, merge_ctx_->get_tablet_id(), origin_table_, + } else if (OB_FAIL(row_iter->init(*merge_param_, merge_ctx_, origin_table_, sql_statistics_, lob_builder_, *sstable_array_, *range_, insert_tablet_ctx_))) { LOG_WARN("fail to init row iter", KR(ret)); @@ -812,7 +916,7 @@ ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::RowIterator::~RowItera int ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::RowIterator::init( const ObDirectLoadMergeParam &merge_param, - const ObTabletID &tablet_id, + ObDirectLoadTabletMergeCtx *merge_ctx, ObDirectLoadOriginTable *origin_table, ObTableLoadSqlStatistics *sql_statistics, ObDirectLoadLobBuilder &lob_builder, @@ -825,30 +929,33 @@ int ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::RowIterator::init( ret = OB_INIT_TWICE; LOG_WARN("ObDirectLoadPartitionHeapTableMultipleMergeTask::RowIterator init twice", KR(ret), KP(this)); - } else if (OB_UNLIKELY(!merge_param.is_valid() || !tablet_id.is_valid() || + } else if (OB_UNLIKELY(!merge_param.is_valid() || nullptr == merge_ctx || nullptr == origin_table || nullptr == heap_table_array || nullptr == insert_tablet_ctx)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(merge_param), K(tablet_id), KP(origin_table), + LOG_WARN("invalid args", KR(ret), K(merge_param), KP(merge_ctx), KP(origin_table), KP(heap_table_array), KP(insert_tablet_ctx)); } else { range_.set_whole_range(); // init row iterator if (OB_FAIL(inner_init(insert_tablet_ctx, sql_statistics, lob_builder))) { LOG_WARN("fail to inner init", KR(ret)); - } else if (OB_FAIL(origin_table->scan(range_, allocator_, origin_iter_))) { - LOG_WARN("fail to scan origin table", KR(ret)); } // init datum_row_ else if (OB_FAIL(insert_tablet_ctx->init_datum_row(datum_row_))) { LOG_WARN("fail to init datum row", KR(ret)); - } else { + } else if (merge_ctx->merge_with_origin_data()) { + if (OB_FAIL(origin_table->scan(range_, allocator_, origin_iter_, false/*skip_read_lob*/))) { + LOG_WARN("fail to scan origin table", KR(ret)); + } + } + if (OB_SUCC(ret)) { deserialize_datums_ = datum_row_.storage_datums_ + merge_param.rowkey_column_num_ + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); deserialize_datum_cnt_ = merge_param.store_column_count_ - merge_param.rowkey_column_num_; rowkey_column_num_ = merge_param.rowkey_column_num_; store_column_count_ = merge_param.store_column_count_; - tablet_id_ = tablet_id; + tablet_id_ = merge_ctx->get_tablet_id(); table_data_desc_ = merge_param.table_data_desc_; heap_table_array_ = heap_table_array; pk_interval_ = pk_interval; @@ -869,37 +976,45 @@ int ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::RowIterator::inner LOG_WARN("ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::RowIterator not init", KR(ret), KP(this)); } else { - // get row from origin table if (pos_ == 0) { - const ObDatumRow *datum_row = nullptr; - if (OB_FAIL(origin_iter_->get_next_row(datum_row))) { - if (OB_UNLIKELY(OB_ITER_END != ret)) { - LOG_WARN("fail to get next row", KR(ret)); - } else { - // switch heap table - ret = OB_SUCCESS; - if (OB_FAIL(switch_next_heap_table())) { - if (OB_UNLIKELY(OB_ITER_END != ret)) { - LOG_WARN("fail to switch next heap table", KR(ret)); - } + if (origin_iter_ == nullptr) { + if (OB_FAIL(switch_next_heap_table())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("fail to switch next heap table", KR(ret)); } } - } else if (OB_UNLIKELY(datum_row->count_ != store_column_count_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected column count", KR(ret), K(store_column_count_), KPC(datum_row)); } else { - // copy rowkey columns - for (int64_t i = 0; i < rowkey_column_num_; ++i) { - datum_row_.storage_datums_[i] = datum_row->storage_datums_[i]; + // get row from origin table + const ObDatumRow *datum_row = nullptr; + if (OB_FAIL(origin_iter_->get_next_row(datum_row))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("fail to get next row", KR(ret)); + } else { + // switch heap table + ret = OB_SUCCESS; + if (OB_FAIL(switch_next_heap_table())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("fail to switch next heap table", KR(ret)); + } + } + } + } else if (OB_UNLIKELY(datum_row->count_ != store_column_count_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected column count", KR(ret), K(store_column_count_), KPC(datum_row)); + } else { + // copy rowkey columns + for (int64_t i = 0; i < rowkey_column_num_; ++i) { + datum_row_.storage_datums_[i] = datum_row->storage_datums_[i]; + } + // copy normal columns + for (int64_t + i = rowkey_column_num_, + j = rowkey_column_num_ + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); + i < datum_row->count_; ++i, ++j) { + datum_row_.storage_datums_[j] = datum_row->storage_datums_[i]; + } + result_row = &datum_row_; } - // copy normal columns - for (int64_t - i = rowkey_column_num_, - j = rowkey_column_num_ + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); - i < datum_row->count_; ++i, ++j) { - datum_row_.storage_datums_[j] = datum_row->storage_datums_[i]; - } - result_row = &datum_row_; } } // get row from load data @@ -1011,7 +1126,7 @@ int ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::construct_row_iter if (OB_ISNULL(row_iter = OB_NEWx(RowIterator, (&allocator)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to new RowIterator", KR(ret)); - } else if (OB_FAIL(row_iter->init(*merge_param_, merge_ctx_->get_tablet_id(), origin_table_, + } else if (OB_FAIL(row_iter->init(*merge_param_, merge_ctx_, origin_table_, sql_statistics_, lob_builder_, heap_table_array_, pk_interval_, insert_tablet_ctx_))) { LOG_WARN("fail to init row iter", KR(ret)); diff --git a/src/storage/direct_load/ob_direct_load_partition_merge_task.h b/src/storage/direct_load/ob_direct_load_partition_merge_task.h index 32230ec342..f0d5af1cd2 100644 --- a/src/storage/direct_load/ob_direct_load_partition_merge_task.h +++ b/src/storage/direct_load/ob_direct_load_partition_merge_task.h @@ -85,7 +85,7 @@ private: RowIterator(); virtual ~RowIterator(); int init(const ObDirectLoadMergeParam &merge_param, - const common::ObTabletID &tablet_id, + ObDirectLoadTabletMergeCtx *merge_ctx, ObDirectLoadOriginTable *origin_table, table::ObTableLoadSqlStatistics *sql_statistics, ObDirectLoadLobBuilder &lob_builder, @@ -94,7 +94,7 @@ private: ObDirectLoadInsertTabletContext *insert_tablet_ctx); int inner_get_next_row(blocksstable::ObDatumRow *&datum_row) override; private: - ObDirectLoadSSTableDataFuse data_fuse_; + ObIStoreRowIterator *data_iter_; blocksstable::ObDatumRow datum_row_; int64_t rowkey_column_num_; }; @@ -125,7 +125,7 @@ private: RowIterator(); virtual ~RowIterator(); int init(const ObDirectLoadMergeParam &merge_param, - const common::ObTabletID &tablet_id, + ObDirectLoadTabletMergeCtx *merge_ctx, ObDirectLoadOriginTable *origin_table, table::ObTableLoadSqlStatistics *sql_statistics, ObDirectLoadLobBuilder &lob_builder, @@ -134,7 +134,7 @@ private: ObDirectLoadInsertTabletContext *insert_tablet_ctx); int inner_get_next_row(blocksstable::ObDatumRow *&datum_row) override; private: - ObDirectLoadMultipleSSTableDataFuse data_fuse_; + ObIStoreRowIterator *data_iter_; blocksstable::ObDatumRow datum_row_; int64_t rowkey_column_num_; }; @@ -243,7 +243,8 @@ private: public: RowIterator(); virtual ~RowIterator(); - int init(const ObDirectLoadMergeParam &merge_param, const common::ObTabletID &tablet_id, + int init(const ObDirectLoadMergeParam &merge_param, + ObDirectLoadTabletMergeCtx *merge_ctx, ObDirectLoadOriginTable *origin_table, table::ObTableLoadSqlStatistics *sql_statistics, ObDirectLoadLobBuilder &lob_builder, diff --git a/src/storage/direct_load/ob_direct_load_range_splitter.cpp b/src/storage/direct_load/ob_direct_load_range_splitter.cpp index b8b563b2d0..eb57245fec 100644 --- a/src/storage/direct_load/ob_direct_load_range_splitter.cpp +++ b/src/storage/direct_load/ob_direct_load_range_splitter.cpp @@ -208,22 +208,7 @@ int ObDirectLoadRangeSplitUtils::construct_origin_table_rowkey_iters( } else { const ObITableReadInfo &read_info = origin_table->get_tablet_handle().get_obj()->get_rowkey_read_info(); - if (!ObDirectLoadInsertMode::need_origin_data(origin_table->get_meta().insert_mode_)) { - ObDirectLoadDatumRowkeyEmptyIterator *rowkey_iter = nullptr; - if (OB_ISNULL(rowkey_iter = OB_NEWx(ObDirectLoadDatumRowkeyEmptyIterator, &allocator))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to new ObDirectLoadDatumRowkeyEmptyIterator", KR(ret)); - } else if (OB_FAIL(rowkey_iters.push_back(rowkey_iter))) { - LOG_WARN("fail to push back rowkey iter", KR(ret)); - } - if (OB_FAIL(ret)) { - if (nullptr != rowkey_iter) { - rowkey_iter->~ObDirectLoadDatumRowkeyEmptyIterator(); - allocator.free(rowkey_iter); - rowkey_iter = nullptr; - } - } - } else if (nullptr != origin_table->get_major_sstable()) { + if (nullptr != origin_table->get_major_sstable()) { ObSSTable *major_sstable = origin_table->get_major_sstable(); ObIDirectLoadDatumRowkeyIterator *rowkey_iter = nullptr; if (OB_FAIL(ObDirectLoadRangeSplitUtils::construct_rowkey_iter( @@ -293,7 +278,7 @@ int ObDirectLoadRowkeyMergeRangeSplitter::init( if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("ObDirectLoadRowkeyMergeRangeSplitter init twice", KR(ret), KP(this)); - } else if (OB_UNLIKELY(rowkey_iters.empty() || nullptr == datum_utils)) { + } else if (OB_UNLIKELY(nullptr == datum_utils)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", KR(ret), K(rowkey_iters.count()), K(total_rowkey_count), KP(datum_utils)); @@ -431,89 +416,6 @@ int ObDirectLoadRowkeyMergeRangeSplitter::split_range(ObIArray &ra return ret; } -/** - * ObDirectLoadSSTableRangeSplitter - */ - -ObDirectLoadSSTableRangeSplitter::ObDirectLoadSSTableRangeSplitter() - : allocator_("TLD_SSTRGSplit"), total_block_count_(0), is_inited_(false) -{ - allocator_.set_tenant_id(MTL_ID()); - rowkey_iters_.set_tenant_id(MTL_ID()); -} - -ObDirectLoadSSTableRangeSplitter::~ObDirectLoadSSTableRangeSplitter() -{ - for (int64_t i = 0; i < rowkey_iters_.count(); ++i) { - ObIDirectLoadDatumRowkeyIterator *rowkey_iter = rowkey_iters_.at(i); - rowkey_iter->~ObIDirectLoadDatumRowkeyIterator(); - allocator_.free(rowkey_iter); - } - rowkey_iters_.reset(); -} - -int ObDirectLoadSSTableRangeSplitter::init(const ObIArray &sstable_array, - const ObStorageDatumUtils *datum_utils) -{ - int ret = OB_SUCCESS; - if (IS_INIT) { - ret = OB_INIT_TWICE; - LOG_WARN("ObDirectLoadSSTableRangeSplitter init twice", KR(ret), KP(this)); - } else if (OB_UNLIKELY(sstable_array.empty() || nullptr == datum_utils)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(sstable_array), KP(datum_utils)); - } else { - if (OB_FAIL(construct_rowkey_iters(sstable_array))) { - LOG_WARN("fail to construct rowkey itres", KR(ret)); - } else if (OB_FAIL( - rowkey_merge_splitter_.init(rowkey_iters_, total_block_count_, datum_utils))) { - LOG_WARN("fail to init rowkey merger", KR(ret)); - } else { - is_inited_ = true; - } - } - return ret; -} - -int ObDirectLoadSSTableRangeSplitter::construct_rowkey_iters( - const ObIArray &sstable_array) -{ - int ret = OB_SUCCESS; - for (int64_t i = 0; OB_SUCC(ret) && i < sstable_array.count(); ++i) { - ObDirectLoadSSTable *sstable = sstable_array.at(i); - ObIDirectLoadDatumRowkeyIterator *rowkey_iter = nullptr; - total_block_count_ += sstable->get_meta().index_block_count_; - if (OB_FAIL( - ObDirectLoadRangeSplitUtils::construct_rowkey_iter(sstable, allocator_, rowkey_iter))) { - LOG_WARN("fail to construct rowkey iter", KR(ret)); - } else if (OB_FAIL(rowkey_iters_.push_back(rowkey_iter))) { - LOG_WARN("fail to push back rowkey iter", KR(ret)); - } - if (OB_FAIL(ret)) { - if (nullptr != rowkey_iter) { - rowkey_iter->~ObIDirectLoadDatumRowkeyIterator(); - allocator_.free(rowkey_iter); - rowkey_iter = nullptr; - } - } - } - return ret; -} - -int ObDirectLoadSSTableRangeSplitter::split_range(ObIArray &range_array, - int64_t max_range_count, - ObIAllocator &allocator) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObDirectLoadSSTableRangeSplitter not init", KR(ret), KP(this)); - } else if (OB_FAIL(rowkey_merge_splitter_.split_range(range_array, max_range_count, allocator))) { - LOG_WARN("fail to split range", KR(ret)); - } - return ret; -} - /** * ObDirectLoadMergeRangeSplitter */ @@ -544,13 +446,13 @@ int ObDirectLoadMergeRangeSplitter::init(ObDirectLoadOriginTable *origin_table, if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("ObDirectLoadMergeRangeSplitter init twice", KR(ret), KP(this)); - } else if (OB_UNLIKELY(nullptr == origin_table || !origin_table->is_valid() || + } else if (OB_UNLIKELY((nullptr != origin_table && !origin_table->is_valid()) || nullptr == datum_utils)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", KR(ret), KPC(origin_table), K(sstable_array), KP(datum_utils)); } else { scan_range_.set_whole_range(); - if (OB_FAIL(ObDirectLoadRangeSplitUtils::construct_origin_table_rowkey_iters( + if (nullptr != origin_table && OB_FAIL(ObDirectLoadRangeSplitUtils::construct_origin_table_rowkey_iters( origin_table, scan_range_, allocator_, total_block_count_, rowkey_iters_))) { LOG_WARN("fail to construct origin table rowkey iters", KR(ret)); } else if (OB_FAIL(construct_sstable_rowkey_iters(sstable_array))) { @@ -640,8 +542,8 @@ int ObDirectLoadMultipleMergeTabletRangeSplitter::init( if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("ObDirectLoadMultipleMergeTabletRangeSplitter init twice", KR(ret), KP(this)); - } else if (OB_UNLIKELY(!tablet_id.is_valid() || nullptr == origin_table || - !origin_table->is_valid() || !table_data_desc.is_valid() || + } else if (OB_UNLIKELY(!tablet_id.is_valid() || (nullptr != origin_table && + !origin_table->is_valid()) || !table_data_desc.is_valid() || nullptr == datum_utils)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", KR(ret), K(tablet_id), KPC(origin_table), K(sstable_array), @@ -649,7 +551,7 @@ int ObDirectLoadMultipleMergeTabletRangeSplitter::init( } else { tablet_id_ = tablet_id; scan_range_.set_whole_range(); - if (OB_FAIL(ObDirectLoadRangeSplitUtils::construct_origin_table_rowkey_iters( + if (nullptr != origin_table && OB_FAIL(ObDirectLoadRangeSplitUtils::construct_origin_table_rowkey_iters( origin_table, scan_range_, allocator_, total_block_count_, rowkey_iters_))) { LOG_WARN("fail to construct origin table rowkey iters", KR(ret)); } else if (OB_FAIL( @@ -1062,8 +964,8 @@ int ObDirectLoadMultipleMergeRangeSplitter::split_range(ObTabletID &tablet_id, if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObDirectLoadMultipleMergeRangeSplitter not init", KR(ret), KP(this)); - } else if (OB_UNLIKELY(!tablet_id.is_valid() || nullptr == origin_table || - !origin_table->is_valid() || max_range_count <= 0)) { + } else if (OB_UNLIKELY( + !tablet_id.is_valid() || (nullptr != origin_table && !origin_table->is_valid()) || max_range_count <= 0)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", KR(ret), K(tablet_id), KPC(origin_table), K(max_range_count)); } else if (last_tablet_id_.is_valid() && OB_UNLIKELY(last_tablet_id_.compare(tablet_id) >= 0)) { @@ -1076,7 +978,7 @@ int ObDirectLoadMultipleMergeRangeSplitter::split_range(ObTabletID &tablet_id, tmp_allocator.set_tenant_id(MTL_ID()); origin_rowkey_array.set_block_allocator(ModulePageAllocator(tmp_allocator)); multiple_rowkey_array.set_block_allocator(ModulePageAllocator(tmp_allocator)); - if (OB_FAIL(get_rowkeys_by_origin(origin_table, origin_rowkey_array, tmp_allocator))) { + if (nullptr != origin_table && OB_FAIL(get_rowkeys_by_origin(origin_table, origin_rowkey_array, tmp_allocator))) { LOG_WARN("fail to get rowkeys by origin", KR(ret)); } else if (OB_FAIL(get_rowkeys_by_multiple(tablet_id, multiple_rowkey_array, tmp_allocator))) { LOG_WARN("fail to get rowkeys by multiple", KR(ret)); diff --git a/src/storage/direct_load/ob_direct_load_range_splitter.h b/src/storage/direct_load/ob_direct_load_range_splitter.h index a4b569342a..85b43af52c 100644 --- a/src/storage/direct_load/ob_direct_load_range_splitter.h +++ b/src/storage/direct_load/ob_direct_load_range_splitter.h @@ -83,25 +83,6 @@ private: bool is_inited_; }; -class ObDirectLoadSSTableRangeSplitter -{ -public: - ObDirectLoadSSTableRangeSplitter(); - ~ObDirectLoadSSTableRangeSplitter(); - int init(const common::ObIArray &sstable_array, - const blocksstable::ObStorageDatumUtils *datum_utils); - int split_range(common::ObIArray &range_array, - int64_t max_range_count, common::ObIAllocator &allocator); -private: - int construct_rowkey_iters(const common::ObIArray &sstable_array); -private: - common::ObArenaAllocator allocator_; - ObArray rowkey_iters_; - int64_t total_block_count_; - ObDirectLoadRowkeyMergeRangeSplitter rowkey_merge_splitter_; - bool is_inited_; -}; - class ObDirectLoadMergeRangeSplitter { public: diff --git a/src/storage/direct_load/ob_direct_load_rowkey_merger.h b/src/storage/direct_load/ob_direct_load_rowkey_merger.h index 85b5381153..42b66a4515 100644 --- a/src/storage/direct_load/ob_direct_load_rowkey_merger.h +++ b/src/storage/direct_load/ob_direct_load_rowkey_merger.h @@ -97,7 +97,7 @@ int ObDirectLoadRowkeyMerger::init(const common::ObIArray::get_next_rowkey(const Rowkey *&ro ret = common::OB_NOT_INIT; STORAGE_LOG(WARN, "ObDirectLoadRowkeyMerger not init", KR(ret), KP(this)); } else { - if (1 == iters_->count()) { + if (0 == iters_->count()) { + ret = OB_ITER_END; + } else if (1 == iters_->count()) { if (OB_FAIL(direct_get_next_rowkey(rowkey))) { if (OB_UNLIKELY(common::OB_ITER_END != ret)) { STORAGE_LOG(WARN, "fail to direct get next rowkey", KR(ret)); diff --git a/src/storage/direct_load/ob_direct_load_struct.h b/src/storage/direct_load/ob_direct_load_struct.h index 1743071e9e..8d0945a391 100644 --- a/src/storage/direct_load/ob_direct_load_struct.h +++ b/src/storage/direct_load/ob_direct_load_struct.h @@ -66,7 +66,6 @@ struct ObDirectLoadInsertMode static bool is_type_valid(const Type type); static bool is_valid_for_full_method(const Type type) { return NORMAL == type || OVERWRITE == type; } static bool is_valid_for_incremental_method(const Type type) { return NORMAL == type || INC_REPLACE == type; } - static bool need_origin_data(const Type type) { return NORMAL == type; } }; } // namespace storage diff --git a/src/storage/lob/ob_lob_meta.cpp b/src/storage/lob/ob_lob_meta.cpp index 46c4b9456b..679a79628f 100644 --- a/src/storage/lob/ob_lob_meta.cpp +++ b/src/storage/lob/ob_lob_meta.cpp @@ -905,11 +905,6 @@ void ObLobMetaWriteIter::reuse() } -void ObLobMetaWriteIter::set_data(const ObString& data) -{ - data_ = data; -} - int ObLobMetaWriteIter::update_disk_lob_locator(ObLobMetaWriteResult &result) { int ret = OB_SUCCESS; @@ -936,7 +931,7 @@ int ObLobMetaWriteIter::update_disk_lob_locator(ObLobMetaWriteResult &result) LOG_DEBUG("non incremental_direct_load does not have seq_no, so skip", KPC(lob_common_), KPC(lob_outrow_ctx), KPC(lob_data), K(result)); } else { transaction::ObTxSEQ cur_seq = transaction::ObTxSEQ::cast_from_int(lob_outrow_ctx->seq_no_st_) + lob_outrow_ctx->seq_no_cnt_; - + result.seq_no_ = cur_seq.cast_to_int(); // update seq_no_cnt lob_outrow_ctx->seq_no_cnt_++; LOG_DEBUG("disk lob locator info", K(cur_seq), KPC(lob_common_), KPC(lob_outrow_ctx), KPC(lob_data), K(result)); diff --git a/src/storage/lob/ob_lob_meta.h b/src/storage/lob/ob_lob_meta.h index 93275c9424..1dff7a1fc9 100644 --- a/src/storage/lob/ob_lob_meta.h +++ b/src/storage/lob/ob_lob_meta.h @@ -132,14 +132,14 @@ class ObLobQueryIter; class ObLobMetaManager; struct ObLobMetaWriteResult { - ObLobMetaWriteResult() : info_(), data_(), need_alloc_macro_id_(false), is_update_(false), old_info_() {} + ObLobMetaWriteResult() : info_(), data_(), need_alloc_macro_id_(false), is_update_(false), old_info_(), seq_no_(0) {} ObLobMetaInfo info_; ObString data_; bool need_alloc_macro_id_; bool is_update_; ObLobMetaInfo old_info_; - - TO_STRING_KV(K_(is_update), K_(info), K_(old_info), K_(data)); + int64_t seq_no_; + TO_STRING_KV(K_(is_update), K_(seq_no), K_(info), K_(old_info), K_(data)); }; class ObLobMetaWriteIter { @@ -179,7 +179,7 @@ public: int close(); void set_end() { is_end_ = true; } void reuse(); - void set_data(const ObString& data); + TO_STRING_KV(K_(seq_id), K_(offset), K_(lob_id), K_(piece_id), K_(coll_type), K_(piece_block_size), K_(scan_iter), K_(padding_size), K_(seq_id_end), K_(last_info), K_(is_store_char_len)); private: diff --git a/src/storage/lob/ob_lob_util.cpp b/src/storage/lob/ob_lob_util.cpp index 346a1c45d7..20215e04a0 100644 --- a/src/storage/lob/ob_lob_util.cpp +++ b/src/storage/lob/ob_lob_util.cpp @@ -339,10 +339,8 @@ int ObInsertLobColumnHelper::insert_lob_column(ObIAllocator &allocator, } } else { ObTransService *txs = MTL(transaction::ObTransService*); - ObTxReadSnapshot snapshot; - // 4.0 text tc compatiable ObLobAccessParam lob_param; - // lob_param.tx_desc_ = tx_desc; + lob_param.tx_desc_ = tx_desc; // lob_param.snapshot_ = snapshot; lob_param.sql_mode_ = SMO_DEFAULT; lob_param.ls_id_ = ls_id;