From 13f9f3dea861b74f1b73051967c693158d8026ea Mon Sep 17 00:00:00 2001 From: obdev Date: Mon, 30 Sep 2024 07:16:12 +0000 Subject: [PATCH] [FEAT MERGE]inc direct load support local index Co-authored-by: medcll <527998250@qq.com> Co-authored-by: coolfishchen Co-authored-by: suz-yang --- src/observer/CMakeLists.txt | 6 + .../table_load/ob_table_load_coordinator.cpp | 31 +- .../ob_table_load_data_row_handler.cpp | 293 +++++++++++ .../ob_table_load_data_row_handler.h | 76 +++ .../ob_table_load_error_row_handler.cpp | 200 +------- .../ob_table_load_error_row_handler.h | 16 +- .../ob_table_load_general_table_compactor.cpp | 14 +- .../ob_table_load_general_table_compactor.h | 1 + .../ob_table_load_index_row_handler.cpp | 42 ++ .../ob_table_load_index_row_handler.h | 57 +++ .../ob_table_load_index_table_projector.cpp | 196 ++++++++ .../ob_table_load_index_table_projector.h | 52 ++ .../ob_table_load_mem_compactor.cpp | 27 +- .../table_load/ob_table_load_mem_compactor.h | 1 + .../table_load/ob_table_load_merger.cpp | 137 +++-- .../table_load/ob_table_load_merger.h | 8 +- .../ob_table_load_merger_manager.cpp | 126 +++++ .../table_load/ob_table_load_merger_manager.h | 43 ++ ...ble_load_multiple_heap_table_compactor.cpp | 18 +- ...table_load_multiple_heap_table_compactor.h | 1 + ...ble_load_open_insert_table_ctx_manager.cpp | 89 ++++ ...table_load_open_insert_table_ctx_manager.h | 43 ++ .../ob_table_load_parallel_merge_ctx.cpp | 16 +- .../ob_table_load_parallel_merge_ctx.h | 4 +- ...le_load_parallel_merge_table_compactor.cpp | 4 +- .../ob_table_load_pre_sort_writer.cpp | 12 +- .../table_load/ob_table_load_pre_sorter.cpp | 30 +- .../table_load/ob_table_load_schema.cpp | 32 ++ .../table_load/ob_table_load_schema.h | 5 +- .../table_load/ob_table_load_service.cpp | 21 +- .../table_load/ob_table_load_store.cpp | 46 +- src/observer/table_load/ob_table_load_store.h | 1 + .../table_load/ob_table_load_store_ctx.cpp | 452 ++++++++--------- .../table_load/ob_table_load_store_ctx.h | 27 +- .../ob_table_load_store_table_ctx.cpp | 473 ++++++++++++++++++ .../ob_table_load_store_table_ctx.h | 87 ++++ .../ob_table_load_store_trans_px_writer.cpp | 7 +- .../ob_table_load_table_compactor.cpp | 55 +- .../ob_table_load_table_compactor.h | 20 +- .../table_load/ob_table_load_table_ctx.h | 2 +- .../ob_table_load_trans_bucket_writer.cpp | 4 +- .../table_load/ob_table_load_trans_store.cpp | 28 +- .../ob_direct_load_conflict_check.cpp | 17 +- .../ob_direct_load_conflict_check.h | 1 - .../direct_load/ob_direct_load_data_fuse.cpp | 12 +- .../ob_direct_load_data_insert.cpp | 12 +- .../ob_direct_load_dml_row_handler.h | 8 +- ...rect_load_external_multi_partition_row.cpp | 6 +- ...direct_load_external_multi_partition_row.h | 1 + ...ct_load_external_multi_partition_table.cpp | 2 +- .../ob_direct_load_external_row.cpp | 13 +- .../direct_load/ob_direct_load_external_row.h | 8 +- .../ob_direct_load_external_table_builder.cpp | 2 +- ...ob_direct_load_fast_heap_table_builder.cpp | 2 +- .../ob_direct_load_insert_table_ctx.cpp | 4 +- .../ob_direct_load_insert_table_ctx.h | 4 +- .../ob_direct_load_lob_meta_row_iter.h | 15 +- .../direct_load/ob_direct_load_mem_dump.cpp | 1 + .../direct_load/ob_direct_load_merge_ctx.cpp | 1 + .../direct_load/ob_direct_load_merge_ctx.h | 7 +- .../ob_direct_load_multiple_datum_row.cpp | 14 +- .../ob_direct_load_multiple_datum_row.h | 3 +- ...b_direct_load_multiple_sstable_builder.cpp | 2 +- ...irect_load_multiple_sstable_scan_merge.cpp | 1 + .../ob_direct_load_partition_merge_task.cpp | 14 +- .../ob_direct_load_partition_merge_task.h | 2 + .../ob_direct_load_sstable_builder.cpp | 2 +- .../ob_direct_load_sstable_scan_merge.cpp | 1 + .../ob_direct_load_sstable_scanner.cpp | 1 + 69 files changed, 2298 insertions(+), 661 deletions(-) create mode 100644 src/observer/table_load/ob_table_load_data_row_handler.cpp create mode 100644 src/observer/table_load/ob_table_load_data_row_handler.h create mode 100644 src/observer/table_load/ob_table_load_index_row_handler.cpp create mode 100644 src/observer/table_load/ob_table_load_index_row_handler.h create mode 100644 src/observer/table_load/ob_table_load_index_table_projector.cpp create mode 100644 src/observer/table_load/ob_table_load_index_table_projector.h create mode 100644 src/observer/table_load/ob_table_load_merger_manager.cpp create mode 100644 src/observer/table_load/ob_table_load_merger_manager.h create mode 100644 src/observer/table_load/ob_table_load_open_insert_table_ctx_manager.cpp create mode 100644 src/observer/table_load/ob_table_load_open_insert_table_ctx_manager.h create mode 100644 src/observer/table_load/ob_table_load_store_table_ctx.cpp create mode 100644 src/observer/table_load/ob_table_load_store_table_ctx.h diff --git a/src/observer/CMakeLists.txt b/src/observer/CMakeLists.txt index 060f2805d..37655e7e6 100644 --- a/src/observer/CMakeLists.txt +++ b/src/observer/CMakeLists.txt @@ -273,6 +273,12 @@ ob_set_subtarget(ob_server table_load table_load/ob_table_load_multiple_heap_table_compactor.cpp table_load/ob_table_load_redef_table.cpp table_load/ob_table_load_stat.cpp + table_load/ob_table_load_index_table_projector.cpp + table_load/ob_table_load_store_table_ctx.cpp + table_load/ob_table_load_merger_manager.cpp + table_load/ob_table_load_index_row_handler.cpp + table_load/ob_table_load_data_row_handler.cpp + table_load/ob_table_load_open_insert_table_ctx_manager.cpp table_load/ob_table_load_pre_sorter.cpp table_load/ob_table_load_mem_chunk_manager.cpp table_load/ob_table_load_pre_sort_writer.cpp diff --git a/src/observer/table_load/ob_table_load_coordinator.cpp b/src/observer/table_load/ob_table_load_coordinator.cpp index 125e1dfc9..3c0593d44 100644 --- a/src/observer/table_load/ob_table_load_coordinator.cpp +++ b/src/observer/table_load/ob_table_load_coordinator.cpp @@ -295,6 +295,7 @@ int ObTableLoadCoordinator::gen_apply_arg(ObDirectLoadResourceApplyArg &apply_ar } else { bool include_cur_addr = false; bool need_sort = ctx_->param_.need_sort_; + bool main_need_sort = ctx_->param_.need_sort_; int64_t total_partitions = 0; ObArray partitions; int64_t store_server_count = all_leader_info_array.count(); @@ -374,41 +375,47 @@ int ObTableLoadCoordinator::gen_apply_arg(ObDirectLoadResourceApplyArg &apply_ar // 直接写宏块需要的内存，对于非排序模式，每个分区各自写宏块，所以要乘分区数 min_unsort_memory = MACROBLOCK_BUFFER_SIZE * partitions[i] * write_session_count; if (min_unsort_memory <= memory_limit) { - need_sort = false; + main_need_sort = false; unit.memory_size_ = min_unsort_memory; } else { - need_sort = ctx_->param_.need_sort_; // allow forced non-sorting + main_need_sort = ctx_->param_.need_sort_; // allow forced non-sorting unit.memory_size_ = MIN(ObTableLoadAssignedMemoryManager::MIN_SORT_MEMORY_PER_TASK, memory_limit); } } else { // 取写宏块或写临时文件需要内存的最小值，对于非排序模式，每个分区各自写临时文件，所以要乘分区数 min_unsort_memory = SSTABLE_BUFFER_SIZE * partitions[i] * unit.thread_count_; - if (need_sort) { + if (main_need_sort) { unit.memory_size_ = MIN(unit.thread_count_ * ObDirectLoadExternalMultiPartitionRowChunk::MIN_MEMORY_LIMIT * 4, memory_limit); } else { // hint指定不排序，如果不排序内存大于内存上限，要改成走排序模式，一般是分区数较大的场景 if (min_unsort_memory < memory_limit) { unit.memory_size_ = MAX(min_unsort_memory, MACROBLOCK_BUFFER_SIZE * write_session_count); } else { - need_sort = true; + main_need_sort = true; unit.memory_size_ = MIN(unit.thread_count_ * ObDirectLoadExternalMultiPartitionRowChunk::MIN_MEMORY_LIMIT * 4, memory_limit); } } } - if (need_sort) { - // 只要有一个节点走排序模式，所有节点统一走排序模式 + + if (main_need_sort) { break; } } - if (need_sort) { - // 排序模式，所有节点都分配固定的最小排序内存 + ObSchemaGetterGuard schema_guard; + const ObTableSchema *table_schema = nullptr; + if (OB_FAIL(ObTableLoadSchema::get_table_schema(tenant_id, ctx_->ddl_param_.dest_table_id_, schema_guard, table_schema))) { + LOG_WARN("fail to get table schema", KR(ret), K(tenant_id), K(ctx_->ddl_param_.dest_table_id_)); + } else if (main_need_sort || (ObDirectLoadMethod::is_incremental(ctx_->param_.method_) && + table_schema->get_simple_index_infos().count() > 0)) { + need_sort = true; for (int64_t i = 0; i < store_server_count; i++) { ObDirectLoadResourceUnit &unit = apply_arg.apply_array_[i]; unit.memory_size_ = MIN(unit.thread_count_ * ObDirectLoadExternalMultiPartitionRowChunk::MIN_MEMORY_LIMIT * 4, memory_limit); } + } else { + need_sort = false; } } - if (OB_SUCC(ret)) { ObDirectLoadResourceOpRes apply_res; if (OB_FAIL(ObTableLoadResourceService::apply_resource(apply_arg, apply_res))) { @@ -425,8 +432,8 @@ int ObTableLoadCoordinator::gen_apply_arg(ObDirectLoadResourceApplyArg &apply_ar ctx_->param_.session_count_ = coordinator_session_count; ctx_->param_.write_session_count_ = write_session_count; ctx_->param_.exe_mode_ = (ctx_->schema_.is_heap_table_ ? - (need_sort ? ObTableLoadExeMode::MULTIPLE_HEAP_TABLE_COMPACT : ObTableLoadExeMode::FAST_HEAP_TABLE) : - (need_sort ? ObTableLoadExeMode::MEM_COMPACT : ObTableLoadExeMode::GENERAL_TABLE_COMPACT)); + (main_need_sort ? ObTableLoadExeMode::MULTIPLE_HEAP_TABLE_COMPACT : ObTableLoadExeMode::FAST_HEAP_TABLE) : + (main_need_sort ? ObTableLoadExeMode::MEM_COMPACT : ObTableLoadExeMode::GENERAL_TABLE_COMPACT)); ctx_->job_stat_->parallel_ = coordinator_session_count; if (OB_FAIL(ObTableLoadService::add_assigned_task(apply_arg))) { LOG_WARN("fail to add_assigned_task", KR(ret)); @@ -1709,7 +1716,7 @@ public: LOG_WARN("column count doesn't match value count", KR(ret), K(src_obj_row), K(ctx_->param_.column_count_)); ObNewRow new_row(src_obj_row.cells_, src_obj_row.count_); - if (OB_FAIL(error_row_handler->handle_error_row(ret, new_row))) { + if (OB_FAIL(error_row_handler->handle_error_row(ret))) { LOG_WARN("fail to handle error row", KR(ret)); } } else if (OB_FAIL(src_obj_row.project(idx_array, out_obj_row))) { diff --git a/src/observer/table_load/ob_table_load_data_row_handler.cpp b/src/observer/table_load/ob_table_load_data_row_handler.cpp new file mode 100644 index 000000000..0bf14ab1a --- /dev/null +++ b/src/observer/table_load/ob_table_load_data_row_handler.cpp @@ -0,0 +1,293 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#define USING_LOG_PREFIX SERVER + +#include "observer/table_load/ob_table_load_data_row_handler.h" +#include "observer/table_load/ob_table_load_error_row_handler.h" +#include "observer/table_load/ob_table_load_store_table_ctx.h" +#include "observer/table_load/ob_table_load_schema.h" +#include "observer/table_load/ob_table_load_index_table_projector.h" +#include "storage/direct_load/ob_direct_load_i_table.h" + +namespace oceanbase +{ +namespace observer +{ +ObTableLoadDataRowHandler::ObTableLoadDataRowHandler() + : error_row_handler_(nullptr), + index_store_table_ctx_map_(nullptr), + result_info_(nullptr), + is_inited_(false) +{ +} + +ObTableLoadDataRowHandler::~ObTableLoadDataRowHandler() +{ + +} + +int ObTableLoadDataRowHandler::init( + const ObTableLoadParam ¶m, table::ObTableLoadResultInfo &result_info, + ObTableLoadErrorRowHandler *error_row_handler, + hash::ObHashMap *index_store_table_ctx_map) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + int ret = OB_INIT_TWICE; + LOG_WARN("ObTableLoadDataRowHandler init twice", KR(ret), KP(this)); + } else if (OB_ISNULL(error_row_handler) || OB_ISNULL(index_store_table_ctx_map)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("allocator is null", KR(ret), KP(error_row_handler), KP(index_store_table_ctx_map)); + } else { + error_row_handler_ = error_row_handler; + index_store_table_ctx_map_ = index_store_table_ctx_map; + result_info_ = &result_info; + dup_action_ = param.dup_action_; + is_inited_ = true; + } + return ret; +} + +int ObTableLoadDataRowHandler::handle_insert_row(const ObTabletID tablet_id, + const blocksstable::ObDatumRow &row) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTableLoadDataRowHandler not init", KR(ret), KP(this)); + } else { + FOREACH_X(iter, *index_store_table_ctx_map_, OB_SUCC(ret)) + { + ObTabletID index_tablet_id; + blocksstable::ObDatumRow index_row; + ObIDirectLoadPartitionTableBuilder *builder = nullptr; + if (OB_FAIL( + iter->second->project_->projector(tablet_id, row, false, index_tablet_id, index_row))) { + LOG_WARN("fail to projector", KR(ret), K(tablet_id), K(row)); + } else { + index_row.row_flag_.set_flag(blocksstable::DF_INSERT, + blocksstable::ObDmlRowFlagType::DF_TYPE_INSERT_DELETE); + if (OB_FAIL(iter->second->get_index_table_builder(builder))) { + LOG_WARN("get builder failed", KR(ret)); + } else if (OB_FAIL(builder->append_row(index_tablet_id, 0, index_row))) { + LOG_WARN("add row failed", KR(ret), K(index_row)); + } + } + } + if (OB_SUCC(ret)) { + ATOMIC_INC(&result_info_->rows_affected_); + } + } + return ret; +} + +int ObTableLoadDataRowHandler::handle_insert_row_with_multi_version(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTableLoadDataRowHandler not init", KR(ret), KP(this)); + } else { + FOREACH_X(iter, *index_store_table_ctx_map_, OB_SUCC(ret)) + { + ObTabletID index_tablet_id; + blocksstable::ObDatumRow index_row; + ObIDirectLoadPartitionTableBuilder *builder = nullptr; + if (OB_FAIL( + iter->second->project_->projector(tablet_id, row, true, index_tablet_id, index_row))) { + LOG_WARN("fail to projector", KR(ret), K(tablet_id), K(row)); + } else { + index_row.row_flag_.set_flag(blocksstable::DF_INSERT, + blocksstable::ObDmlRowFlagType::DF_TYPE_INSERT_DELETE); + if (OB_FAIL(iter->second->get_index_table_builder(builder))) { + LOG_WARN("get builder failed", KR(ret)); + } else if (OB_FAIL(builder->append_row(index_tablet_id, 0, index_row))) { + LOG_WARN("add row failed", KR(ret), K(index_row)); + } + } + } + if (OB_SUCC(ret)) { + ATOMIC_INC(&result_info_->rows_affected_); + } + } + return ret; +} + +int ObTableLoadDataRowHandler::handle_update_row(const blocksstable::ObDatumRow &row) +{ + UNUSED(row); + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else { + if (ObLoadDupActionType::LOAD_STOP_ON_DUP == dup_action_) { + if (OB_FAIL(error_row_handler_->handle_error_row(OB_ERR_PRIMARY_KEY_DUPLICATE))) { + LOG_WARN("fail to handle error row", KR(ret)); + } + } else if (ObLoadDupActionType::LOAD_REPLACE == dup_action_) { + ATOMIC_AAF(&result_info_->rows_affected_, 2); + ATOMIC_INC(&result_info_->deleted_); + } else if (ObLoadDupActionType::LOAD_IGNORE == dup_action_) { + ATOMIC_INC(&result_info_->skipped_); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected dup action", KR(ret), K_(dup_action)); + } + } + return ret; +} + +int ObTableLoadDataRowHandler::handle_update_row(const ObTabletID tablet_id, + const blocksstable::ObDatumRow &old_row, + const blocksstable::ObDatumRow &new_row, + const blocksstable::ObDatumRow *&result_row) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTableLoadDataRowHandler not init", KR(ret), KP(this)); + } else { + if (ObLoadDupActionType::LOAD_STOP_ON_DUP == dup_action_) { + if (OB_FAIL(error_row_handler_->handle_error_row(OB_ERR_PRIMARY_KEY_DUPLICATE))) { + LOG_WARN("fail to handle error row", KR(ret)); + } else { + result_row = &old_row; + } + } else if (ObLoadDupActionType::LOAD_IGNORE == dup_action_) { + result_row = &old_row; + ATOMIC_INC(&result_info_->skipped_); + } else if (ObLoadDupActionType::LOAD_REPLACE == dup_action_) { + result_row = &new_row; + ATOMIC_INC(&result_info_->deleted_); + ATOMIC_AAF(&result_info_->rows_affected_, 2); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected dup action", KR(ret), K_(dup_action)); + } + if (OB_SUCC(ret) && result_row == &new_row) { + FOREACH_X(iter, *index_store_table_ctx_map_, OB_SUCC(ret)) { + ObTabletID index_tablet_id; + blocksstable::ObDatumRow index_row_old; + blocksstable::ObDatumRow index_row_new; + ObIDirectLoadPartitionTableBuilder *builder = nullptr; + if (OB_FAIL(iter->second->project_->projector(tablet_id, old_row, false, index_tablet_id, + index_row_old))) { + LOG_WARN("fail to projector", KR(ret), K(tablet_id), K(old_row)); + } else if (OB_FAIL(iter->second->project_->projector(tablet_id, new_row, false, + index_tablet_id, index_row_new))) { + LOG_WARN("fail to projector", KR(ret), K(tablet_id), K(new_row)); + } else if (OB_FAIL(iter->second->get_index_table_builder(builder))) { + LOG_WARN("get builder failed", KR(ret)); + } else { + ObDatumRowkey old_key(index_row_old.storage_datums_, + iter->second->schema_->rowkey_column_count_); + ObDatumRowkey new_key(index_row_new.storage_datums_, + iter->second->schema_->rowkey_column_count_); + int cmp_ret; + if (OB_FAIL(old_key.compare(new_key, iter->second->schema_->datum_utils_, cmp_ret))) { + LOG_WARN("fail to compare", KR(ret)); + } else { + // when cmp_ret == 0, insert new row is equal to delete old row and insert new row. + if (0 != cmp_ret) { + index_row_old.row_flag_.set_flag(blocksstable::DF_DELETE, + blocksstable::ObDmlRowFlagType::DF_TYPE_NORMAL); + if (OB_FAIL(builder->append_row(index_tablet_id, 0, index_row_old))) { + LOG_WARN("add row failed", KR(ret), K(index_row_old)); + } + } + } + if (OB_SUCC(ret)) { + index_row_new.row_flag_.set_flag(blocksstable::DF_INSERT, + blocksstable::ObDmlRowFlagType::DF_TYPE_INSERT_DELETE); + if (OB_FAIL(builder->append_row(index_tablet_id, 0, index_row_new))) { + LOG_WARN("add row failed", KR(ret), K(index_row_new)); + } + } + } + } + } + } + return ret; +} + +int ObTableLoadDataRowHandler::handle_update_row(common::ObArray &rows, + const ObDirectLoadExternalRow *&row) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_UNLIKELY(rows.count() < 2)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret)); + } else { + int64_t duplicate_row_count = rows.count() - 1; + lib::ob_sort(rows.begin(), rows.end(), ObTableLoadDataRowHandler::external_row_compare); + if (ObLoadDupActionType::LOAD_STOP_ON_DUP == dup_action_) { + if (OB_FAIL(error_row_handler_->handle_error_row(OB_ERR_PRIMARY_KEY_DUPLICATE, duplicate_row_count))) { + LOG_WARN("fail to handle error row", KR(ret)); + } else { + row = rows.at(0); + } + } else if (ObLoadDupActionType::LOAD_REPLACE == dup_action_) { + ATOMIC_AAF(&result_info_->rows_affected_, 2 * duplicate_row_count); + ATOMIC_AAF(&result_info_->deleted_, duplicate_row_count); + row = rows.at(duplicate_row_count); + } else if (ObLoadDupActionType::LOAD_IGNORE == dup_action_) { + ATOMIC_AAF(&result_info_->skipped_, duplicate_row_count); + row = rows.at(0); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected dup action", KR(ret), K_(dup_action)); + } + } + return ret; +} + +int ObTableLoadDataRowHandler::handle_update_row( + common::ObArray &rows, + const ObDirectLoadMultipleDatumRow *&row) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_UNLIKELY(rows.count() < 2)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret)); + } else { + int64_t duplicate_row_count = rows.count() - 1; + lib::ob_sort(rows.begin(), rows.end(), ObTableLoadDataRowHandler::multiple_external_row_compare); + if (ObLoadDupActionType::LOAD_STOP_ON_DUP == dup_action_) { + if (OB_FAIL(error_row_handler_->handle_error_row(OB_ERR_PRIMARY_KEY_DUPLICATE, duplicate_row_count))) { + LOG_WARN("fail to handle error row", KR(ret)); + } else { + row = rows.at(0); + } + } else if (ObLoadDupActionType::LOAD_REPLACE == dup_action_) { + ATOMIC_AAF(&result_info_->rows_affected_, 2 * duplicate_row_count); + ATOMIC_AAF(&result_info_->deleted_, duplicate_row_count); + row = rows.at(duplicate_row_count); + } else if (ObLoadDupActionType::LOAD_IGNORE == dup_action_) { + ATOMIC_AAF(&result_info_->skipped_, duplicate_row_count); + row = rows.at(0); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected dup action", KR(ret), K_(dup_action)); + } + } + return ret; +} + +} // namespace observer +} // namespace oceanbase \ No newline at end of file diff --git a/src/observer/table_load/ob_table_load_data_row_handler.h b/src/observer/table_load/ob_table_load_data_row_handler.h new file mode 100644 index 000000000..b71945287 --- /dev/null +++ b/src/observer/table_load/ob_table_load_data_row_handler.h @@ -0,0 +1,76 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#pragma once + +#include "storage/blocksstable/ob_datum_row.h" +#include "sql/engine/cmd/ob_load_data_utils.h" +#include "storage/direct_load/ob_direct_load_dml_row_handler.h" + +namespace oceanbase +{ +namespace table +{ +class ObTableLoadResultInfo; +} // namespace table +namespace observer +{ +class ObTableLoadErrorRowHandler; +class ObTableLoadParam; +class ObTableLoadStoreTableCtx; + +class ObTableLoadDataRowHandler : public ObDirectLoadDMLRowHandler +{ +private: + static bool external_row_compare(const ObDirectLoadExternalRow *lhs, + const ObDirectLoadExternalRow *rhs) + { + return lhs->seq_no_ < rhs->seq_no_; + } + static bool multiple_external_row_compare(const ObDirectLoadMultipleDatumRow *lhs, + const ObDirectLoadMultipleDatumRow *rhs) + { + return lhs->seq_no_ < rhs->seq_no_; + } + +public: + ObTableLoadDataRowHandler(); + virtual ~ObTableLoadDataRowHandler(); + int init(const ObTableLoadParam ¶m, table::ObTableLoadResultInfo &result_info, + ObTableLoadErrorRowHandler *error_row_handler, + hash::ObHashMap *index_store_table_ctx_map); + int handle_insert_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) override; + int handle_delete_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) + { + return OB_ERR_UNEXPECTED; + } + int handle_insert_row_with_multi_version(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) override; + int handle_update_row(const blocksstable::ObDatumRow &row); + int handle_update_row(common::ObArray &rows, + const ObDirectLoadExternalRow *&row); + int handle_update_row(common::ObArray &rows, + const ObDirectLoadMultipleDatumRow *&row) override; + int handle_update_row(const ObTabletID tablet_id, + const blocksstable::ObDatumRow &old_row, + const blocksstable::ObDatumRow &new_row, + const blocksstable::ObDatumRow *&result_row); + TO_STRING_KV(KPC_(error_row_handler), K_(result_info), K_(dup_action), K_(is_inited)); +private: + ObTableLoadErrorRowHandler * error_row_handler_; + hash::ObHashMap *index_store_table_ctx_map_; + table::ObTableLoadResultInfo *result_info_; + sql::ObLoadDupActionType dup_action_; + bool is_inited_; +}; + +} +} \ No newline at end of file diff --git a/src/observer/table_load/ob_table_load_error_row_handler.cpp b/src/observer/table_load/ob_table_load_error_row_handler.cpp index 0fa754409..81c734dc5 100644 --- a/src/observer/table_load/ob_table_load_error_row_handler.cpp +++ b/src/observer/table_load/ob_table_load_error_row_handler.cpp @@ -57,190 +57,12 @@ int ObTableLoadErrorRowHandler::init(const ObTableLoadParam ¶m, return ret; } -int ObTableLoadErrorRowHandler::handle_insert_row(const blocksstable::ObDatumRow &row) -{ - UNUSED(row); - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } else { - ATOMIC_INC(&result_info_->rows_affected_); - } - return ret; -} -int ObTableLoadErrorRowHandler::handle_update_row(const ObDatumRow &row) -{ - UNUSED(row); - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } else { - if (ObLoadDupActionType::LOAD_STOP_ON_DUP == dup_action_) { - if (0 == max_error_row_count_) { - ret = OB_ERR_PRIMARY_KEY_DUPLICATE; - } else { - ObMutexGuard guard(mutex_); - if (error_row_count_ >= max_error_row_count_) { - ret = OB_ERR_TOO_MANY_ROWS; - LOG_WARN("error row count reaches its maximum value", KR(ret), K_(max_error_row_count), - K_(error_row_count)); - } else { - ++error_row_count_; - } - } - ATOMIC_INC(&job_stat_->detected_error_rows_); - } else if (ObLoadDupActionType::LOAD_REPLACE == dup_action_) { - ATOMIC_AAF(&result_info_->rows_affected_, 2); - ATOMIC_INC(&result_info_->deleted_); - } else if (ObLoadDupActionType::LOAD_IGNORE == dup_action_) { - ATOMIC_INC(&result_info_->skipped_); - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected dup action", KR(ret), K_(dup_action)); - } - } - return ret; -} -int ObTableLoadErrorRowHandler::handle_update_row( - common::ObArray &rows, const ObDirectLoadExternalRow *&row) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } else if (OB_UNLIKELY(rows.count() < 2)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret)); - } else { - int64_t duplicate_row_count = rows.count() - 1; - lib::ob_sort(rows.begin(), rows.end(), - [](const ObDirectLoadExternalRow *lhs, const ObDirectLoadExternalRow *rhs) { - return lhs->seq_no_ < rhs->seq_no_; - }); - if (ObLoadDupActionType::LOAD_STOP_ON_DUP == dup_action_) { - if (0 == max_error_row_count_) { - ret = OB_ERR_PRIMARY_KEY_DUPLICATE; - } else { - ObMutexGuard guard(mutex_); - error_row_count_ += duplicate_row_count; - if (error_row_count_ >= max_error_row_count_) { - ret = OB_ERR_TOO_MANY_ROWS; - LOG_WARN("error row count reaches its maximum value", KR(ret), K_(max_error_row_count), - K_(error_row_count)); - } - } - ATOMIC_AAF(&job_stat_->detected_error_rows_, duplicate_row_count); - row = rows.at(0); - } else if (ObLoadDupActionType::LOAD_REPLACE == dup_action_) { - ATOMIC_AAF(&result_info_->rows_affected_, 2 * duplicate_row_count); - ATOMIC_AAF(&result_info_->deleted_, duplicate_row_count); - row = rows.at(duplicate_row_count); - } else if (ObLoadDupActionType::LOAD_IGNORE == dup_action_) { - ATOMIC_AAF(&result_info_->skipped_, duplicate_row_count); - row = rows.at(0); - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected dup action", KR(ret), K_(dup_action)); - } - } - return ret; -} -int ObTableLoadErrorRowHandler::handle_update_row( - common::ObArray &rows, - const ObDirectLoadMultipleDatumRow *&row) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } else if (OB_UNLIKELY(rows.count() < 2)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret)); - } else { - int64_t duplicate_row_count = rows.count() - 1; - lib::ob_sort(rows.begin(), rows.end(), - [](const ObDirectLoadMultipleDatumRow *lhs, const ObDirectLoadMultipleDatumRow *rhs) { - return lhs->seq_no_ < rhs->seq_no_; - }); - if (ObLoadDupActionType::LOAD_STOP_ON_DUP == dup_action_) { - if (0 == max_error_row_count_) { - ret = OB_ERR_PRIMARY_KEY_DUPLICATE; - } else { - error_row_count_ += duplicate_row_count; - ObMutexGuard guard(mutex_); - if (error_row_count_ >= max_error_row_count_) { - ret = OB_ERR_TOO_MANY_ROWS; - LOG_WARN("error row count reaches its maximum value", KR(ret), K_(max_error_row_count), - K_(error_row_count)); - } - } - ATOMIC_AAF(&job_stat_->detected_error_rows_, duplicate_row_count); - row = rows.at(0); - } else if (ObLoadDupActionType::LOAD_REPLACE == dup_action_) { - ATOMIC_AAF(&result_info_->rows_affected_, 2 * duplicate_row_count); - ATOMIC_AAF(&result_info_->deleted_, duplicate_row_count); - row = rows.at(duplicate_row_count); - } else if (ObLoadDupActionType::LOAD_IGNORE == dup_action_) { - ATOMIC_AAF(&result_info_->skipped_, duplicate_row_count); - row = rows.at(0); - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected dup action", KR(ret), K_(dup_action)); - } - } - return ret; -} -int ObTableLoadErrorRowHandler::handle_update_row(const ObDatumRow &old_row, - const ObDatumRow &new_row, - const ObDatumRow *&result_row) +int ObTableLoadErrorRowHandler::handle_error_row(int error_code) { - int ret = OB_SUCCESS; - result_row = nullptr; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } else { - if (ObLoadDupActionType::LOAD_STOP_ON_DUP == dup_action_) { - if (0 == max_error_row_count_) { - ret = OB_ERR_PRIMARY_KEY_DUPLICATE; - } else { - ObMutexGuard guard(mutex_); - if (error_row_count_ >= max_error_row_count_) { - ret = OB_ERR_TOO_MANY_ROWS; - LOG_WARN("error row count reaches its maximum value", KR(ret), K_(max_error_row_count), - K_(error_row_count)); - } else { - ++error_row_count_; - } - } - if (OB_SUCC(ret)) { - result_row = &old_row; - } - ATOMIC_INC(&job_stat_->detected_error_rows_); - } else if (ObLoadDupActionType::LOAD_IGNORE == dup_action_) { - result_row = &old_row; - ATOMIC_INC(&result_info_->skipped_); - } else if (ObLoadDupActionType::LOAD_REPLACE == dup_action_) { - result_row = &new_row; - ATOMIC_INC(&result_info_->deleted_); - ATOMIC_AAF(&result_info_->rows_affected_, 2); - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected dup action", KR(ret), K_(dup_action)); - } - } - return ret; -} - -int ObTableLoadErrorRowHandler::handle_error_row(int error_code, const ObNewRow &row) -{ - UNUSED(row); int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -261,9 +83,8 @@ int ObTableLoadErrorRowHandler::handle_error_row(int error_code, const ObNewRow return ret; } -int ObTableLoadErrorRowHandler::handle_error_row(int error_code, const ObDatumRow &row) +int ObTableLoadErrorRowHandler::handle_error_row(int error_code, int64_t duplicate_row_count) { - UNUSED(row); int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -271,15 +92,18 @@ int ObTableLoadErrorRowHandler::handle_error_row(int error_code, const ObDatumRo } else if (max_error_row_count_ == 0) { ret = error_code; } else { - ObMutexGuard guard(mutex_); - if (error_row_count_ >= max_error_row_count_) { - ret = OB_ERR_TOO_MANY_ROWS; - LOG_WARN("error row count reaches its maximum value", KR(ret), K_(max_error_row_count), - K_(error_row_count)); + if (0 == max_error_row_count_) { + ret = OB_ERR_PRIMARY_KEY_DUPLICATE; } else { - ++error_row_count_; + ObMutexGuard guard(mutex_); + error_row_count_ += duplicate_row_count; + if (error_row_count_ > max_error_row_count_) { + ret = OB_ERR_TOO_MANY_ROWS; + LOG_WARN("error row count reaches its maximum value", KR(ret), K_(max_error_row_count), + K_(error_row_count)); + } } - ATOMIC_INC(&job_stat_->detected_error_rows_); + ATOMIC_AAF(&job_stat_->detected_error_rows_, duplicate_row_count); } return ret; } diff --git a/src/observer/table_load/ob_table_load_error_row_handler.h b/src/observer/table_load/ob_table_load_error_row_handler.h index 8af67d098..88fcde2a2 100644 --- a/src/observer/table_load/ob_table_load_error_row_handler.h +++ b/src/observer/table_load/ob_table_load_error_row_handler.h @@ -15,7 +15,6 @@ #include "common/row/ob_row.h" #include "sql/engine/cmd/ob_load_data_utils.h" #include "storage/blocksstable/ob_datum_row.h" -#include "storage/direct_load/ob_direct_load_dml_row_handler.h" namespace oceanbase { @@ -29,24 +28,15 @@ class ObTableLoadStoreCtx; class ObTableLoadCoordinatorCtx; class ObTableLoadParam; -class ObTableLoadErrorRowHandler : public ObDirectLoadDMLRowHandler +class ObTableLoadErrorRowHandler { public: ObTableLoadErrorRowHandler(); virtual ~ObTableLoadErrorRowHandler(); int init(const ObTableLoadParam ¶m, table::ObTableLoadResultInfo &result_info, sql::ObLoadDataStat *job_stat); - int handle_insert_row(const blocksstable::ObDatumRow &row) override; - int handle_update_row(const blocksstable::ObDatumRow &row) override; - int handle_update_row(common::ObArray &rows, - const ObDirectLoadExternalRow *&row) override; - int handle_update_row(common::ObArray &rows, - const ObDirectLoadMultipleDatumRow *&row) override; - int handle_update_row(const blocksstable::ObDatumRow &old_row, - const blocksstable::ObDatumRow &new_row, - const blocksstable::ObDatumRow *&result_row) override; - int handle_error_row(int error_code, const common::ObNewRow &row); - int handle_error_row(int error_code, const blocksstable::ObDatumRow &row); + int handle_error_row(int error_code); + int handle_error_row(int error_code, int64_t duplicate_row_count); uint64_t get_error_row_count() const; TO_STRING_KV(K_(dup_action), K_(max_error_row_count), K_(error_row_count)); private: diff --git a/src/observer/table_load/ob_table_load_general_table_compactor.cpp b/src/observer/table_load/ob_table_load_general_table_compactor.cpp index 7e0dff4a3..8b4d631a9 100644 --- a/src/observer/table_load/ob_table_load_general_table_compactor.cpp +++ b/src/observer/table_load/ob_table_load_general_table_compactor.cpp @@ -22,6 +22,7 @@ #include "observer/table_load/ob_table_load_trans_store.h" #include "storage/direct_load/ob_direct_load_external_table_compactor.h" #include "storage/direct_load/ob_direct_load_sstable_compactor.h" +#include "observer/table_load/ob_table_load_store_table_ctx.h" namespace oceanbase { @@ -206,6 +207,7 @@ int ObTableLoadGeneralTableCompactor::CompactorTaskIter::get_next_compactor_task ObTableLoadGeneralTableCompactor::ObTableLoadGeneralTableCompactor() : store_ctx_(nullptr), + store_table_ctx_(nullptr), param_(nullptr), allocator_("TLD_GeneralTC"), running_thread_count_(0), @@ -225,6 +227,7 @@ void ObTableLoadGeneralTableCompactor::reset() { abort_unless(compacting_list_.is_empty()); store_ctx_ = nullptr; + store_table_ctx_ = nullptr; param_ = nullptr; for (int64_t i = 0; i < all_compactor_array_.count(); ++i) { ObIDirectLoadTabletTableCompactor *compactor = all_compactor_array_.at(i); @@ -242,6 +245,7 @@ int ObTableLoadGeneralTableCompactor::inner_init() { int ret = OB_SUCCESS; store_ctx_ = compact_ctx_->store_ctx_; + store_table_ctx_ = compact_ctx_->store_table_ctx_; param_ = &store_ctx_->ctx_->param_; return ret; } @@ -362,10 +366,10 @@ int ObTableLoadGeneralTableCompactor::create_tablet_table_compactor( { int ret = OB_SUCCESS; table_compactor = nullptr; - if (store_ctx_->ctx_->schema_.is_heap_table_) { + if (store_table_ctx_->schema_->is_heap_table_) { ObDirectLoadExternalTableCompactParam compact_param; compact_param.tablet_id_ = tablet_id; - compact_param.table_data_desc_ = store_ctx_->table_data_desc_; + compact_param.table_data_desc_ = store_table_ctx_->table_data_desc_; ObDirectLoadExternalTableCompactor *external_table_compactor = nullptr; if (OB_ISNULL(external_table_compactor = OB_NEWx(ObDirectLoadExternalTableCompactor, (&allocator_)))) { @@ -375,11 +379,11 @@ int ObTableLoadGeneralTableCompactor::create_tablet_table_compactor( LOG_WARN("fail to init external table compactor", KR(ret)); } table_compactor = external_table_compactor; - } else if (!param_->need_sort_) { + } else if (!store_table_ctx_->need_sort_) { ObDirectLoadSSTableCompactParam compact_param; compact_param.tablet_id_ = tablet_id; - compact_param.table_data_desc_ = store_ctx_->table_data_desc_; - compact_param.datum_utils_ = &(store_ctx_->ctx_->schema_.datum_utils_); + compact_param.table_data_desc_ = store_table_ctx_->table_data_desc_; + compact_param.datum_utils_ = &(store_table_ctx_->schema_->datum_utils_); ObDirectLoadSSTableCompactor *sstable_compactor = nullptr; if (OB_ISNULL(sstable_compactor = OB_NEWx(ObDirectLoadSSTableCompactor, (&allocator_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; diff --git a/src/observer/table_load/ob_table_load_general_table_compactor.h b/src/observer/table_load/ob_table_load_general_table_compactor.h index f429bd2d8..215ed40ee 100644 --- a/src/observer/table_load/ob_table_load_general_table_compactor.h +++ b/src/observer/table_load/ob_table_load_general_table_compactor.h @@ -75,6 +75,7 @@ private: int build_result(); private: ObTableLoadStoreCtx *store_ctx_; + ObTableLoadStoreTableCtx * store_table_ctx_; const ObTableLoadParam *param_; common::ObArenaAllocator allocator_; common::ObArray all_compactor_array_; diff --git a/src/observer/table_load/ob_table_load_index_row_handler.cpp b/src/observer/table_load/ob_table_load_index_row_handler.cpp new file mode 100644 index 000000000..014a76237 --- /dev/null +++ b/src/observer/table_load/ob_table_load_index_row_handler.cpp @@ -0,0 +1,42 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#define USING_LOG_PREFIX SERVER + +#include "observer/table_load/ob_table_load_index_row_handler.h" + +namespace oceanbase +{ +namespace observer +{ +using namespace blocksstable; + +ObTableLoadIndexRowHandler::ObTableLoadIndexRowHandler() {} + +ObTableLoadIndexRowHandler::~ObTableLoadIndexRowHandler() {} + +int ObTableLoadIndexRowHandler::handle_insert_row(const ObTabletID tablet_id, + const blocksstable::ObDatumRow &row) +{ + int ret = OB_SUCCESS; + //do nothing + return ret; +} + +int ObTableLoadIndexRowHandler::handle_delete_row(const ObTabletID tablet_id, + const blocksstable::ObDatumRow &row) +{ + int ret = OB_SUCCESS; + // do nothing + return ret; +} +} // namespace observer +} // namespace oceanbase \ No newline at end of file diff --git a/src/observer/table_load/ob_table_load_index_row_handler.h b/src/observer/table_load/ob_table_load_index_row_handler.h new file mode 100644 index 000000000..cb2962e8f --- /dev/null +++ b/src/observer/table_load/ob_table_load_index_row_handler.h @@ -0,0 +1,57 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#pragma once + +#include "storage/blocksstable/ob_datum_row.h" +#include "storage/direct_load/ob_direct_load_dml_row_handler.h" + +namespace oceanbase +{ +namespace observer +{ +class ObTableLoadStoreTableCtx; + +class ObTableLoadIndexRowHandler : public ObDirectLoadDMLRowHandler +{ +public: + ObTableLoadIndexRowHandler(); + virtual ~ObTableLoadIndexRowHandler(); + int handle_insert_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row); + int handle_delete_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row); + int handle_insert_row_with_multi_version(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) + { + return OB_ERR_UNEXPECTED; + } + int handle_update_row(const blocksstable::ObDatumRow &row) override { return OB_ERR_UNEXPECTED; }; + int handle_update_row(common::ObArray &rows, + const ObDirectLoadExternalRow *&row) override + { + return OB_ERR_UNEXPECTED; + }; + int handle_update_row(common::ObArray &rows, + const ObDirectLoadMultipleDatumRow *&row) override + { + return OB_ERR_UNEXPECTED; + }; + int handle_update_row(const ObTabletID tablet_id, + const blocksstable::ObDatumRow &old_row, + const blocksstable::ObDatumRow &new_row, + const blocksstable::ObDatumRow *&result_row) override + { + return OB_ERR_UNEXPECTED; + }; + TO_STRING_EMPTY(); +}; + +} // namespace observer +} // namespace oceanbase \ No newline at end of file diff --git a/src/observer/table_load/ob_table_load_index_table_projector.cpp b/src/observer/table_load/ob_table_load_index_table_projector.cpp new file mode 100644 index 000000000..303387024 --- /dev/null +++ b/src/observer/table_load/ob_table_load_index_table_projector.cpp @@ -0,0 +1,196 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#define USING_LOG_PREFIX STORAGE +#include "observer/table_load/ob_table_load_index_table_projector.h" +#include "lib/oblog/ob_log_module.h" +#include "share/rc/ob_tenant_base.h" +#include "storage/ob_i_store.h" + +namespace oceanbase +{ +namespace observer +{ +ObTableLoadIndexTableProjector::~ObTableLoadIndexTableProjector() +{ + row_projector_.reset(); + tablet_projector_.destroy(); + index_tablet_id_to_part_id_map_.destroy(); +} + +int ObTableLoadIndexTableProjector::init(const share::schema::ObTableSchema *data_table_schema, + const share::schema::ObTableSchema *index_table_schema) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObTableLoadIndexTableProjector init twice", KR(ret)); + } else if (OB_FAIL(tablet_projector_.create(1024, "TLD_ITP", "TLD_ITP", MTL_ID()))) { + LOG_WARN("fail to create tablet projector", KR(ret)); + } else if (OB_FAIL( + index_tablet_id_to_part_id_map_.create(1024, "TLD_ITP", "TLD_ITP", MTL_ID()))) { + LOG_WARN("fail to create index tablet id to part id map", KR(ret)); + } else if (OB_FAIL(build_projector(data_table_schema, index_table_schema))) { + LOG_WARN("fail to build projector", KR(ret), KPC(data_table_schema), KPC(index_table_schema)); + } else { + is_inited_ = true; + } + return ret; +} + +int ObTableLoadIndexTableProjector::get_index_tablet_id_and_part_id_by_data_tablet_id( + const ObTabletID &data_tablet_id, ObTabletID &index_tablet_id, ObObjectID &part_id) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTableLoadIndexTableProjector not init", KR(ret)); + } else if (OB_FAIL(tablet_projector_.get_refactored(data_tablet_id, index_tablet_id))) { + LOG_WARN("fail to get index tablet id", KR(ret), K(data_tablet_id)); + } else if (OB_FAIL(index_tablet_id_to_part_id_map_.get_refactored(index_tablet_id, part_id))) { + LOG_WARN("fail to get index tablet id", KR(ret), K(index_tablet_id)); + } + return ret; +} + +int ObTableLoadIndexTableProjector::build_projector( + const share::schema::ObTableSchema *data_table_schema, + const share::schema::ObTableSchema *index_table_schema) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(data_table_schema) || OB_ISNULL(index_table_schema)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), KP(data_table_schema), KP(index_table_schema)); + } else if (OB_FAIL(build_row_projector(data_table_schema, index_table_schema))) { + LOG_WARN("fail to build row projector", KR(ret)); + } else if (OB_FAIL(build_tablet_projector(data_table_schema, index_table_schema))) { + LOG_WARN("fail to build tablet projector", KR(ret)); + } else { + column_num_ = index_table_schema->get_column_count(); + main_table_rowkey_column_num_ = data_table_schema->get_rowkey_column_num(); + } + return ret; +} + +int ObTableLoadIndexTableProjector::build_row_projector( + const share::schema::ObTableSchema *data_table_schema, + const share::schema::ObTableSchema *index_table_schema) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(data_table_schema) || OB_ISNULL(index_table_schema)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), KP(data_table_schema), KP(index_table_schema)); + } else { + common::ObArray main_column_descs; + common::ObArray index_column_descs; + if (OB_FAIL(data_table_schema->get_column_ids(main_column_descs, false))) { + LOG_WARN("fail to get column ids", KR(ret)); + } else if (OB_FAIL(index_table_schema->get_column_ids(index_column_descs, false))) { + LOG_WARN("fail to get column ids", KR(ret)); + } else { + FOREACH_X(iter, index_column_descs, OB_SUCC(ret)) { + share::schema::ObColDesc index_col_desc = *iter; + for (int64_t i = 0; OB_SUCC(ret) && i < main_column_descs.count(); i++) { + if (index_col_desc.col_id_ == main_column_descs.at(i).col_id_) { + if (OB_FAIL(row_projector_.push_back(i))) { + LOG_WARN("fail to push back", KR(ret), K(i)); + } + } + } + } + } + } + return ret; +} + +int ObTableLoadIndexTableProjector::build_tablet_projector( + const share::schema::ObTableSchema *data_table_schema, + const share::schema::ObTableSchema *index_table_schema) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(data_table_schema) || OB_ISNULL(index_table_schema)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), KP(data_table_schema), KP(index_table_schema)); + } else { + ObArray tablet_ids; + if (OB_FAIL(data_table_schema->get_tablet_ids(tablet_ids))) { + LOG_WARN("fail to get tablet ids", KR(ret)); + } else { + int64_t main_part_idx = OB_INVALID_ID; + int64_t main_subpart_idx = OB_INVALID_ID; + ObObjectID index_part_id = OB_INVALID_ID; + ObObjectID index_subpart_id = OB_INVALID_ID; + ObTabletID index_tablet_id(ObTabletID::INVALID_TABLET_ID); + for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); i++) { + if (data_table_schema->is_partitioned_table() && + OB_FAIL(data_table_schema->get_part_idx_by_tablet(tablet_ids.at(i), main_part_idx, + main_subpart_idx))) { + LOG_WARN("fail to get part idx by tablet", KR(ret), K(tablet_ids.at(i))); + } else if (OB_FAIL(index_table_schema->get_part_id_and_tablet_id_by_idx( + main_part_idx, main_subpart_idx, index_part_id, index_subpart_id, + index_tablet_id))) { + LOG_WARN("fail to get index tablet id", KR(ret), K(main_part_idx), K(main_subpart_idx)); + } else if (OB_FAIL(tablet_projector_.set_refactored(tablet_ids.at(i), index_tablet_id))) { + LOG_WARN("fail to add tablet projector", KR(ret), K(index_tablet_id), + K(tablet_ids.at(i))); + } else if (OB_FAIL(index_tablet_id_to_part_id_map_.set_refactored(index_tablet_id, + index_part_id))) { + LOG_WARN("fail to add index tablet id to part id map", KR(ret), K(index_tablet_id), + K(index_part_id)); + } + } + } + } + return ret; +} + +int ObTableLoadIndexTableProjector::projector(const ObTabletID &data_tablet_id, + const blocksstable::ObDatumRow &origin_datum_row, + const bool &have_multiversion_col, + ObTabletID &index_tablet_id, + blocksstable::ObDatumRow &out_datum_row) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTableLoadIndexTableProjector not init", KR(ret)); + } else if (OB_FAIL(out_datum_row.init(column_num_))) { + LOG_WARN("fail to init index datum row", KR(ret)); + } else if (OB_FAIL(tablet_projector_.get_refactored(data_tablet_id, index_tablet_id))) { + LOG_WARN("fail to get index id", KR(ret), K(data_tablet_id)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < row_projector_.size(); i++) { + if (have_multiversion_col && row_projector_.at(i) >= main_table_rowkey_column_num_) { + if (row_projector_.at(i) + storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt() >= + origin_datum_row.get_column_count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get datum", KR(ret), K(row_projector_.at(i)), K(origin_datum_row)); + } else { + out_datum_row.storage_datums_[i] = + origin_datum_row + .storage_datums_[row_projector_.at(i) + + storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt()]; + } + } else { + if (row_projector_.at(i) >= origin_datum_row.get_column_count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get datum", KR(ret), K(row_projector_.at(i)), K(origin_datum_row)); + } else { + out_datum_row.storage_datums_[i] = origin_datum_row.storage_datums_[row_projector_.at(i)]; + } + } + } + } + return ret; +} + +} // namespace observer +} // namespace oceanbase \ No newline at end of file diff --git a/src/observer/table_load/ob_table_load_index_table_projector.h b/src/observer/table_load/ob_table_load_index_table_projector.h new file mode 100644 index 000000000..3d26c2e2b --- /dev/null +++ b/src/observer/table_load/ob_table_load_index_table_projector.h @@ -0,0 +1,52 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + + #pragma once +#include "lib/ob_define.h" +#include "lib/container/ob_array.h" +#include "lib/hash/ob_hashmap.h" +#include "ob_tablet_id.h" +#include "share/schema/ob_table_schema.h" +#include "storage/blocksstable/ob_datum_row.h" + +namespace oceanbase +{ +namespace observer +{ +class ObTableLoadIndexTableProjector +{ +public: + ObTableLoadIndexTableProjector() + : column_num_(0), main_table_rowkey_column_num_(0), is_inited_(false) + { + } + ~ObTableLoadIndexTableProjector(); + int init(const share::schema::ObTableSchema *data_table_schema, const share::schema::ObTableSchema *index_table_schema); + int projector(const ObTabletID &data_tablet_id, const blocksstable::ObDatumRow &origin_datum_row, + const bool &have_multiversion_col, ObTabletID &index_tablet_id, + blocksstable::ObDatumRow &out_datum_row); + int get_index_tablet_id_and_part_id_by_data_tablet_id(const ObTabletID &data_tablet_id, ObTabletID &index_tablet_id, ObObjectID &part_id); +private: + int build_projector(const share::schema::ObTableSchema *data_table_schema, const share::schema::ObTableSchema *index_table_schema); + int build_tablet_projector(const share::schema::ObTableSchema *data_table_schema, const share::schema::ObTableSchema *index_table_schema); + int build_row_projector(const share::schema::ObTableSchema *data_table_schema, const share::schema::ObTableSchema *index_table_schema); +private: + common::ObArray row_projector_; + common::hash::ObHashMap tablet_projector_;//main tablet id -> index tablet id + common::hash::ObHashMap index_tablet_id_to_part_id_map_; + int64_t column_num_; + int64_t main_table_rowkey_column_num_; + bool is_inited_; +}; + +} +} \ No newline at end of file diff --git a/src/observer/table_load/ob_table_load_mem_compactor.cpp b/src/observer/table_load/ob_table_load_mem_compactor.cpp index 6557a70a7..4815b0636 100644 --- a/src/observer/table_load/ob_table_load_mem_compactor.cpp +++ b/src/observer/table_load/ob_table_load_mem_compactor.cpp @@ -21,6 +21,7 @@ #include "observer/table_load/ob_table_load_task.h" #include "observer/table_load/ob_table_load_task_scheduler.h" #include "observer/table_load/ob_table_load_trans_store.h" +#include "observer/table_load/ob_table_load_store_table_ctx.h" #include "storage/direct_load/ob_direct_load_external_table.h" #include "storage/direct_load/ob_direct_load_mem_loader.h" #include "storage/direct_load/ob_direct_load_mem_sample.h" @@ -232,6 +233,7 @@ private: ObTableLoadMemCompactor::ObTableLoadMemCompactor() : store_ctx_(nullptr), + store_table_ctx_(nullptr), param_(nullptr), allocator_("TLD_MemC"), finish_task_count_(0), @@ -249,6 +251,7 @@ ObTableLoadMemCompactor::~ObTableLoadMemCompactor() void ObTableLoadMemCompactor::reset() { store_ctx_ = nullptr; + store_table_ctx_ = nullptr; param_ = nullptr; finish_task_count_ = 0; @@ -268,6 +271,7 @@ int ObTableLoadMemCompactor::inner_init() int ret = OB_SUCCESS; const uint64_t tenant_id = MTL_ID(); store_ctx_ = compact_ctx_->store_ctx_; + store_table_ctx_ = compact_ctx_->store_table_ctx_; param_ = &(store_ctx_->ctx_->param_); if (OB_UNLIKELY(param_->session_count_ < 2)) { // 排序至少需要两个线程 @@ -282,21 +286,21 @@ int ObTableLoadMemCompactor::inner_init() } if (compact_ctx_->compact_config_->is_sort_lobid_) { - mem_ctx_.table_data_desc_ = store_ctx_->lob_id_table_data_desc_; - mem_ctx_.datum_utils_ = &(store_ctx_->ctx_->schema_.lob_meta_datum_utils_); + mem_ctx_.table_data_desc_ = store_table_ctx_->lob_id_table_data_desc_; + mem_ctx_.datum_utils_ = &(store_table_ctx_->schema_->lob_meta_datum_utils_); mem_ctx_.need_sort_ = true; mem_ctx_.column_count_ = 1; } else { - mem_ctx_.table_data_desc_ = store_ctx_->table_data_desc_; - mem_ctx_.datum_utils_ = &(store_ctx_->ctx_->schema_.datum_utils_); - mem_ctx_.need_sort_ = param_->need_sort_; - mem_ctx_.column_count_ = (store_ctx_->ctx_->schema_.is_heap_table_ - ? store_ctx_->ctx_->schema_.store_column_count_ - 1 - : store_ctx_->ctx_->schema_.store_column_count_); + mem_ctx_.table_data_desc_ = store_table_ctx_->table_data_desc_; + mem_ctx_.datum_utils_ = &(store_table_ctx_->schema_->datum_utils_); + mem_ctx_.need_sort_ = store_table_ctx_->need_sort_; + mem_ctx_.column_count_ = (store_table_ctx_->schema_->is_heap_table_ + ? store_table_ctx_->schema_->store_column_count_ - 1 + : store_table_ctx_->schema_->store_column_count_); } mem_ctx_.mem_load_task_count_ = param_->session_count_; - mem_ctx_.dml_row_handler_ = store_ctx_->error_row_handler_; + mem_ctx_.dml_row_handler_ = store_table_ctx_->row_handler_; mem_ctx_.file_mgr_ = store_ctx_->tmp_file_mgr_; mem_ctx_.dup_action_ = param_->dup_action_; } @@ -311,7 +315,7 @@ int ObTableLoadMemCompactor::inner_init() if (OB_SUCC(ret)) { if (OB_FAIL(mem_ctx_.init())) { LOG_WARN("fail to init compactor ctx", KR(ret)); - } else if (OB_FAIL(parallel_merge_ctx_.init(store_ctx_, mem_ctx_.table_data_desc_))) { + } else if (OB_FAIL(parallel_merge_ctx_.init(store_ctx_, store_table_ctx_, mem_ctx_.table_data_desc_))) { LOG_WARN("fail to init parallel merge ctx", KR(ret)); } } @@ -540,7 +544,8 @@ int ObTableLoadMemCompactor::start_compact() { int ret = OB_SUCCESS; if (store_ctx_->enable_pre_sort_ - && !store_ctx_->insert_table_ctx_->need_del_lob()) { + && !store_table_ctx_->insert_table_ctx_->need_del_lob() + && !store_table_ctx_->is_index_table_) { // do nothing } else { if (OB_FAIL(start_load())) { diff --git a/src/observer/table_load/ob_table_load_mem_compactor.h b/src/observer/table_load/ob_table_load_mem_compactor.h index 310988bf5..2c2fdc32c 100644 --- a/src/observer/table_load/ob_table_load_mem_compactor.h +++ b/src/observer/table_load/ob_table_load_mem_compactor.h @@ -86,6 +86,7 @@ private: int64_t get_compact_task_count() const; private: ObTableLoadStoreCtx *store_ctx_; + ObTableLoadStoreTableCtx *store_table_ctx_; const ObTableLoadParam *param_; common::ObArenaAllocator allocator_; //需要最后析构 int64_t finish_task_count_ CACHE_ALIGNED; diff --git a/src/observer/table_load/ob_table_load_merger.cpp b/src/observer/table_load/ob_table_load_merger.cpp index 58a4cd53f..89523eaff 100644 --- a/src/observer/table_load/ob_table_load_merger.cpp +++ b/src/observer/table_load/ob_table_load_merger.cpp @@ -14,6 +14,7 @@ #include "observer/table_load/ob_table_load_merger.h" #include "observer/table_load/ob_table_load_error_row_handler.h" +#include "observer/table_load/ob_table_load_data_row_handler.h" #include "observer/table_load/ob_table_load_service.h" #include "observer/table_load/ob_table_load_stat.h" #include "observer/table_load/ob_table_load_store_ctx.h" @@ -24,6 +25,8 @@ #include "storage/direct_load/ob_direct_load_fast_heap_table.h" #include "storage/direct_load/ob_direct_load_multi_map.h" #include "storage/direct_load/ob_direct_load_range_splitter.h" +#include "observer/table_load/ob_table_load_store_table_ctx.h" +#include "observer/table_load/ob_table_load_merger_manager.h" #include "storage/blocksstable/ob_sstable.h" namespace oceanbase @@ -246,20 +249,29 @@ private: * ObTableLoadMerger */ -ObTableLoadMerger::ObTableLoadMerger(ObTableLoadStoreCtx *store_ctx) - : store_ctx_(store_ctx), +ObTableLoadMerger::ObTableLoadMerger(ObTableLoadStoreCtx *store_ctx, ObTableLoadStoreTableCtx *store_table_ctx) + : store_table_ctx_(store_table_ctx), + allocator_("TLD_TLM"), + store_ctx_(store_ctx), param_(store_ctx->ctx_->param_), + table_compact_config_(nullptr), running_thread_count_(0), has_error_(false), is_stop_(false), is_inited_(false) { + allocator_.set_tenant_id(MTL_ID()); } ObTableLoadMerger::~ObTableLoadMerger() { abort_unless(merging_list_.is_empty()); abort_unless(rescan_list_.is_empty()); + if (OB_NOT_NULL(table_compact_config_)) { + table_compact_config_->~ObTableLoadTableCompactConfig(); + allocator_.free(table_compact_config_); + table_compact_config_ = nullptr; + } } int ObTableLoadMerger::init() @@ -269,12 +281,39 @@ int ObTableLoadMerger::init() ret = OB_INIT_TWICE; LOG_WARN("ObTableLoadMerger init twice", KR(ret), KP(this)); } else { - if (OB_FAIL(table_compact_config_.init(store_ctx_, *this))) { - LOG_WARN("fail to init table_compact_config_", KR(ret)); + if (store_table_ctx_->is_index_table_) { + ObTableLoadTableCompactConfigIndexTable *index_table_compact_config = nullptr; + if (OB_ISNULL(index_table_compact_config = OB_NEWx(ObTableLoadTableCompactConfigIndexTable, &allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate memory for table compact config", KR(ret)); + } else if (OB_FAIL((index_table_compact_config->init(*this)))) { + LOG_WARN("fail to init table compact config", KR(ret)); + } else { + table_compact_config_ = index_table_compact_config; + } + } else { + ObTableLoadTableCompactConfigMainTable *main_table_compact_config = nullptr; + if (OB_ISNULL(main_table_compact_config = OB_NEWx(ObTableLoadTableCompactConfigMainTable, &allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate memory for table compact config", KR(ret)); + } else if (OB_FAIL((main_table_compact_config->init(store_ctx_, *this)))) { + LOG_WARN("fail to init table compact config", KR(ret)); + } else { + table_compact_config_ = main_table_compact_config; + } + } + if (OB_FAIL(ret)) { } else if (OB_FAIL(lob_id_compact_config_.init(*this))) { LOG_WARN("fail to init lob id compact config", KR(ret)); - } else if (OB_FAIL(table_compact_ctx_.init(store_ctx_, &table_compact_config_))) { + } else if (OB_FAIL(table_compact_ctx_.init(store_ctx_, store_table_ctx_, table_compact_config_))) { LOG_WARN("fail to init table compact ctx", KR(ret)); + } + if (OB_FAIL(ret)) { + if (OB_NOT_NULL(table_compact_config_)) { + table_compact_config_->~ObTableLoadTableCompactConfig(); + allocator_.free(table_compact_config_); + table_compact_config_ = nullptr; + } } else { is_inited_ = true; } @@ -289,7 +328,7 @@ int ObTableLoadMerger::start() ret = OB_NOT_INIT; LOG_WARN("ObTableLoadMerger not init", KR(ret), KP(this)); } else { - if (store_ctx_->is_fast_heap_table_) { + if (store_table_ctx_->is_fast_heap_table_) { if (OB_FAIL(build_merge_ctx())) { LOG_WARN("fail to build merge ctx", KR(ret)); } else if (OB_FAIL(start_merge())) { @@ -361,31 +400,31 @@ int ObTableLoadMerger::build_merge_ctx() { int ret = OB_SUCCESS; ObDirectLoadMergeParam merge_param; - merge_param.table_id_ = param_.table_id_; - merge_param.lob_meta_table_id_ = store_ctx_->ctx_->schema_.lob_meta_table_id_; - merge_param.target_table_id_ = store_ctx_->ctx_->ddl_param_.dest_table_id_; - merge_param.rowkey_column_num_ = store_ctx_->ctx_->schema_.rowkey_column_count_; - merge_param.store_column_count_ = store_ctx_->ctx_->schema_.store_column_count_; + merge_param.table_id_ = store_table_ctx_->table_id_; + merge_param.lob_meta_table_id_ = store_table_ctx_->schema_->lob_meta_table_id_; + merge_param.target_table_id_ = store_table_ctx_->table_id_; + merge_param.rowkey_column_num_ = store_table_ctx_->schema_->rowkey_column_count_; + merge_param.store_column_count_ = store_table_ctx_->schema_->store_column_count_; merge_param.fill_cg_thread_cnt_ = param_.session_count_; - merge_param.lob_column_idxs_ = &(store_ctx_->ctx_->schema_.lob_column_idxs_); - merge_param.table_data_desc_ = store_ctx_->table_data_desc_; - merge_param.datum_utils_ = &(store_ctx_->ctx_->schema_.datum_utils_); - merge_param.lob_column_idxs_ = &(store_ctx_->ctx_->schema_.lob_column_idxs_); - merge_param.col_descs_ = &(store_ctx_->ctx_->schema_.column_descs_); - merge_param.lob_id_table_data_desc_ = store_ctx_->lob_id_table_data_desc_; - merge_param.lob_meta_datum_utils_ = &(store_ctx_->ctx_->schema_.lob_meta_datum_utils_); - merge_param.lob_meta_col_descs_ = &(store_ctx_->ctx_->schema_.lob_meta_column_descs_); - merge_param.is_heap_table_ = store_ctx_->ctx_->schema_.is_heap_table_; - merge_param.is_fast_heap_table_ = store_ctx_->is_fast_heap_table_; + merge_param.lob_column_idxs_ = &(store_table_ctx_->schema_->lob_column_idxs_); + merge_param.table_data_desc_ = store_table_ctx_->table_data_desc_; + merge_param.datum_utils_ = &(store_table_ctx_->schema_->datum_utils_); + merge_param.col_descs_ = &(store_table_ctx_->schema_->column_descs_); + merge_param.lob_id_table_data_desc_ = store_table_ctx_->lob_id_table_data_desc_; + merge_param.lob_meta_datum_utils_ = &(store_table_ctx_->schema_->lob_meta_datum_utils_); + merge_param.lob_meta_col_descs_ = &(store_table_ctx_->schema_->lob_meta_column_descs_); + merge_param.is_heap_table_ = store_table_ctx_->schema_->is_heap_table_; + merge_param.is_fast_heap_table_ = store_table_ctx_->is_fast_heap_table_; merge_param.is_incremental_ = ObDirectLoadMethod::is_incremental(param_.method_); merge_param.insert_mode_ = param_.insert_mode_; - merge_param.insert_table_ctx_ = store_ctx_->insert_table_ctx_; - merge_param.dml_row_handler_ = store_ctx_->error_row_handler_; + merge_param.insert_table_ctx_ = store_table_ctx_->insert_table_ctx_; + merge_param.dml_row_handler_ = store_table_ctx_->row_handler_; merge_param.file_mgr_ = store_ctx_->tmp_file_mgr_; merge_param.trans_param_ = store_ctx_->trans_param_; - if (OB_FAIL(merge_ctx_.init(store_ctx_->ctx_, merge_param, store_ctx_->ls_partition_ids_))) { + merge_param.index_table_count_ = store_table_ctx_->schema_->index_table_count_; + if (OB_FAIL(merge_ctx_.init(store_ctx_->ctx_, merge_param, store_table_ctx_->ls_partition_ids_))) { LOG_WARN("fail to init merge ctx", KR(ret)); - } else if (store_ctx_->is_multiple_mode_) { + } else if (store_table_ctx_->is_multiple_mode_) { const ObIArray &tablet_merge_ctxs = merge_ctx_.get_tablet_merge_ctxs(); ObArray empty_table_array; @@ -407,7 +446,7 @@ int ObTableLoadMerger::build_merge_ctx() } else { table_array = &empty_table_array; } - if (!merge_param.is_heap_table_ && !table_array->empty()) { + if (!store_table_ctx_->schema_->is_heap_table_ && !table_array->empty()) { // for optimize split range is too slow ObArray multiple_sstable_array; ObDirectLoadMultipleMergeRangeSplitter range_splitter; @@ -422,8 +461,8 @@ int ObTableLoadMerger::build_merge_ctx() } } if (OB_SUCC(ret)) { - if (OB_FAIL(range_splitter.init(multiple_sstable_array, merge_param.table_data_desc_, - merge_param.datum_utils_, *merge_param.col_descs_))) { + if (OB_FAIL(range_splitter.init(multiple_sstable_array, store_table_ctx_->table_data_desc_, + &(store_table_ctx_->schema_->datum_utils_), (store_table_ctx_->schema_->column_descs_)))) { LOG_WARN("fail to init range splitter", KR(ret)); } } @@ -434,7 +473,7 @@ int ObTableLoadMerger::build_merge_ctx() LOG_WARN("fail to build merge task for multiple pk table", KR(ret)); } } - } else if (merge_param.is_heap_table_ && !table_array->empty() && + } else if (store_table_ctx_->schema_->is_heap_table_ && !table_array->empty() && tablet_merge_ctxs.count() > param_.session_count_ * 2) { // for optimize the super multi-partition heap table space serious enlargement for (int64_t i = 0; OB_SUCC(ret) && i < tablet_merge_ctxs.count(); ++i) { @@ -448,8 +487,8 @@ int ObTableLoadMerger::build_merge_ctx() for (int64_t i = 0; OB_SUCC(ret) && i < tablet_merge_ctxs.count(); ++i) { ObDirectLoadTabletMergeCtx *tablet_merge_ctx = tablet_merge_ctxs.at(i); if (OB_FAIL(tablet_merge_ctx->build_merge_task( - *table_array, store_ctx_->ctx_->schema_.column_descs_, param_.session_count_, - store_ctx_->is_multiple_mode_))) { + *table_array, store_table_ctx_->schema_->column_descs_, param_.session_count_, + store_table_ctx_->is_multiple_mode_))) { LOG_WARN("fail to build merge task", KR(ret)); } } @@ -475,8 +514,8 @@ int ObTableLoadMerger::build_merge_ctx() } if (OB_SUCC(ret)) { if (OB_FAIL(tablet_merge_ctx->build_merge_task( - *table_array, store_ctx_->ctx_->schema_.column_descs_, param_.session_count_, - store_ctx_->is_multiple_mode_))) { + *table_array, store_table_ctx_->schema_->column_descs_, param_.session_count_, + store_table_ctx_->is_multiple_mode_))) { LOG_WARN("fail to build merge task", KR(ret)); } } @@ -551,21 +590,21 @@ int ObTableLoadMerger::build_del_lob_ctx(bool &need_del_lob) } } if (OB_SUCC(ret)) { - if (OB_FAIL(range_splitter.init(multiple_sstable_array, store_ctx_->lob_id_table_data_desc_, - &(store_ctx_->ctx_->schema_.lob_meta_datum_utils_), - store_ctx_->ctx_->schema_.lob_meta_column_descs_))) { + if (OB_FAIL(range_splitter.init(multiple_sstable_array, store_table_ctx_->lob_id_table_data_desc_, + &(store_table_ctx_->schema_->lob_meta_datum_utils_), + store_table_ctx_->schema_->lob_meta_column_descs_))) { LOG_WARN("fail to init range splitter", KR(ret)); } } for (int64_t i = 0; OB_SUCC(ret) && i < tablet_merge_ctxs.count(); ++i) { ObDirectLoadTabletMergeCtx *tablet_merge_ctx = tablet_merge_ctxs.at(i); ObDirectLoadInsertTabletContext *insert_tablet_ctx = nullptr; - if (OB_FAIL(store_ctx_->insert_table_ctx_->get_tablet_context( - tablet_merge_ctx->get_tablet_id(), insert_tablet_ctx))) { + if (OB_FAIL(store_table_ctx_->insert_table_ctx_->get_tablet_context( + tablet_merge_ctx->get_tablet_id(), insert_tablet_ctx))) { LOG_WARN("fail to get tablet ctx", KR(ret), K(tablet_merge_ctx->get_tablet_id())); } else if (OB_FAIL(tablet_merge_ctx->build_del_lob_task( - multiple_sstable_array, range_splitter, param_.session_count_, - insert_tablet_ctx->get_min_insert_lob_id()))) { + multiple_sstable_array, range_splitter, param_.session_count_, + insert_tablet_ctx->get_min_insert_lob_id()))) { LOG_WARN("fail to build del lob task for multiple pk table", KR(ret)); } } @@ -790,23 +829,23 @@ int ObTableLoadMerger::handle_merge_thread_finish(int ret_code) // release tmpfile // TODO(suzhi.yt) release all tables and merge tasks table_compact_ctx_.result_.release_all_table_data(); - if (store_ctx_->insert_table_ctx_->need_rescan()) { + if (store_table_ctx_->insert_table_ctx_->need_rescan()) { if (OB_FAIL(build_rescan_ctx())) { LOG_WARN("fail to build rescan ctx", KR(ret)); } else if (OB_FAIL(start_rescan())) { LOG_WARN("fail to start rescan", KR(ret)); } - } else if (store_ctx_->insert_table_ctx_->need_del_lob()) { + } else if (store_table_ctx_->insert_table_ctx_->need_del_lob()) { if (OB_FAIL(merge_ctx_.close_table_builder())) { LOG_WARN("fail to close table builder", KR(ret)); - } else if (OB_FAIL(lob_id_compact_ctx_.init(store_ctx_, &lob_id_compact_config_))) { + } else if (OB_FAIL(lob_id_compact_ctx_.init(store_ctx_, store_table_ctx_, &lob_id_compact_config_))) { LOG_WARN("fail to init lob compact", KR(ret)); } else if (OB_FAIL(lob_id_compact_ctx_.start())) { LOG_WARN("fail to start lob compact", KR(ret)); } } else { - if (OB_FAIL(store_ctx_->set_status_merged())) { - LOG_WARN("fail to set store status merged", KR(ret)); + if (OB_FAIL(store_ctx_->merger_manager_->handle_merge_finish())) { + LOG_WARN("fail to handle merge finish", KR(ret)); } } } @@ -825,8 +864,8 @@ int ObTableLoadMerger::handle_rescan_thread_finish(const int ret_code) if (OB_UNLIKELY(is_stop_ || has_error_)) { } else { FLOG_INFO("LOAD RESCAN COMPLETED"); - if (OB_FAIL(store_ctx_->set_status_merged())) { - LOG_WARN("fail to set store status merged", KR(ret)); + if (OB_FAIL(store_ctx_->merger_manager_->handle_merge_finish())) { + LOG_WARN("fail to handle merge finish", KR(ret)); } } } @@ -845,8 +884,8 @@ int ObTableLoadMerger::handle_del_lob_thread_finish(int ret_code) } else { FLOG_INFO("LOAD DEL LOB COMPLETED"); lob_id_compact_ctx_.result_.release_all_table_data(); - if (OB_FAIL(store_ctx_->set_status_merged())) { - LOG_WARN("fail to set store status merged", KR(ret)); + if (OB_FAIL(store_ctx_->merger_manager_->handle_merge_finish())) { + LOG_WARN("fail to handle merge finish", KR(ret)); } } } diff --git a/src/observer/table_load/ob_table_load_merger.h b/src/observer/table_load/ob_table_load_merger.h index ee4bea1d0..528f4483b 100644 --- a/src/observer/table_load/ob_table_load_merger.h +++ b/src/observer/table_load/ob_table_load_merger.h @@ -28,6 +28,7 @@ namespace observer { class ObTableLoadParam; class ObTableLoadStoreCtx; +class ObTableLoadStoreTableCtx; class ObTableLoadMerger { @@ -39,7 +40,7 @@ class ObTableLoadMerger class RescanTaskCallback; class DelLobTaskCallback; public: - ObTableLoadMerger(ObTableLoadStoreCtx *store_ctx); + ObTableLoadMerger(ObTableLoadStoreCtx *store_ctx, ObTableLoadStoreTableCtx *store_table_ctx); ~ObTableLoadMerger(); int init(); int start(); @@ -63,10 +64,13 @@ private: int handle_rescan_thread_finish(const int ret_code); void handle_del_lob_task_finish(ObDirectLoadPartitionDelLobTask *&del_lob_task); int handle_del_lob_thread_finish(int ret_code); +public: + ObTableLoadStoreTableCtx * const store_table_ctx_; private: + common::ObArenaAllocator allocator_; ObTableLoadStoreCtx * const store_ctx_; const ObTableLoadParam ¶m_; - ObTableLoadTableCompactConfigMainTable table_compact_config_; + ObTableLoadTableCompactConfig * table_compact_config_; ObTableLoadTableCompactConfigLobIdTable lob_id_compact_config_; ObTableLoadTableCompactCtx table_compact_ctx_; ObTableLoadTableCompactCtx lob_id_compact_ctx_; diff --git a/src/observer/table_load/ob_table_load_merger_manager.cpp b/src/observer/table_load/ob_table_load_merger_manager.cpp new file mode 100644 index 000000000..453580b2b --- /dev/null +++ b/src/observer/table_load/ob_table_load_merger_manager.cpp @@ -0,0 +1,126 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "observer/table_load/ob_table_load_merger_manager.h" +#include "observer/table_load/ob_table_load_store_ctx.h" +#include "observer/table_load/ob_table_load_merger.h" +#include "observer/table_load/ob_table_load_store_table_ctx.h" +#include "share/table/ob_table_load_sql_statistics.h" + +#define USING_LOG_PREFIX SERVER +namespace oceanbase +{ +namespace observer +{ +void ObTableLoadMergerManager::stop() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTableLoadMergerManager not init", KR(ret)); + } else { + if (OB_ISNULL(store_ctx_->data_store_table_ctx_) || OB_ISNULL(store_ctx_->data_store_table_ctx_->merger_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected main store table ctx or merger is NULL", KR(ret)); + } else { + store_ctx_->data_store_table_ctx_->merger_->stop(); + } + FOREACH(it, store_ctx_->index_store_table_ctx_map_) { + if (OB_ISNULL(it->second) || OB_ISNULL(it->second->merger_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected index store table ctx or merger is NULL", KR(ret), K(it->first)); + } else { + it->second->merger_->stop(); + } + } + + } +} + +int ObTableLoadMergerManager::handle_merge_finish() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTableLoadMergerManager not init", KR(ret)); + } else { + if (index_merger_idx_ < index_store_array_.size()) { + if (OB_FAIL(index_store_array_.at(index_merger_idx_)->close_index_table_builder())) { + LOG_WARN("fail to close index table builder", KR(ret)); + } else if (OB_FAIL(index_store_array_.at(index_merger_idx_)->merger_->start())) { + LOG_WARN("fail to start index ObTableLoadMerger", KR(ret), K(index_merger_idx_)); + } + LOG_INFO ("merge index start", K(index_merger_idx_)); + index_merger_idx_++; + } else { + if (OB_FAIL(store_ctx_->set_status_merged())) { + LOG_WARN("fail to set status merged", KR(ret)); + } + } + } + return ret; +} + +int ObTableLoadMergerManager::init() +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObTableLoadMergerManager init twice", KR(ret), KP(this)); + } else { + FOREACH_X(it, store_ctx_->index_store_table_ctx_map_, OB_SUCC(ret)) { + if (OB_ISNULL(it->second)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected index store table ctx is NULL", KR(ret), K(it->first)); + } else if (OB_FAIL(index_store_array_.push_back(it->second))) { + LOG_WARN("fail to push back index store table ctx", KR(ret), K(it->first)); + } + } + if (OB_SUCC(ret)) { + LOG_INFO("INDEX MERGER COUNT", K(index_store_array_.size())); + is_inited_ = true; + } + } + return ret; +} + +int ObTableLoadMergerManager::commit(table::ObTableLoadDmlStat &dml_stats, table::ObTableLoadSqlStatistics &sql_statistics) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTableLoadMergerManager not init", KR(ret)); + } else if (OB_ISNULL(store_ctx_->data_store_table_ctx_) || OB_ISNULL(store_ctx_->data_store_table_ctx_->insert_table_ctx_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected main store table ctx or insert_table_ctx is NULL", KR(ret)); + } else if (OB_FAIL(store_ctx_->data_store_table_ctx_->insert_table_ctx_->commit(dml_stats, sql_statistics))) { + LOG_WARN("fail to commit main ObTableLoadMerger", KR(ret)); + } + return ret; +} + +int ObTableLoadMergerManager::start() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTableLoadMergerManager not init", KR(ret)); + } else if (OB_ISNULL(store_ctx_->data_store_table_ctx_) || OB_ISNULL(store_ctx_->data_store_table_ctx_->merger_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected main store table ctx or merger is NULL", KR(ret)); + } else if (OB_FAIL(store_ctx_->data_store_table_ctx_->merger_->start())) { + LOG_WARN("fail to start main ObTableLoadMerger", KR(ret)); + } + return ret; +} +} +} \ No newline at end of file diff --git a/src/observer/table_load/ob_table_load_merger_manager.h b/src/observer/table_load/ob_table_load_merger_manager.h new file mode 100644 index 000000000..2edf562a4 --- /dev/null +++ b/src/observer/table_load/ob_table_load_merger_manager.h @@ -0,0 +1,43 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#pragma once +#include "lib/container/ob_array.h" +namespace oceanbase +{ +namespace table +{ +class ObTableLoadDmlStat; +class ObTableLoadSqlStatistics; +} // namespace table +namespace observer +{ +class ObTableLoadStoreTableCtx; +class ObTableLoadStoreCtx; +class ObTableLoadMergerManager +{ +public: + ObTableLoadMergerManager(ObTableLoadStoreCtx *store_ctx) : store_ctx_(store_ctx), index_merger_idx_(0), is_inited_(false) {} + void stop(); + int start(); + int init(); + int handle_merge_finish(); + int commit(table::ObTableLoadDmlStat &dml_stats, table::ObTableLoadSqlStatistics &sql_statistics); +private: + ObTableLoadStoreCtx * store_ctx_; + common::ObArray index_store_array_; + int index_merger_idx_; + bool is_inited_; +}; + +} +} \ No newline at end of file diff --git a/src/observer/table_load/ob_table_load_multiple_heap_table_compactor.cpp b/src/observer/table_load/ob_table_load_multiple_heap_table_compactor.cpp index aaee61ca0..06abb36c7 100644 --- a/src/observer/table_load/ob_table_load_multiple_heap_table_compactor.cpp +++ b/src/observer/table_load/ob_table_load_multiple_heap_table_compactor.cpp @@ -14,6 +14,7 @@ #include "observer/table_load/ob_table_load_multiple_heap_table_compactor.h" #include "observer/table_load/ob_table_load_error_row_handler.h" +#include "observer/table_load/ob_table_load_data_row_handler.h" #include "observer/table_load/ob_table_load_service.h" #include "observer/table_load/ob_table_load_stat.h" #include "observer/table_load/ob_table_load_store_ctx.h" @@ -23,6 +24,7 @@ #include "observer/table_load/ob_table_load_trans_store.h" #include "storage/direct_load/ob_direct_load_multiple_heap_table_builder.h" #include "storage/direct_load/ob_direct_load_multiple_heap_table_compactor.h" +#include "observer/table_load/ob_table_load_store_table_ctx.h" namespace oceanbase { @@ -295,6 +297,7 @@ private: ObTableLoadMultipleHeapTableCompactor::ObTableLoadMultipleHeapTableCompactor() : store_ctx_(nullptr), + store_table_ctx_(nullptr), param_(nullptr), allocator_("TLD_MemC"), finish_task_count_(0) @@ -310,6 +313,7 @@ ObTableLoadMultipleHeapTableCompactor::~ObTableLoadMultipleHeapTableCompactor() void ObTableLoadMultipleHeapTableCompactor::reset() { store_ctx_ = nullptr; + store_table_ctx_ = nullptr; param_ = nullptr; finish_task_count_ = 0; mem_ctx_.reset(); @@ -321,20 +325,20 @@ int ObTableLoadMultipleHeapTableCompactor::inner_init() int ret = OB_SUCCESS; const uint64_t tenant_id = MTL_ID(); store_ctx_ = compact_ctx_->store_ctx_; + store_table_ctx_ = compact_ctx_->store_table_ctx_; param_ = &(store_ctx_->ctx_->param_); mem_ctx_.mem_dump_task_count_ = param_->session_count_ / 3; //暂时先写成1/3，后续再优化 if (mem_ctx_.mem_dump_task_count_ == 0) { mem_ctx_.mem_dump_task_count_ = 1; } - mem_ctx_.table_data_desc_ = store_ctx_->table_data_desc_; - mem_ctx_.datum_utils_ = &(store_ctx_->ctx_->schema_.datum_utils_); - mem_ctx_.need_sort_ = param_->need_sort_; + mem_ctx_.table_data_desc_ = store_table_ctx_->table_data_desc_; + mem_ctx_.datum_utils_ = &(store_table_ctx_->schema_->datum_utils_); + mem_ctx_.need_sort_ = store_table_ctx_->need_sort_; mem_ctx_.mem_load_task_count_ = param_->session_count_; - mem_ctx_.column_count_ = - (store_ctx_->ctx_->schema_.is_heap_table_ ? store_ctx_->ctx_->schema_.store_column_count_ - 1 - : store_ctx_->ctx_->schema_.store_column_count_); - mem_ctx_.dml_row_handler_ = store_ctx_->error_row_handler_; + mem_ctx_.column_count_ = (store_table_ctx_->schema_->is_heap_table_ ? store_table_ctx_->schema_->store_column_count_ - 1 + : store_table_ctx_->schema_->store_column_count_); + mem_ctx_.dml_row_handler_ = store_table_ctx_->row_handler_; mem_ctx_.file_mgr_ = store_ctx_->tmp_file_mgr_; mem_ctx_.dup_action_ = param_->dup_action_; diff --git a/src/observer/table_load/ob_table_load_multiple_heap_table_compactor.h b/src/observer/table_load/ob_table_load_multiple_heap_table_compactor.h index 32d23bfc4..1c1fa7503 100644 --- a/src/observer/table_load/ob_table_load_multiple_heap_table_compactor.h +++ b/src/observer/table_load/ob_table_load_multiple_heap_table_compactor.h @@ -72,6 +72,7 @@ private: int create_heap_table_sorter(ObDirectLoadMultipleHeapTableSorter *&heap_table_sorter); private: ObTableLoadStoreCtx *store_ctx_; + ObTableLoadStoreTableCtx *store_table_ctx_; const ObTableLoadParam *param_; common::ObArenaAllocator allocator_; //需要最后析构 int64_t finish_task_count_ CACHE_ALIGNED; diff --git a/src/observer/table_load/ob_table_load_open_insert_table_ctx_manager.cpp b/src/observer/table_load/ob_table_load_open_insert_table_ctx_manager.cpp new file mode 100644 index 000000000..bb11e8992 --- /dev/null +++ b/src/observer/table_load/ob_table_load_open_insert_table_ctx_manager.cpp @@ -0,0 +1,89 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "observer/table_load/ob_table_load_open_insert_table_ctx_manager.h" +#include "observer/table_load/ob_table_load_store_ctx.h" +#include "observer/table_load/ob_table_load_store_table_ctx.h" + +#define USING_LOG_PREFIX SERVER +namespace oceanbase +{ +namespace observer +{ +int ObTableLoadOpenInsertTableCtxManager::init() +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObTableLoadOpenInsertTableCtxManager init twice", KR(ret), K(is_inited_)); + } else { + common::hash::ObHashMap &data_insert_tablet_ctxs = + store_ctx_->data_store_table_ctx_->insert_table_ctx_->get_tablet_ctx_map(); + FOREACH_X(iter, data_insert_tablet_ctxs, OB_SUCC(ret)) { + if (OB_ISNULL(iter->second)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("insert tablet ctx is null", KR(ret)); + } else if (OB_FAIL(insert_tablet_ctxs_.push_back(iter->second))) { + LOG_WARN("fail to push back insert tablet ctx", KR(ret)); + } + } + FOREACH_X(it, store_ctx_->index_store_table_ctx_map_, OB_SUCC(ret)) { + common::hash::ObHashMap + &index_insert_tablet_ctxs = it->second->insert_table_ctx_->get_tablet_ctx_map(); + FOREACH_X(iter, index_insert_tablet_ctxs, OB_SUCC(ret)) + { + if (OB_ISNULL(iter->second)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("insert tablet ctx is null", KR(ret)); + } else if (OB_FAIL(insert_tablet_ctxs_.push_back(iter->second))) { + LOG_WARN("fail to push back insert tablet ctx", KR(ret)); + } + } + } + if (OB_SUCC(ret)) { + is_inited_ = true; + } + } + return ret; +} + +int ObTableLoadOpenInsertTableCtxManager::get_next_insert_tablet_ctx( + ObDirectLoadInsertTabletContext *&tablet_ctx) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTableLoadOpenInsertTableCtxManager not init", KR(ret), K(is_inited_)); + } else { + ObMutexGuard guard(op_lock_); + if (insert_tablet_context_idx_ < insert_tablet_ctxs_.count()) { + tablet_ctx = insert_tablet_ctxs_[insert_tablet_context_idx_]; + insert_tablet_context_idx_++; + } else { + ret = OB_ITER_END; + } + } + return ret; +} + +void ObTableLoadOpenInsertTableCtxManager::handle_open_insert_tablet_ctx_finish(bool &is_finish) +{ + ObMutexGuard guard(op_lock_); + is_finish = false; + opened_insert_tablet_count_++; + if (opened_insert_tablet_count_ == insert_tablet_ctxs_.count()) { + is_finish = true; + } +} + +} +} // namespace oceanbase \ No newline at end of file diff --git a/src/observer/table_load/ob_table_load_open_insert_table_ctx_manager.h b/src/observer/table_load/ob_table_load_open_insert_table_ctx_manager.h new file mode 100644 index 000000000..03642cf83 --- /dev/null +++ b/src/observer/table_load/ob_table_load_open_insert_table_ctx_manager.h @@ -0,0 +1,43 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#pragma once +#include "storage/direct_load/ob_direct_load_insert_table_ctx.h" + +namespace oceanbase +{ +namespace observer +{ +class ObTableLoadStoreCtx; +class ObTableLoadOpenInsertTableCtxManager +{ +public: + ObTableLoadOpenInsertTableCtxManager(ObTableLoadStoreCtx *store_ctx) + : store_ctx_(store_ctx), + insert_tablet_context_idx_(0), + opened_insert_tablet_count_(0), + is_inited_(false) + { + } + int get_next_insert_tablet_ctx(ObDirectLoadInsertTabletContext *&tablet_ctx); + void handle_open_insert_tablet_ctx_finish(bool &is_finish); + int init(); +private: + ObTableLoadStoreCtx * store_ctx_; + common::ObArray insert_tablet_ctxs_; + int insert_tablet_context_idx_; + int opened_insert_tablet_count_; + lib::ObMutex op_lock_; + bool is_inited_; +}; +} +} // namespace oceanbase \ No newline at end of file diff --git a/src/observer/table_load/ob_table_load_parallel_merge_ctx.cpp b/src/observer/table_load/ob_table_load_parallel_merge_ctx.cpp index 38110303a..7ae040e4e 100644 --- a/src/observer/table_load/ob_table_load_parallel_merge_ctx.cpp +++ b/src/observer/table_load/ob_table_load_parallel_merge_ctx.cpp @@ -14,6 +14,7 @@ #include "observer/table_load/ob_table_load_parallel_merge_ctx.h" #include "observer/table_load/ob_table_load_error_row_handler.h" +#include "observer/table_load/ob_table_load_data_row_handler.h" #include "observer/table_load/ob_table_load_service.h" #include "observer/table_load/ob_table_load_stat.h" #include "observer/table_load/ob_table_load_store_ctx.h" @@ -25,6 +26,7 @@ #include "storage/direct_load/ob_direct_load_multiple_sstable_scan_merge.h" #include "storage/direct_load/ob_direct_load_multiple_sstable_scanner.h" #include "storage/direct_load/ob_direct_load_range_splitter.h" +#include "observer/table_load/ob_table_load_store_table_ctx.h" namespace oceanbase { @@ -372,7 +374,7 @@ public: ObDirectLoadMultipleSSTableScanMergeParam scan_merge_param; scan_merge_param.table_data_desc_ = parallel_merge_ctx_->table_data_desc_; scan_merge_param.datum_utils_ = &(ctx_->schema_.datum_utils_); - scan_merge_param.dml_row_handler_ = parallel_merge_ctx_->store_ctx_->error_row_handler_; + scan_merge_param.dml_row_handler_ = parallel_merge_ctx_->store_table_ctx_->row_handler_; for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ctx_->merge_sstable_count_; ++i) { ObDirectLoadMultipleSSTable *sstable = tablet_ctx_->sstables_.at(i); if (OB_FAIL(sstable_array_.push_back(sstable))) { @@ -397,7 +399,7 @@ public: } else { ObDirectLoadMultipleSSTableBuildParam build_param; build_param.tablet_id_ = tablet_ctx_->tablet_id_; - build_param.table_data_desc_ = parallel_merge_ctx_->store_ctx_->table_data_desc_; + build_param.table_data_desc_ = parallel_merge_ctx_->table_data_desc_; build_param.datum_utils_ = &(ctx_->schema_.datum_utils_); build_param.file_mgr_ = parallel_merge_ctx_->store_ctx_->tmp_file_mgr_; build_param.extra_buf_ = extra_buf_; @@ -456,7 +458,7 @@ public: int ret = OB_SUCCESS; ObDirectLoadMultipleSSTableCompactParam compact_param; compact_param.tablet_id_ = tablet_ctx_->tablet_id_; - compact_param.table_data_desc_ = parallel_merge_ctx_->store_ctx_->table_data_desc_; + compact_param.table_data_desc_ = parallel_merge_ctx_->table_data_desc_; compact_param.datum_utils_ = &ctx_->schema_.datum_utils_; if (OB_FAIL(compactor_.init(compact_param))) { LOG_WARN("fail to init sstable compactor", KR(ret)); @@ -539,6 +541,7 @@ private: ObTableLoadParallelMergeCtx::ObTableLoadParallelMergeCtx() : store_ctx_(nullptr), + store_table_ctx_(nullptr), thread_count_(0), cb_(nullptr), allocator_("TLD_ParalMerge"), @@ -572,21 +575,22 @@ ObTableLoadParallelMergeCtx::~ObTableLoadParallelMergeCtx() heavy_task_list_.reset(); } -int ObTableLoadParallelMergeCtx::init(ObTableLoadStoreCtx *store_ctx, +int ObTableLoadParallelMergeCtx::init(ObTableLoadStoreCtx *store_ctx, ObTableLoadStoreTableCtx *store_table_ctx, const ObDirectLoadTableDataDesc &table_data_desc) { int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("ObTableLoadParallelMergeCtx init twice", KR(ret), KP(this)); - } else if (OB_UNLIKELY(nullptr == store_ctx || !table_data_desc.is_valid())) { + } else if (OB_UNLIKELY(nullptr == store_ctx || nullptr == store_table_ctx || !table_data_desc.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), KP(store_ctx), K(table_data_desc)); + LOG_WARN("invalid args", KR(ret), KP(store_ctx), KP(store_table_ctx), K(table_data_desc)); } else { if (OB_FAIL(tablet_ctx_map_.create(1024, "TLD_CptCtxMap", "TLD_CptCtxMap", MTL_ID()))) { LOG_WARN("fail to create ctx map", KR(ret)); } else { store_ctx_ = store_ctx; + store_table_ctx_ = store_table_ctx; table_data_desc_ = table_data_desc; thread_count_ = store_ctx_->task_scheduler_->get_thread_count(); is_inited_ = true; diff --git a/src/observer/table_load/ob_table_load_parallel_merge_ctx.h b/src/observer/table_load/ob_table_load_parallel_merge_ctx.h index a2df06974..10aae0078 100644 --- a/src/observer/table_load/ob_table_load_parallel_merge_ctx.h +++ b/src/observer/table_load/ob_table_load_parallel_merge_ctx.h @@ -25,6 +25,7 @@ namespace oceanbase namespace observer { class ObTableLoadStoreCtx; +class ObTableLoadStoreTableCtx; class ObTableLoadTask; class ObTableLoadParallelMergeSSTableCompare @@ -81,7 +82,7 @@ public: typedef TabletCtxMap::const_iterator TabletCtxIterator; ObTableLoadParallelMergeCtx(); ~ObTableLoadParallelMergeCtx(); - int init(ObTableLoadStoreCtx *store_ctx, const ObDirectLoadTableDataDesc &table_data_desc); + int init(ObTableLoadStoreCtx *store_ctx, ObTableLoadStoreTableCtx *store_table_ctx, const ObDirectLoadTableDataDesc &table_data_desc); int add_tablet_sstable(storage::ObDirectLoadMultipleSSTable *sstable); int start(ObTableLoadParallelMergeCb *cb); void stop(); @@ -105,6 +106,7 @@ private: private: ObTableLoadStoreCtx *store_ctx_; ObDirectLoadTableDataDesc table_data_desc_; + ObTableLoadStoreTableCtx *store_table_ctx_; int64_t thread_count_; ObTableLoadParallelMergeCb *cb_; common::ObArenaAllocator allocator_; diff --git a/src/observer/table_load/ob_table_load_parallel_merge_table_compactor.cpp b/src/observer/table_load/ob_table_load_parallel_merge_table_compactor.cpp index 35caf1eef..f1c81bf88 100644 --- a/src/observer/table_load/ob_table_load_parallel_merge_table_compactor.cpp +++ b/src/observer/table_load/ob_table_load_parallel_merge_table_compactor.cpp @@ -77,8 +77,8 @@ int ObTableLoadParallelMergeTableCompactor::inner_init() { int ret = OB_SUCCESS; const uint64_t tenant_id = MTL_ID(); - if (OB_FAIL(parallel_merge_ctx_.init(compact_ctx_->store_ctx_, - compact_ctx_->store_ctx_->table_data_desc_))) { + if (OB_FAIL(parallel_merge_ctx_.init(compact_ctx_->store_ctx_, compact_ctx_->store_table_ctx_, + compact_ctx_->store_table_ctx_->table_data_desc_))) { LOG_WARN("fail to init parallel merge ctx", KR(ret)); } else if (OB_FAIL(parallel_merge_cb_.init(this))) { LOG_WARN("fail to init parallel merge cb", KR(ret)); diff --git a/src/observer/table_load/ob_table_load_pre_sort_writer.cpp b/src/observer/table_load/ob_table_load_pre_sort_writer.cpp index d1f2197b0..a142023f1 100644 --- a/src/observer/table_load/ob_table_load_pre_sort_writer.cpp +++ b/src/observer/table_load/ob_table_load_pre_sort_writer.cpp @@ -82,7 +82,7 @@ int ObTableLoadPreSortWriter::write(int32_t session_id, if (OB_ISNULL(store_writer_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("store writer is nullptr", KR(ret)); - } else if (OB_FAIL(datum_row.init(pre_sorter_->store_ctx_->table_data_desc_.column_count_))) { + } else if (OB_FAIL(datum_row.init(pre_sorter_->store_ctx_->data_store_table_ctx_->table_data_desc_.column_count_))) { LOG_WARN("fail to init datum row", KR(ret)); } else if (OB_FAIL(store_writer_->cast_row(session_id, new_row, datum_row))) { if (OB_UNLIKELY(OB_EAGAIN != ret)) { @@ -92,8 +92,9 @@ int ObTableLoadPreSortWriter::write(int32_t session_id, } } else if (OB_FAIL(external_row.external_row_.from_datums(datum_row.storage_datums_, datum_row.count_, - pre_sorter_->store_ctx_->table_data_desc_.rowkey_column_num_, - row.obj_row_.seq_no_))) { + pre_sorter_->store_ctx_->data_store_table_ctx_->table_data_desc_.rowkey_column_num_, + row.obj_row_.seq_no_, + false))) { LOG_WARN("fail to cast to external row", KR(ret)); } else if (OB_FAIL(append_row(external_row))) { LOG_WARN("fail to append row", KR(ret)); @@ -118,8 +119,9 @@ int ObTableLoadPreSortWriter::px_write(const ObTabletID &tablet_id, // do nothing } else if (OB_FAIL(external_row.external_row_.from_datums(row.storage_datums_, row.count_, - pre_sorter_->store_ctx_->table_data_desc_.rowkey_column_num_, - seq_no))) { + pre_sorter_->store_ctx_->data_store_table_ctx_->table_data_desc_.rowkey_column_num_, + seq_no, + false))) { LOG_WARN("fail to cast to external row", KR(ret)); } else if (OB_FAIL(append_row(external_row))) { LOG_WARN("fail to append row", KR(ret)); diff --git a/src/observer/table_load/ob_table_load_pre_sorter.cpp b/src/observer/table_load/ob_table_load_pre_sorter.cpp index 4ad6f2e44..0971a2569 100644 --- a/src/observer/table_load/ob_table_load_pre_sorter.cpp +++ b/src/observer/table_load/ob_table_load_pre_sorter.cpp @@ -75,7 +75,7 @@ int ObTableLoadPreSorter::init() LOG_WARN("fail to init task count", KR(ret), K(dump_task_count_), K(other_task_count_)); } else if (OB_FAIL(init_mem_ctx())) { LOG_WARN("fail to init mem ctx", KR(ret)); - } else if (OB_FAIL(parallel_merge_ctx_.init(store_ctx_, mem_ctx_.table_data_desc_))) { + } else if (OB_FAIL(parallel_merge_ctx_.init(store_ctx_, store_ctx_->data_store_table_ctx_, mem_ctx_.table_data_desc_))) { LOG_WARN("fail to init parallel_merge_ctx_", KR(ret)); } else if (OB_FAIL(init_sample_task_scheduler())) { LOG_WARN("fail to init sample task scheduler", KR(ret)); @@ -107,14 +107,14 @@ int ObTableLoadPreSorter::init_task_count() int ObTableLoadPreSorter::init_mem_ctx() { int ret = OB_SUCCESS; - mem_ctx_.table_data_desc_ = store_ctx_->table_data_desc_; + mem_ctx_.table_data_desc_ = store_ctx_->data_store_table_ctx_->table_data_desc_; mem_ctx_.datum_utils_ = &(ctx_->schema_.datum_utils_); mem_ctx_.need_sort_ = ctx_->param_.need_sort_; mem_ctx_.column_count_ = (store_ctx_->ctx_->schema_.is_heap_table_ ? store_ctx_->ctx_->schema_.store_column_count_ - 1 : store_ctx_->ctx_->schema_.store_column_count_); mem_ctx_.mem_dump_task_count_ = dump_task_count_; - mem_ctx_.dml_row_handler_ = store_ctx_->error_row_handler_; + mem_ctx_.dml_row_handler_ = store_ctx_->data_store_table_ctx_->row_handler_; mem_ctx_.dup_action_ = ctx_->param_.dup_action_; mem_ctx_.file_mgr_ = store_ctx_->tmp_file_mgr_; if (OB_FAIL(mem_ctx_.init())) { @@ -324,7 +324,7 @@ int ObTableLoadPreSorter::handle_parallel_merge_success() int ObTableLoadPreSorter::build_parallel_merge_result() { int ret = OB_SUCCESS; - ObTableLoadMerger *merger = store_ctx_->merger_; + ObTableLoadMerger *merger = store_ctx_->data_store_table_ctx_->merger_; const ObTableLoadParallelMergeCtx::TabletCtxMap &tablet_ctx_map = parallel_merge_ctx_.get_tablet_ctx_map(); if (OB_ISNULL(merger)) { @@ -401,19 +401,19 @@ int ObTableLoadPreSorter::build_merge_param(ObDirectLoadMergeParam& merge_param) merge_param.store_column_count_ = ctx_->schema_.store_column_count_; merge_param.fill_cg_thread_cnt_ = ctx_->param_.session_count_; merge_param.lob_column_idxs_ = &(ctx_->schema_.lob_column_idxs_); - merge_param.table_data_desc_ = store_ctx_->table_data_desc_; + merge_param.table_data_desc_ = store_ctx_->data_store_table_ctx_->table_data_desc_; merge_param.datum_utils_ = &(ctx_->schema_.datum_utils_); merge_param.lob_column_idxs_ = &(ctx_->schema_.lob_column_idxs_); merge_param.col_descs_ = &(ctx_->schema_.column_descs_); - merge_param.lob_id_table_data_desc_ = store_ctx_->lob_id_table_data_desc_; + merge_param.lob_id_table_data_desc_ = store_ctx_->data_store_table_ctx_->lob_id_table_data_desc_; merge_param.lob_meta_datum_utils_ = &(ctx_->schema_.lob_meta_datum_utils_); merge_param.lob_meta_col_descs_ = &(ctx_->schema_.lob_meta_column_descs_); merge_param.is_heap_table_ = ctx_->schema_.is_heap_table_; - merge_param.is_fast_heap_table_ = store_ctx_->is_fast_heap_table_; + merge_param.is_fast_heap_table_ = store_ctx_->data_store_table_ctx_->is_fast_heap_table_; merge_param.is_incremental_ = ObDirectLoadMethod::is_incremental(ctx_->param_.method_); merge_param.insert_mode_ = ctx_->param_.insert_mode_; - merge_param.insert_table_ctx_ = store_ctx_->insert_table_ctx_; - merge_param.dml_row_handler_ = store_ctx_->error_row_handler_; + merge_param.insert_table_ctx_ = store_ctx_->data_store_table_ctx_->insert_table_ctx_; + merge_param.dml_row_handler_ = store_ctx_->data_store_table_ctx_->row_handler_; merge_param.file_mgr_ = store_ctx_->tmp_file_mgr_; merge_param.trans_param_ = store_ctx_->trans_param_; return ret; @@ -488,7 +488,7 @@ int ObTableLoadPreSorter::build_merge_ctx_for_multiple_mode(ObDirectLoadMergePar ObDirectLoadTabletMergeCtx *tablet_merge_ctx = tablet_merge_ctxs.at(i); if (OB_FAIL(tablet_merge_ctx->build_merge_task( *table_array, ctx_->schema_.column_descs_, ctx_->param_.session_count_, - store_ctx_->is_multiple_mode_))) { + store_ctx_->data_store_table_ctx_->is_multiple_mode_))) { LOG_WARN("fail to build merge task", KR(ret)); } } @@ -522,7 +522,7 @@ int ObTableLoadPreSorter::build_merge_ctx_for_non_multiple_mode(ObDirectLoadMerg if (OB_SUCC(ret)) { if (OB_FAIL(tablet_merge_ctx->build_merge_task( *table_array, ctx_->schema_.column_descs_, ctx_->param_.session_count_, - store_ctx_->is_multiple_mode_))) { + store_ctx_->data_store_table_ctx_->is_multiple_mode_))) { LOG_WARN("fail to build merge task", KR(ret)); } } @@ -537,7 +537,7 @@ int ObTableLoadPreSorter::build_merge_ctx() { int ret = OB_SUCCESS; ObDirectLoadMergeParam merge_param; - ObTableLoadMerger *merger = store_ctx_->merger_; + ObTableLoadMerger *merger = store_ctx_->data_store_table_ctx_->merger_; if (OB_ISNULL(merger)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("merger shoult not be nullptr", KR(ret)); @@ -545,9 +545,9 @@ int ObTableLoadPreSorter::build_merge_ctx() LOG_WARN("fail to build merge param", KR(ret)); } else { ObTableLoadTableCompactResult &result = merger->table_compact_ctx_.result_; - if (OB_FAIL(merger->merge_ctx_.init(ctx_, merge_param, store_ctx_->ls_partition_ids_))) { + if (OB_FAIL(merger->merge_ctx_.init(ctx_, merge_param, store_ctx_->data_store_table_ctx_->ls_partition_ids_))) { LOG_WARN("fail to init merge ctx", KR(ret)); - } else if (store_ctx_->is_multiple_mode_) { + } else if (store_ctx_->data_store_table_ctx_->is_multiple_mode_) { if (OB_FAIL(build_merge_ctx_for_multiple_mode(merge_param, merger, result))) { LOG_WARN("fail to build merge ctx for multiple mode", KR(ret)); } @@ -567,7 +567,7 @@ int ObTableLoadPreSorter::build_merge_ctx() int ObTableLoadPreSorter::start_merge() { int ret = OB_SUCCESS; - if (OB_FAIL(store_ctx_->merger_->start_merge())) { + if (OB_FAIL(store_ctx_->data_store_table_ctx_->merger_->start_merge())) { LOG_WARN("fail to start merge", KR(ret)); } return ret; diff --git a/src/observer/table_load/ob_table_load_schema.cpp b/src/observer/table_load/ob_table_load_schema.cpp index 77c040d93..d23de3aed 100644 --- a/src/observer/table_load/ob_table_load_schema.cpp +++ b/src/observer/table_load/ob_table_load_schema.cpp @@ -380,6 +380,35 @@ int ObTableLoadSchema::check_has_lob_column(const ObTableSchema *table_schema, b return ret; } +int ObTableLoadSchema::check_has_non_local_index(share::schema::ObSchemaGetterGuard &schema_guard, + const share::schema::ObTableSchema *table_schema, + bool &bret) +{ + int ret = OB_SUCCESS; + bret = false; + if (OB_UNLIKELY(nullptr == table_schema)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(table_schema)); + } else { + const ObIArray &simple_index_infos = table_schema->get_simple_index_infos(); + for (int64_t i = 0; OB_SUCC(ret) && i < simple_index_infos.count(); i++) { + const ObAuxTableMetaInfo &index_table_info = simple_index_infos.at(i); + const share::schema::ObTableSchema *index_table_schema = nullptr; + if (OB_FAIL(get_table_schema(schema_guard, MTL_ID(), index_table_info.table_id_, + index_table_schema))) { + LOG_WARN("fail to get table shema of index table", KR(ret), K(index_table_info.table_id_)); + } else { + if (INDEX_TYPE_NORMAL_LOCAL != index_table_schema->get_index_type() && + INDEX_TYPE_NORMAL_GLOBAL_LOCAL_STORAGE != index_table_schema->get_index_type()) { + bret = true; + break; + } + } + } + } + return ret; +} + int ObTableLoadSchema::get_tenant_optimizer_gather_stats_on_load(const uint64_t tenant_id, bool &value) { @@ -437,6 +466,7 @@ ObTableLoadSchema::ObTableLoadSchema() part_level_(PARTITION_LEVEL_ZERO), schema_version_(0), lob_meta_table_id_(OB_INVALID_ID), + index_table_count_(0), is_inited_(false) { allocator_.set_tenant_id(MTL_ID()); @@ -465,6 +495,7 @@ void ObTableLoadSchema::reset() part_level_ = PARTITION_LEVEL_ZERO; schema_version_ = 0; lob_meta_table_id_ = OB_INVALID_ID; + index_table_count_ = 0; lob_column_idxs_.reset(); column_descs_.reset(); multi_version_column_descs_.reset(); @@ -515,6 +546,7 @@ int ObTableLoadSchema::init_table_schema(const ObTableSchema *table_schema) if (table_schema->has_lob_aux_table()) { lob_meta_table_id_ = table_schema->get_aux_lob_meta_tid(); } + index_table_count_ = table_schema->get_simple_index_infos().count(); if (OB_FAIL(ObTableLoadUtils::deep_copy(table_schema->get_table_name_str(), table_name_, allocator_))) { LOG_WARN("fail to deep copy table name", KR(ret)); diff --git a/src/observer/table_load/ob_table_load_schema.h b/src/observer/table_load/ob_table_load_schema.h index fb9539bba..f9237047e 100644 --- a/src/observer/table_load/ob_table_load_schema.h +++ b/src/observer/table_load/ob_table_load_schema.h @@ -75,7 +75,9 @@ public: static int check_has_unused_column(const share::schema::ObTableSchema *table_schema, bool &bret); static int check_has_roaringbitmap_column(const share::schema::ObTableSchema *table_schema, bool &bret); static int check_has_lob_column(const share::schema::ObTableSchema *table_schema, bool &bret); - + static int check_has_non_local_index(share::schema::ObSchemaGetterGuard &schema_guard, + const share::schema::ObTableSchema *table_schema, + bool &bret); static int get_tenant_optimizer_gather_stats_on_load(const uint64_t tenant_id, bool &value); public: @@ -112,6 +114,7 @@ public: share::schema::ObPartitionLevel part_level_; int64_t schema_version_; uint64_t lob_meta_table_id_; + int64_t index_table_count_; common::ObArray lob_column_idxs_; // if it is a heap table, it contains hidden primary key column // does not contain virtual generated columns diff --git a/src/observer/table_load/ob_table_load_service.cpp b/src/observer/table_load/ob_table_load_service.cpp index 509493963..1a11470ef 100644 --- a/src/observer/table_load/ob_table_load_service.cpp +++ b/src/observer/table_load/ob_table_load_service.cpp @@ -510,6 +510,7 @@ int ObTableLoadService::check_support_direct_load(ObSchemaGetterGuard &schema_gu bool has_invisible_column = false; bool has_unused_column = false; bool has_roaringbitmap_column = false; + bool has_non_normal_local_index = false; // check if it is a user table const char *tmp_prefix = ObDirectLoadMode::is_insert_overwrite(load_mode) ? InsertOverwritePrefix : EmptyPrefix; @@ -607,10 +608,24 @@ int ObTableLoadService::check_support_direct_load(ObSchemaGetterGuard &schema_gu ret = OB_NOT_SUPPORTED; LOG_WARN("version lower than 4.3.1.0 does not support incremental direct-load", KR(ret)); FORWARD_USER_ERROR_MSG(ret, "version lower than 4.3.1.0 does not support incremental direct-load"); - } else if (table_schema->get_index_tid_count() > 0) { + } else if (table_schema->get_simple_index_infos().count() > 0 && + OB_FAIL(ObTableLoadSchema::check_has_non_local_index( + schema_guard, table_schema, has_non_normal_local_index))) { + LOG_WARN("fail to check support direct load for local index", KR(ret)); + } else if (has_non_normal_local_index) { ret = OB_NOT_SUPPORTED; - LOG_WARN("incremental direct-load does not support table with indexes", KR(ret)); - FORWARD_USER_ERROR_MSG(ret, "incremental direct-load does not support table with indexes"); + LOG_WARN("incremental direct-load does not support table with non-normal local index", + KR(ret)); + FORWARD_USER_ERROR_MSG( + ret, "incremental direct-load does not support table with global index or unique index"); + } else if (table_schema->get_simple_index_infos().count() > 0 && !has_non_normal_local_index && compat_version < DATA_VERSION_4_3_4_0) { + ret = OB_NOT_SUPPORTED; + LOG_WARN( + "version lower than 4.3.4.0 incremental direct-load does not support table with non-normal local index", + KR(ret)); + FORWARD_USER_ERROR_MSG( + ret, + "version lower than 4.3.4.0 incremental direct-load does not support table with non-normal local index"); } else if (table_schema->get_foreign_key_infos().count() > 0) { ret = OB_NOT_SUPPORTED; LOG_WARN("incremental direct-load does not support table with foreign keys", KR(ret)); diff --git a/src/observer/table_load/ob_table_load_store.cpp b/src/observer/table_load/ob_table_load_store.cpp index 23e825205..57ac101c9 100644 --- a/src/observer/table_load/ob_table_load_store.cpp +++ b/src/observer/table_load/ob_table_load_store.cpp @@ -24,6 +24,8 @@ #include "observer/table_load/ob_table_load_task_scheduler.h" #include "observer/table_load/ob_table_load_trans_store.h" #include "observer/table_load/ob_table_load_utils.h" +#include "observer/table_load/ob_table_load_store_table_ctx.h" +#include "observer/table_load/ob_table_load_merger_manager.h" #include "observer/table_load/ob_table_load_pre_sorter.h" #include "storage/direct_load/ob_direct_load_insert_table_ctx.h" #include "share/stat/ob_opt_stat_monitor_manager.h" @@ -59,6 +61,29 @@ int ObTableLoadStore::init_ctx( return ret; } +void ObTableLoadStore::cancel_table_ctx(ObTableLoadTableCtx *ctx) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(ctx->store_ctx_->data_store_table_ctx_) || + OB_ISNULL(ctx->store_ctx_->data_store_table_ctx_->insert_table_ctx_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected main store table ctx or insert_table_ctx is NULL", KR(ret)); + } else { + ctx->store_ctx_->data_store_table_ctx_->insert_table_ctx_->cancel(); + } + + FOREACH(it, ctx->store_ctx_->index_store_table_ctx_map_) + { + if (OB_ISNULL(it->second) || OB_ISNULL(it->second->insert_table_ctx_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected index store table ctx or insert_table_ctx is NULL", KR(ret), + K(it->first)); + } else { + it->second->insert_table_ctx_->cancel(); + } + } +} + void ObTableLoadStore::abort_ctx(ObTableLoadTableCtx *ctx, bool &is_stopped) { int ret = OB_SUCCESS; @@ -82,8 +107,8 @@ void ObTableLoadStore::abort_ctx(ObTableLoadTableCtx *ctx, bool &is_stopped) if (OB_SUCCESS != (tmp_ret = abort_active_trans(ctx))) { LOG_WARN("fail to abort active trans", KR(tmp_ret)); } - ctx->store_ctx_->insert_table_ctx_->cancel(); - ctx->store_ctx_->merger_->stop(); + cancel_table_ctx(ctx); + ctx->store_ctx_->merger_manager_->stop(); ctx->store_ctx_->task_scheduler_->stop(); is_stopped = ctx->store_ctx_->task_scheduler_->is_stopped() && (0 == ATOMIC_LOAD(&ctx->store_ctx_->px_writer_count_)); if (OB_NOT_NULL(ctx->store_ctx_->pre_sorter_)) { @@ -227,17 +252,14 @@ public: OB_TABLE_LOAD_STATISTICS_TIME_COST(INFO, store_open_tablet_time_us); int ret = OB_SUCCESS; while (OB_SUCC(ret)) { - ObTabletID tablet_id; ObDirectLoadInsertTabletContext *tablet_ctx = nullptr; - if (OB_FAIL(ctx_->store_ctx_->get_next_insert_tablet_ctx(tablet_id))) { + if (OB_FAIL(ctx_->store_ctx_->get_next_insert_tablet_ctx(tablet_ctx))) { if (OB_UNLIKELY(ret != OB_ITER_END)) { LOG_WARN("fail to get next insert tablet context", KR(ret)); } else { ret = OB_SUCCESS; break; } - } else if (OB_FAIL(ctx_->store_ctx_->insert_table_ctx_->get_tablet_context(tablet_id, tablet_ctx))) { - LOG_WARN("fail to get tablet context", KR(ret), K(tablet_id)); } else { bool is_finish = false; while (OB_SUCC(ret)) { @@ -247,9 +269,9 @@ public: } else if (OB_FAIL(ctx_->store_ctx_->check_status(ObTableLoadStatusType::INITED))) { LOG_WARN("fail to check status", KR(ret)); } else if (OB_FAIL(tablet_ctx->open())) { - LOG_WARN("fail to open tablet context", KR(ret), K(tablet_id)); + LOG_WARN("fail to open tablet context", KR(ret), KPC(tablet_ctx)); if (ret == OB_EAGAIN || ret == OB_MINOR_FREEZE_NOT_ALLOW) { - LOG_WARN("retry to open tablet context", K(tablet_id)); + LOG_WARN("retry to open tablet context"); ret = OB_SUCCESS; } } else { @@ -320,7 +342,7 @@ public: LOG_WARN("fail to close all chunk", KR(ret)); } } else { - if (OB_FAIL(ctx_->store_ctx_->merger_->start())) { + if (OB_FAIL(ctx_->store_ctx_->merger_manager_->start())) { LOG_WARN("fail to start merger", KR(ret)); } } @@ -460,15 +482,15 @@ int ObTableLoadStore::commit(ObTableLoadResultInfo &result_info, LOG_WARN("trans service is null", KR(ret)); } else if (OB_FAIL(store_ctx_->check_status(ObTableLoadStatusType::MERGED))) { LOG_WARN("fail to check store status", KR(ret)); - } else if (OB_FAIL(store_ctx_->insert_table_ctx_->commit(dml_stats, sql_statistics))) { + } else if (OB_FAIL(store_ctx_->merger_manager_->commit(dml_stats, sql_statistics))) { LOG_WARN("fail to commit insert table", KR(ret)); - } else if (ctx_->schema_.has_autoinc_column_ && OB_FAIL(store_ctx_->commit_autoinc_value())) { + } else if (store_ctx_->data_store_table_ctx_->schema_->has_autoinc_column_ && OB_FAIL(store_ctx_->commit_autoinc_value())) { LOG_WARN("fail to commit sync auto increment value", KR(ret)); } // 全量旁路导入的dml_stat在执行节点更新 // 增量旁路导入的dml_stat收集到协调节点在事务中更新 else if (ObDirectLoadMethod::is_full(param_.method_) && - OB_FAIL(ObOptStatMonitorManager::update_dml_stat_info_from_direct_load(dml_stats.dml_stat_array_))) { + OB_FAIL(ObOptStatMonitorManager::update_dml_stat_info_from_direct_load(dml_stats.dml_stat_array_))) { LOG_WARN("fail to update dml stat info", KR(ret)); } else if (ObDirectLoadMethod::is_full(param_.method_) && FALSE_IT(dml_stats.reset())) { } else if (ObDirectLoadMethod::is_incremental(param_.method_) && diff --git a/src/observer/table_load/ob_table_load_store.h b/src/observer/table_load/ob_table_load_store.h index 3f0c10e26..80d6d4dee 100644 --- a/src/observer/table_load/ob_table_load_store.h +++ b/src/observer/table_load/ob_table_load_store.h @@ -43,6 +43,7 @@ public: static void abort_ctx(ObTableLoadTableCtx *ctx, bool &is_stopped); int init(); private: + static void cancel_table_ctx(ObTableLoadTableCtx *ctx); static int abort_active_trans(ObTableLoadTableCtx *ctx); // table load ctrl interface public: diff --git a/src/observer/table_load/ob_table_load_store_ctx.cpp b/src/observer/table_load/ob_table_load_store_ctx.cpp index 71e6733d1..732e2e566 100644 --- a/src/observer/table_load/ob_table_load_store_ctx.cpp +++ b/src/observer/table_load/ob_table_load_store_ctx.cpp @@ -14,12 +14,16 @@ #include "observer/table_load/ob_table_load_store_ctx.h" #include "observer/table_load/ob_table_load_error_row_handler.h" +#include "observer/table_load/ob_table_load_data_row_handler.h" #include "observer/table_load/ob_table_load_merger.h" #include "observer/table_load/ob_table_load_store_trans.h" #include "observer/table_load/ob_table_load_table_ctx.h" #include "observer/table_load/ob_table_load_task_scheduler.h" #include "observer/table_load/ob_table_load_trans_store.h" #include "observer/table_load/ob_table_load_utils.h" +#include "observer/table_load/ob_table_load_store_table_ctx.h" +#include "observer/table_load/ob_table_load_merger_manager.h" +#include "observer/table_load/ob_table_load_open_insert_table_ctx_manager.h" #include "share/ob_autoincrement_service.h" #include "share/sequence/ob_sequence_cache.h" #include "sql/engine/cmd/ob_load_data_utils.h" @@ -47,15 +51,12 @@ ObTableLoadStoreCtx::ObTableLoadStoreCtx(ObTableLoadTableCtx *ctx) : ctx_(ctx), allocator_("TLD_StoreCtx"), task_scheduler_(nullptr), - merger_(nullptr), - insert_table_ctx_(nullptr), - next_tablet_idx_(0), - opened_insert_tablet_count_(0), - is_multiple_mode_(false), - is_fast_heap_table_(false), + error_row_handler_(nullptr), + data_store_table_ctx_(nullptr), + merger_manager_(nullptr), + open_insert_tablet_ctx_manager_(nullptr), px_writer_count_(0), tmp_file_mgr_(nullptr), - error_row_handler_(nullptr), sequence_schema_(&allocator_), next_session_id_(0), pre_sorter_(nullptr), @@ -67,8 +68,6 @@ ObTableLoadStoreCtx::ObTableLoadStoreCtx(ObTableLoadTableCtx *ctx) is_inited_(false) { allocator_.set_tenant_id(MTL_ID()); - ls_partition_ids_.set_tenant_id(MTL_ID()); - target_ls_partition_ids_.set_tenant_id(MTL_ID()); committed_trans_store_array_.set_tenant_id(MTL_ID()); } @@ -77,6 +76,68 @@ ObTableLoadStoreCtx::~ObTableLoadStoreCtx() destroy(); } +int ObTableLoadStoreCtx::init_store_table_ctxs( + const table::ObTableLoadArray &partition_id_array, + const table::ObTableLoadArray &target_partition_id_array) +{ + int ret = OB_SUCCESS; + ObSchemaGetterGuard schema_guard; + const share::schema::ObTableSchema *data_table_schema = nullptr; + if (OB_FAIL(ObTableLoadSchema::get_schema_guard(ctx_->param_.tenant_id_, schema_guard))) { + LOG_WARN("fail to get schema guard", KR(ret)); + } else if (OB_FAIL(ObTableLoadSchema::get_table_schema( + schema_guard, ctx_->param_.tenant_id_, ctx_->param_.table_id_, data_table_schema))) { + LOG_WARN("fail to get table shema of main table", KR(ret)); + } else if (OB_ISNULL(data_table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("data table schema is null", KR(ret)); + } else { + if (ObDirectLoadMethod::is_incremental(ctx_->param_.method_)) { + const ObIArray &simple_index_infos = + data_table_schema->get_simple_index_infos(); + for (int64_t i = 0; OB_SUCC(ret) && i < simple_index_infos.count(); i++) { + ObTableLoadStoreTableCtx *index_store_table_ctx = nullptr; + if (OB_ISNULL(index_store_table_ctx = OB_NEWx(ObTableLoadStoreTableCtx, (&allocator_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObTableLoadStoreTableCtx", KR(ret)); + } else if (OB_FAIL(index_store_table_ctx->init(simple_index_infos.at(i).table_id_, true, + this, partition_id_array, + target_partition_id_array))) { + LOG_WARN("fail to init ObTableLoadStoreTableCtx", KR(ret)); + } else if (OB_FAIL(index_store_table_ctx_map_.set_refactored( + simple_index_infos.at(i).table_id_, index_store_table_ctx))) { + LOG_WARN("fail to set index_insert_table_ctx", KR(ret)); + } + if (OB_FAIL(ret)) { + if (OB_NOT_NULL(index_store_table_ctx)) { + index_store_table_ctx->~ObTableLoadStoreTableCtx(); + allocator_.free(index_store_table_ctx); + index_store_table_ctx = nullptr; + } + } + } + } + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(data_store_table_ctx_ = + OB_NEWx(ObTableLoadStoreTableCtx, (&allocator_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObTableLoadStoreTableCtx", KR(ret)); + } else if (OB_FAIL(data_store_table_ctx_->init(ctx_->ddl_param_.dest_table_id_, false, this, + partition_id_array, + target_partition_id_array))) { + LOG_WARN("fail to init ObTableLoadStoreTableCtx", KR(ret)); + } + if (OB_FAIL(ret)) { + if (OB_NOT_NULL(data_store_table_ctx_)) { + data_store_table_ctx_->~ObTableLoadStoreTableCtx(); + allocator_.free(data_store_table_ctx_); + data_store_table_ctx_ = nullptr; + } + } + } + return ret; +} + int ObTableLoadStoreCtx::init( const ObTableLoadArray &partition_id_array, const ObTableLoadArray &target_partition_id_array) @@ -88,221 +149,114 @@ int ObTableLoadStoreCtx::init( } else if (OB_UNLIKELY(partition_id_array.empty() || target_partition_id_array.empty())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", KR(ret), K(partition_id_array), K(target_partition_id_array)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < partition_id_array.count(); ++i) { - const ObLSID &ls_id = partition_id_array[i].ls_id_; - const ObTableLoadPartitionId &part_tablet_id = partition_id_array[i].part_tablet_id_; - if (OB_FAIL(ls_partition_ids_.push_back(ObTableLoadLSIdAndPartitionId(ls_id, part_tablet_id)))) { - LOG_WARN("fail to push back ls tablet id", KR(ret)); - } - } - for (int64_t i = 0; OB_SUCC(ret) && i < target_partition_id_array.count(); ++i) { - const ObLSID &ls_id = target_partition_id_array[i].ls_id_; - const ObTableLoadPartitionId &tablet_id = target_partition_id_array[i].part_tablet_id_; - if (OB_FAIL(target_ls_partition_ids_.push_back(ObTableLoadLSIdAndPartitionId(ls_id, tablet_id)))) { - LOG_WARN("fail to push back ls tablet id", KR(ret)); - } - } - if (OB_SUCC(ret)) { - table_data_desc_.rowkey_column_num_ = - (!ctx_->schema_.is_heap_table_ ? ctx_->schema_.rowkey_column_count_ : 0); - table_data_desc_.column_count_ = - (!ctx_->schema_.is_heap_table_ ? ctx_->schema_.store_column_count_ - : ctx_->schema_.store_column_count_ - 1); - - table_data_desc_.sstable_index_block_size_ = ObDirectLoadSSTableIndexBlock::DEFAULT_INDEX_BLOCK_SIZE; - table_data_desc_.sstable_data_block_size_ = ObDirectLoadSSTableDataBlock::DEFAULT_DATA_BLOCK_SIZE; - if (!GCTX.is_shared_storage_mode()) { - table_data_desc_.external_data_block_size_ = ObDirectLoadDataBlock::SN_DEFAULT_DATA_BLOCK_SIZE; - } else { - table_data_desc_.external_data_block_size_ = ObDirectLoadDataBlock::SS_DEFAULT_DATA_BLOCK_SIZE; - } - - table_data_desc_.extra_buf_size_ = ObDirectLoadTableDataDesc::DEFAULT_EXTRA_BUF_SIZE; - table_data_desc_.compressor_type_ = ctx_->param_.compressor_type_; - table_data_desc_.is_heap_table_ = ctx_->schema_.is_heap_table_; - table_data_desc_.session_count_ = ctx_->param_.session_count_; - table_data_desc_.exe_mode_ = ctx_->param_.exe_mode_; - - int64_t wa_mem_limit = 0; - if (table_data_desc_.exe_mode_ == ObTableLoadExeMode::MAX_TYPE) { - if (OB_FAIL(ObTableLoadService::get_memory_limit(wa_mem_limit))) { - LOG_WARN("failed to get work area memory limit", KR(ret), K(ctx_->param_.tenant_id_)); - } else if (wa_mem_limit < ObDirectLoadMemContext::MIN_MEM_LIMIT) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("wa_mem_limit is too small", KR(ret), K(wa_mem_limit)); - } else { - table_data_desc_.merge_count_per_round_ = min(wa_mem_limit / table_data_desc_.sstable_data_block_size_ / ctx_->param_.session_count_, - ObDirectLoadSSTableScanMerge::MAX_SSTABLE_COUNT); - table_data_desc_.max_mem_chunk_count_ = 128; - int64_t mem_chunk_size = wa_mem_limit / table_data_desc_.max_mem_chunk_count_; - if (mem_chunk_size <= ObDirectLoadExternalMultiPartitionRowChunk::MIN_MEMORY_LIMIT) { - mem_chunk_size = ObDirectLoadExternalMultiPartitionRowChunk::MIN_MEMORY_LIMIT; - table_data_desc_.max_mem_chunk_count_ = wa_mem_limit / mem_chunk_size; - } - table_data_desc_.mem_chunk_size_ = mem_chunk_size; - table_data_desc_.heap_table_mem_chunk_size_ = wa_mem_limit / ctx_->param_.session_count_; - } - if (OB_SUCC(ret)) { - if (table_data_desc_.is_heap_table_) { - int64_t bucket_cnt = wa_mem_limit / (ctx_->param_.session_count_ * MACRO_BLOCK_WRITER_MEM_SIZE); - if ((ls_partition_ids_.count() <= bucket_cnt) || !ctx_->param_.need_sort_) { - is_fast_heap_table_ = true; - } else { - is_multiple_mode_ = true; - } - } else { - int64_t bucket_cnt = wa_mem_limit / ctx_->param_.session_count_ / - (table_data_desc_.sstable_index_block_size_ + table_data_desc_.sstable_data_block_size_); - is_multiple_mode_ = ctx_->param_.need_sort_ || ls_partition_ids_.count() > bucket_cnt; - } - } - } else { - wa_mem_limit = ctx_->param_.avail_memory_; - if (ctx_->param_.exe_mode_ == ObTableLoadExeMode::FAST_HEAP_TABLE || - ctx_->param_.exe_mode_ == ObTableLoadExeMode::GENERAL_TABLE_COMPACT) { - is_fast_heap_table_ = (ctx_->param_.exe_mode_ == ObTableLoadExeMode::FAST_HEAP_TABLE); - } else { - is_multiple_mode_ = true; - } - table_data_desc_.merge_count_per_round_ = min(wa_mem_limit / table_data_desc_.sstable_data_block_size_ / ctx_->param_.session_count_, - ObDirectLoadSSTableScanMerge::MAX_SSTABLE_COUNT); - table_data_desc_.max_mem_chunk_count_ = wa_mem_limit / ObDirectLoadExternalMultiPartitionRowChunk::MIN_MEMORY_LIMIT; - } - if (OB_SUCC(ret)) { - lob_id_table_data_desc_ = table_data_desc_; - lob_id_table_data_desc_.rowkey_column_num_ = 1; - lob_id_table_data_desc_.column_count_ = 1; - lob_id_table_data_desc_.is_heap_table_ = false; - } - } - if (OB_FAIL(ret)) { - } - // init trans_param_ - else if (ObDirectLoadMethod::is_incremental(ctx_->param_.method_) && OB_FAIL(init_trans_param())) { - LOG_WARN("fail to init trans param", KR(ret)); - } - // init trans_allocator_ - else if (OB_FAIL(trans_allocator_.init("TLD_STransPool", ctx_->param_.tenant_id_))) { - LOG_WARN("fail to init trans allocator", KR(ret)); - } - // init trans_map_ - else if (OB_FAIL(trans_map_.create(1024, "TLD_STransMap", "TLD_STransMap", - ctx_->param_.tenant_id_))) { - LOG_WARN("fail to create trans map", KR(ret)); - } - // init trans_ctx_map_ - else if (OB_FAIL(trans_ctx_map_.create(1024, "TLD_TCtxMap", "TLD_TCtxMap", - ctx_->param_.tenant_id_))) { - LOG_WARN("fail to create trans ctx map", KR(ret)); - } - // init segment_trans_ctx_map_ - else if (OB_FAIL(segment_ctx_map_.init("TLD_SegCtxMap", ctx_->param_.tenant_id_))) { - LOG_WARN("fail to init segment ctx map", KR(ret)); - } - // 初始化task_scheduler_ - else if (OB_ISNULL(task_scheduler_ = OB_NEWx(ObTableLoadTaskThreadPoolScheduler, (&allocator_), - ctx_->param_.session_count_, ctx_->param_.table_id_, "Store"))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to new ObTableLoadTaskThreadPoolScheduler", KR(ret)); - } else if (OB_FAIL(task_scheduler_->init())) { - LOG_WARN("fail to init task scheduler", KR(ret)); - } else if (OB_FAIL(task_scheduler_->start())) { - LOG_WARN("fail to start task scheduler", KR(ret)); - } - // 初始化merger_ - else if (OB_ISNULL(merger_ = OB_NEWx(ObTableLoadMerger, (&allocator_), this))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to new ObTableLoadMerger", KR(ret)); - } else if (OB_FAIL(merger_->init())) { - LOG_WARN("fail to init merger", KR(ret)); - } - // init insert_table_ctx_ - if (OB_SUCC(ret)) { - ObDirectLoadInsertTableParam insert_table_param; - insert_table_param.table_id_ = ctx_->ddl_param_.dest_table_id_; - insert_table_param.schema_version_ = ctx_->ddl_param_.schema_version_; - insert_table_param.snapshot_version_ = ctx_->ddl_param_.snapshot_version_; - insert_table_param.ddl_task_id_ = ctx_->ddl_param_.task_id_; - insert_table_param.data_version_ = ctx_->ddl_param_.data_version_; - insert_table_param.parallel_ = ctx_->param_.session_count_; - insert_table_param.reserved_parallel_ = is_fast_heap_table_ ? ctx_->param_.session_count_ : 0; - insert_table_param.rowkey_column_count_ = ctx_->schema_.rowkey_column_count_; - insert_table_param.column_count_ = ctx_->schema_.store_column_count_; - insert_table_param.lob_column_count_ = ctx_->schema_.lob_column_idxs_.count(); - insert_table_param.is_partitioned_table_ = ctx_->schema_.is_partitioned_table_; - insert_table_param.is_heap_table_ = ctx_->schema_.is_heap_table_; - insert_table_param.is_column_store_ = ctx_->schema_.is_column_store_; - insert_table_param.online_opt_stat_gather_ = ctx_->param_.online_opt_stat_gather_; - insert_table_param.is_incremental_ = ObDirectLoadMethod::is_incremental(ctx_->param_.method_); - insert_table_param.trans_param_ = trans_param_; - insert_table_param.datum_utils_ = &(ctx_->schema_.datum_utils_); - insert_table_param.col_descs_ = &(ctx_->schema_.column_descs_); - insert_table_param.cmp_funcs_ = &(ctx_->schema_.cmp_funcs_); - insert_table_param.online_sample_percent_ = ctx_->param_.online_sample_percent_; - if (OB_ISNULL(insert_table_ctx_ = - OB_NEWx(ObDirectLoadInsertTableContext, (&allocator_)))) { + } + // init trans_param_ + else if (ObDirectLoadMethod::is_incremental(ctx_->param_.method_) && + OB_FAIL(init_trans_param())) { + LOG_WARN("fail to init trans param", KR(ret)); + } + // init trans_allocator_ + else if (OB_FAIL(trans_allocator_.init("TLD_STransPool", ctx_->param_.tenant_id_))) { + LOG_WARN("fail to init trans allocator", KR(ret)); + } + // init trans_map_ + else if (OB_FAIL( + trans_map_.create(1024, "TLD_STransMap", "TLD_STransMap", ctx_->param_.tenant_id_))) { + LOG_WARN("fail to create trans map", KR(ret)); + } + // init trans_ctx_map_ + else if (OB_FAIL( + trans_ctx_map_.create(1024, "TLD_TCtxMap", "TLD_TCtxMap", ctx_->param_.tenant_id_))) { + LOG_WARN("fail to create trans ctx map", KR(ret)); + } + // init segment_trans_ctx_map_ + else if (OB_FAIL(segment_ctx_map_.init("TLD_SegCtxMap", ctx_->param_.tenant_id_))) { + LOG_WARN("fail to init segment ctx map", KR(ret)); + } else if (OB_FAIL(index_store_table_ctx_map_.create(1024, "TLD_IdxStCtxMap", "TLD_IdxStCtxMap", + ctx_->param_.tenant_id_))) { + LOG_WARN("fail to create index_store_table_ctx_map", KR(ret)); + } + // 初始化task_scheduler_ + else if (OB_ISNULL(task_scheduler_ = + OB_NEWx(ObTableLoadTaskThreadPoolScheduler, (&allocator_), + ctx_->param_.session_count_, ctx_->param_.table_id_, "Store"))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObTableLoadTaskThreadPoolScheduler", KR(ret)); + } else if (OB_FAIL(task_scheduler_->init())) { + LOG_WARN("fail to init task scheduler", KR(ret)); + } else if (OB_FAIL(task_scheduler_->start())) { + LOG_WARN("fail to start task scheduler", KR(ret)); + } else if (OB_ISNULL(error_row_handler_ = OB_NEWx(ObTableLoadErrorRowHandler, (&allocator_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObTableLoadErrorRowHandler", KR(ret)); + } else if (OB_FAIL(error_row_handler_->init(ctx_->param_, result_info_, ctx_->job_stat_))) { + LOG_WARN("fail to init error row handler", KR(ret)); + } else if (OB_FAIL(init_store_table_ctxs(partition_id_array, target_partition_id_array))) { + LOG_WARN("fail to init store table ctxs", KR(ret)); + } else if (OB_ISNULL(merger_manager_ = OB_NEWx(ObTableLoadMergerManager, (&allocator_), this))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObTableLoadMerger", KR(ret)); + } else if (OB_FAIL(merger_manager_->init())) { + LOG_WARN("fail to init merger", KR(ret)); + } else if (OB_ISNULL(open_insert_tablet_ctx_manager_ = + OB_NEWx(ObTableLoadOpenInsertTableCtxManager, (&allocator_), this))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObTableLoadOpenInsertTableCtxManager", KR(ret)); + } else if (OB_FAIL(open_insert_tablet_ctx_manager_->init())) { + LOG_WARN("fail to init open insert tablet ctx manager", KR(ret)); + } else if (OB_ISNULL(error_row_handler_ = OB_NEWx(ObTableLoadErrorRowHandler, (&allocator_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObTableLoadErrorRowHandler", KR(ret)); + } else if (OB_FAIL(error_row_handler_->init(ctx_->param_, result_info_, ctx_->job_stat_))) { + LOG_WARN("fail to init error row handler", KR(ret)); + } + if (OB_FAIL(ret)) { + } + // init tmp_file_mgr_ + else if (OB_ISNULL(tmp_file_mgr_ = OB_NEWx(ObDirectLoadTmpFileManager, (&allocator_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObDirectLoadTmpFileManager", KR(ret)); + } else if (OB_FAIL(tmp_file_mgr_->init(ctx_->param_.tenant_id_))) { + LOG_WARN("fail to init tmp file manager", KR(ret)); + } + // init session_ctx_array_ + else if (OB_FAIL(init_session_ctx_array())) { + LOG_WARN("fail to init session ctx array", KR(ret)); + } + // init sequence_cache_ and sequence_schema_ + else if (data_store_table_ctx_->schema_->has_identity_column_ && OB_FAIL(init_sequence())) { + LOG_WARN("fail to init sequence", KR(ret)); + } + // init enable_pre_sort_ and pre_sorter_ + if (OB_SUCC(ret)) { + enable_pre_sort_ = (data_store_table_ctx_->is_multiple_mode_ + && !data_store_table_ctx_->table_data_desc_.is_heap_table_); + if (enable_pre_sort_) { + if (OB_ISNULL(pre_sorter_ = OB_NEWx(ObTableLoadPreSorter, + (&allocator_), + this->ctx_, + this))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to new ObDirectLoadInsertTableContext", KR(ret)); - } else if (OB_FAIL(insert_table_ctx_->init(insert_table_param, ls_partition_ids_, target_ls_partition_ids_))) { - LOG_WARN("fail to init insert table ctx", KR(ret)); + LOG_WARN("fail to allocate TableLoadPreSorter", KR(ret)); + } else if (OB_FAIL(pre_sorter_->init())) { + LOG_WARN("fail to init pre sorter", KR(ret)); + } else if (OB_FAIL(pre_sorter_->start())) { + LOG_WARN("fail to start pre_sorter", KR(ret)); } } - if (OB_FAIL(ret)) { - } - // init tmp_file_mgr_ - else if (OB_ISNULL(tmp_file_mgr_ = OB_NEWx(ObDirectLoadTmpFileManager, (&allocator_)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to new ObDirectLoadTmpFileManager", KR(ret)); - } else if (OB_FAIL(tmp_file_mgr_->init(ctx_->param_.tenant_id_))) { - LOG_WARN("fail to init tmp file manager", KR(ret)); - } - // init error_row_handler_ - else if (OB_ISNULL(error_row_handler_ = - OB_NEWx(ObTableLoadErrorRowHandler, (&allocator_)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to new ObTableLoadErrorRowHandler", KR(ret)); - } else if (OB_FAIL(error_row_handler_->init(ctx_->param_, result_info_, ctx_->job_stat_))) { - LOG_WARN("fail to init error row handler", KR(ret)); - } - // init session_ctx_array_ - else if (OB_FAIL(init_session_ctx_array())) { - LOG_WARN("fail to init session ctx array", KR(ret)); - } - // init sequence_cache_ and sequence_schema_ - else if (ctx_->schema_.has_identity_column_ && OB_FAIL(init_sequence())) { - LOG_WARN("fail to init sequence", KR(ret)); - } - // init enable_pre_sort_ and pre_sorter_ - if (OB_SUCC(ret)) { - enable_pre_sort_ = (is_multiple_mode_ && !table_data_desc_.is_heap_table_); - if (enable_pre_sort_) { - if (OB_ISNULL(pre_sorter_ = OB_NEWx(ObTableLoadPreSorter, - (&allocator_), - this->ctx_, - this))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to allocate TableLoadPreSorter", KR(ret)); - } else if (OB_FAIL(pre_sorter_->init())) { - LOG_WARN("fail to init pre sorter", KR(ret)); - } else if (OB_FAIL(pre_sorter_->start())) { - LOG_WARN("fail to start pre_sorter", KR(ret)); - } - } - } - if (OB_SUCC(ret)) { - is_inited_ = true; - } else { - destroy(); - } + } + if (OB_SUCC(ret)) { + is_inited_ = true; + } else { + destroy(); } return ret; } void ObTableLoadStoreCtx::stop() { - if (nullptr != merger_) { - merger_->stop(); + if (nullptr != merger_manager_) { + merger_manager_->stop(); } if (nullptr != pre_sorter_) { pre_sorter_->stop(); @@ -316,10 +270,33 @@ void ObTableLoadStoreCtx::stop() void ObTableLoadStoreCtx::destroy() { - if (nullptr != merger_) { - merger_->~ObTableLoadMerger(); - allocator_.free(merger_); - merger_ = nullptr; + if (OB_NOT_NULL(error_row_handler_)) { + error_row_handler_->~ObTableLoadErrorRowHandler(); + allocator_.free(error_row_handler_); + error_row_handler_ = nullptr; + } + FOREACH(it, index_store_table_ctx_map_) { + ObTableLoadStoreTableCtx* index_store_table_ctx = it->second; + if (OB_NOT_NULL(index_store_table_ctx)) { + index_store_table_ctx->~ObTableLoadStoreTableCtx(); + allocator_.free(index_store_table_ctx); + } + } + index_store_table_ctx_map_.destroy(); + if (OB_NOT_NULL(data_store_table_ctx_)) { + data_store_table_ctx_->~ObTableLoadStoreTableCtx(); + allocator_.free(data_store_table_ctx_); + data_store_table_ctx_ = nullptr; + } + if (OB_NOT_NULL(merger_manager_)) { + merger_manager_->~ObTableLoadMergerManager(); + allocator_.free(merger_manager_); + merger_manager_ = nullptr; + } + if (OB_NOT_NULL(open_insert_tablet_ctx_manager_)) { + open_insert_tablet_ctx_manager_->~ObTableLoadOpenInsertTableCtxManager(); + allocator_.free(open_insert_tablet_ctx_manager_); + open_insert_tablet_ctx_manager_ = nullptr; } if (nullptr != task_scheduler_) { task_scheduler_->stop(); @@ -347,26 +324,16 @@ void ObTableLoadStoreCtx::destroy() ctx_->free_trans_ctx(trans_ctx); } trans_ctx_map_.reuse(); - if (nullptr != insert_table_ctx_) { - insert_table_ctx_->~ObDirectLoadInsertTableContext(); - allocator_.free(insert_table_ctx_); - insert_table_ctx_ = nullptr; - } if (nullptr != pre_sorter_) { pre_sorter_->~ObTableLoadPreSorter(); allocator_.free(pre_sorter_); pre_sorter_ = nullptr; } - if (nullptr != tmp_file_mgr_) { + if (OB_NOT_NULL(tmp_file_mgr_)) { tmp_file_mgr_->~ObDirectLoadTmpFileManager(); allocator_.free(tmp_file_mgr_); tmp_file_mgr_ = nullptr; } - if (nullptr != error_row_handler_) { - error_row_handler_->~ObTableLoadErrorRowHandler(); - allocator_.free(error_row_handler_); - error_row_handler_ = nullptr; - } } int ObTableLoadStoreCtx::advance_status(ObTableLoadStatusType status) @@ -446,6 +413,7 @@ int ObTableLoadStoreCtx::check_status(ObTableLoadStatusType status) const } else { ret = OB_STATE_NOT_MATCH; } + LOG_WARN("unexpected status", KR(ret), K(status), K(status_)); } return ret; } @@ -621,15 +589,15 @@ int ObTableLoadStoreCtx::init_session_ctx_array() if (OB_ISNULL(buf = allocator_.alloc(sizeof(SessionContext) * ctx_->param_.write_session_count_))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to allocate memory", KR(ret)); - } else if (ctx_->schema_.has_autoinc_column_ && OB_FAIL(generate_autoinc_params(autoinc_param))) { + } else if (data_store_table_ctx_->schema_->has_autoinc_column_ && OB_FAIL(generate_autoinc_params(autoinc_param))) { LOG_WARN("fail to init auto increment param", KR(ret)); } else { session_ctx_array_ = new (buf) SessionContext[ctx_->param_.write_session_count_]; for (int64_t i = 0; OB_SUCC(ret) && i < ctx_->param_.write_session_count_; ++i) { SessionContext *session_ctx = session_ctx_array_ + i; session_ctx->autoinc_param_ = autoinc_param; - if (!is_fast_heap_table_) { - session_ctx->extra_buf_size_ = table_data_desc_.extra_buf_size_; + if (!data_store_table_ctx_->is_fast_heap_table_) { + session_ctx->extra_buf_size_ = data_store_table_ctx_->table_data_desc_.extra_buf_size_; if (OB_ISNULL(session_ctx->extra_buf_ = static_cast(allocator_.alloc(session_ctx->extra_buf_size_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -1002,26 +970,14 @@ void ObTableLoadStoreCtx::clear_committed_trans_stores() committed_trans_store_array_.reset(); } -int ObTableLoadStoreCtx::get_next_insert_tablet_ctx(ObTabletID &tablet_id) +int ObTableLoadStoreCtx::get_next_insert_tablet_ctx(ObDirectLoadInsertTabletContext *&tablet_ctx) { - int ret = OB_SUCCESS; - ObMutexGuard guard(op_lock_); - if (next_tablet_idx_ < ls_partition_ids_.count()) { - tablet_id = ls_partition_ids_[next_tablet_idx_++].part_tablet_id_.tablet_id_; - } else { - ret = OB_ITER_END; - } - - return ret; + return open_insert_tablet_ctx_manager_->get_next_insert_tablet_ctx(tablet_ctx); } void ObTableLoadStoreCtx::handle_open_insert_tablet_ctx_finish(bool &is_finish) { - is_finish = false; - ObMutexGuard guard(op_lock_); - if (++opened_insert_tablet_count_ == ls_partition_ids_.count()) { - is_finish = true; - } + open_insert_tablet_ctx_manager_->handle_open_insert_tablet_ctx_finish(is_finish); } } // namespace observer diff --git a/src/observer/table_load/ob_table_load_store_ctx.h b/src/observer/table_load/ob_table_load_store_ctx.h index a4222e228..e402c3415 100644 --- a/src/observer/table_load/ob_table_load_store_ctx.h +++ b/src/observer/table_load/ob_table_load_store_ctx.h @@ -46,11 +46,13 @@ class ObTableLoadTransStore; class ObITableLoadTaskScheduler; class ObTableLoadMerger; class ObTableLoadErrorRowHandler; +class ObTableLoadStoreTableCtx; +class ObTableLoadMergerManager; +class ObTableLoadOpenInsertTableCtxManager; class ObTableLoadPreSorter; class ObTableLoadStoreCtx { -static const int64_t MACRO_BLOCK_WRITER_MEM_SIZE = 10 * 1024LL * 1024LL; public: ObTableLoadStoreCtx(ObTableLoadTableCtx *ctx); ~ObTableLoadStoreCtx(); @@ -135,32 +137,29 @@ private: int alloc_trans_ctx(const table::ObTableLoadTransId &trans_id, ObTableLoadTransCtx *&trans_ctx); int alloc_trans(const table::ObTableLoadTransId &trans_id, ObTableLoadStoreTrans *&trans); int init_session_ctx_array(); + int init_store_table_ctxs( + const table::ObTableLoadArray &partition_id_array, + const table::ObTableLoadArray &target_partition_id_array); int init_trans_param(); int generate_autoinc_params(share::AutoincParam &autoinc_param); int init_sequence(); public: int commit_autoinc_value(); - int get_next_insert_tablet_ctx(ObTabletID &tablet_id); + int get_next_insert_tablet_ctx(ObDirectLoadInsertTabletContext *&tablet_ctx); void handle_open_insert_tablet_ctx_finish(bool &is_finish); public: - ObTableLoadTableCtx * const ctx_; + ObTableLoadTableCtx * ctx_; common::ObArenaAllocator allocator_; - common::ObArray ls_partition_ids_; - common::ObArray target_ls_partition_ids_; - storage::ObDirectLoadTableDataDesc table_data_desc_; - storage::ObDirectLoadTableDataDesc lob_id_table_data_desc_; storage::ObDirectLoadTransParam trans_param_; table::ObTableLoadResultInfo result_info_; ObITableLoadTaskScheduler *task_scheduler_; - ObTableLoadMerger *merger_; - storage::ObDirectLoadInsertTableContext *insert_table_ctx_; - int64_t next_tablet_idx_; - int64_t opened_insert_tablet_count_; - bool is_multiple_mode_; - bool is_fast_heap_table_; + ObTableLoadErrorRowHandler *error_row_handler_; + hash::ObHashMap index_store_table_ctx_map_; + ObTableLoadStoreTableCtx *data_store_table_ctx_; + ObTableLoadMergerManager *merger_manager_; + ObTableLoadOpenInsertTableCtxManager *open_insert_tablet_ctx_manager_; int64_t px_writer_count_; storage::ObDirectLoadTmpFileManager *tmp_file_mgr_; - ObTableLoadErrorRowHandler *error_row_handler_; share::schema::ObSequenceSchema sequence_schema_; uint64_t next_session_id_ CACHE_ALIGNED; struct SessionContext diff --git a/src/observer/table_load/ob_table_load_store_table_ctx.cpp b/src/observer/table_load/ob_table_load_store_table_ctx.cpp new file mode 100644 index 000000000..ff95e017c --- /dev/null +++ b/src/observer/table_load/ob_table_load_store_table_ctx.cpp @@ -0,0 +1,473 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#define USING_LOG_PREFIX SERVER +#include "observer/table_load/ob_table_load_store_table_ctx.h" +#include "observer/table_load/ob_table_load_store_ctx.h" +#include "observer/table_load/ob_table_load_merger.h" +#include "observer/table_load/ob_table_load_data_row_handler.h" +#include "observer/table_load/ob_table_load_index_row_handler.h" +#include "storage/direct_load/ob_direct_load_external_multi_partition_table.h" +#include "observer/table_load/ob_table_load_index_table_projector.h" +#include "storage/direct_load/ob_direct_load_sstable_data_block.h" +#include "storage/direct_load/ob_direct_load_mem_context.h" +#include "storage/direct_load/ob_direct_load_i_table.h" +#include "storage/direct_load/ob_direct_load_dml_row_handler.h" + +namespace oceanbase +{ +namespace observer +{ +ObTableLoadStoreTableCtx::ObTableLoadStoreTableCtx() + : allocator_("TLD_STCtx"), + table_id_(OB_INVALID_ID), + is_index_table_(false), + store_ctx_(NULL), + need_sort_(false), + schema_(NULL), + project_(NULL), + is_fast_heap_table_(false), + is_multiple_mode_(false), + insert_table_ctx_(NULL), + row_handler_(NULL), + merger_(NULL), + is_inited_(false) +{ +} + +ObTableLoadStoreTableCtx::~ObTableLoadStoreTableCtx() +{ + if (OB_NOT_NULL(schema_)) { + schema_->~ObTableLoadSchema(); + allocator_.free(schema_); + schema_ = NULL; + } + if (OB_NOT_NULL(project_)) { + project_->~ObTableLoadIndexTableProjector(); + allocator_.free(project_); + project_ = NULL; + } + if (OB_NOT_NULL(insert_table_ctx_)) { + insert_table_ctx_->~ObDirectLoadInsertTableContext(); + allocator_.free(insert_table_ctx_); + insert_table_ctx_ = NULL; + } + if (OB_NOT_NULL(row_handler_)) { + row_handler_->~ObDirectLoadDMLRowHandler(); + allocator_.free(row_handler_); + row_handler_ = NULL; + } + if (OB_NOT_NULL(merger_)) { + merger_->~ObTableLoadMerger(); + allocator_.free(merger_); + merger_ = NULL; + } + FOREACH(iter, index_table_builder_map_) { + ObIDirectLoadPartitionTableBuilder *table_builder = iter->second; + table_builder->~ObIDirectLoadPartitionTableBuilder(); + allocator_.free(table_builder); + } + index_table_builder_map_.destroy(); +} + +int ObTableLoadStoreTableCtx::init_table_load_schema() +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(schema_ = OB_NEWx(ObTableLoadSchema, (&allocator_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObTableLoadSchema", KR(ret)); + } else if (OB_FAIL(schema_->init(store_ctx_->ctx_->param_.tenant_id_, table_id_))) { + LOG_WARN("fail to init schema", KR(ret)); + } + if (OB_FAIL(ret)) { + if (OB_NOT_NULL(schema_)) { + schema_->~ObTableLoadSchema(); + allocator_.free(schema_); + schema_ = NULL; + } + } + return ret; +} + +int ObTableLoadStoreTableCtx::init_index_projector() +{ + int ret = OB_SUCCESS; + if (is_index_table_) { + ObSchemaGetterGuard schema_guard; + const share::schema::ObTableSchema *data_table_schema = nullptr; + const share::schema::ObTableSchema *index_table_schema = nullptr; + if (OB_FAIL( + ObTableLoadSchema::get_schema_guard(store_ctx_->ctx_->param_.tenant_id_, schema_guard))) { + LOG_WARN("fail to get schema guard", KR(ret)); + } else if (OB_FAIL(ObTableLoadSchema::get_table_schema( + schema_guard, store_ctx_->ctx_->param_.tenant_id_, + store_ctx_->ctx_->ddl_param_.dest_table_id_, data_table_schema))) { + LOG_WARN("fail to get table shema of main table", KR(ret)); + } else if (OB_FAIL(ObTableLoadSchema::get_table_schema(schema_guard, + store_ctx_->ctx_->param_.tenant_id_, table_id_, index_table_schema))) { + LOG_WARN("fail to get table shema of index table", KR(ret)); + } + else if (OB_ISNULL(project_ = OB_NEWx(ObTableLoadIndexTableProjector, &allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObTableLoadIndexTableProjector", KR(ret)); + } else if (OB_FAIL(project_->init(data_table_schema, index_table_schema))) { + LOG_WARN("fail to init ObTableLoadIndexTableProjector", KR(ret)); + } + if (OB_FAIL(ret)) { + if (OB_NOT_NULL(project_)) { + project_->~ObTableLoadIndexTableProjector(); + allocator_.free(project_); + project_ = NULL; + } + } + } + return ret; +} + +int ObTableLoadStoreTableCtx::init_ls_partition_ids( + const table::ObTableLoadArray &partition_id_array, + const table::ObTableLoadArray &target_partition_id_array) +{ + int ret = OB_SUCCESS; + if (is_index_table_) { + for (int64_t i = 0; OB_SUCC(ret) && i < partition_id_array.count(); ++i) { + ObTabletID index_tablet_id; + ObObjectID index_part_id; + const ObTabletID &data_tablet_id = partition_id_array[i].part_tablet_id_.tablet_id_; + if (OB_FAIL(project_->get_index_tablet_id_and_part_id_by_data_tablet_id( + data_tablet_id, index_tablet_id, index_part_id))) { + LOG_WARN("fail to get index_tablet_id", KR(ret), K(data_tablet_id)); + } else if (OB_FAIL(ls_partition_ids_.push_back(table::ObTableLoadLSIdAndPartitionId( + partition_id_array[i].ls_id_, + table::ObTableLoadPartitionId(index_part_id, index_tablet_id))))) { + LOG_WARN("fail to push back ls tablet id", KR(ret)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < target_partition_id_array.count(); ++i) { + ObTabletID index_tablet_id; + ObObjectID index_part_id; + const ObTabletID &data_tablet_id = target_partition_id_array[i].part_tablet_id_.tablet_id_; + if (OB_FAIL(project_->get_index_tablet_id_and_part_id_by_data_tablet_id( + data_tablet_id, index_tablet_id, index_part_id))) { + LOG_WARN("fail to get index_tablet_id", KR(ret), K(data_tablet_id)); + } else if (OB_FAIL(target_ls_partition_ids_.push_back(table::ObTableLoadLSIdAndPartitionId( + target_partition_id_array[i].ls_id_, + table::ObTableLoadPartitionId(index_part_id, index_tablet_id))))) { + LOG_WARN("fail to push back ls tablet id", KR(ret)); + } + } + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < partition_id_array.count(); ++i) { + const ObLSID &ls_id = partition_id_array[i].ls_id_; + const table::ObTableLoadPartitionId &part_tablet_id = partition_id_array[i].part_tablet_id_; + if (OB_FAIL(ls_partition_ids_.push_back(table::ObTableLoadLSIdAndPartitionId(ls_id, part_tablet_id)))) { + LOG_WARN("fail to push back ls tablet id", KR(ret)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < target_partition_id_array.count(); ++i) { + const ObLSID &ls_id = target_partition_id_array[i].ls_id_; + const table::ObTableLoadPartitionId &tablet_id = target_partition_id_array[i].part_tablet_id_; + if (OB_FAIL(target_ls_partition_ids_.push_back(table::ObTableLoadLSIdAndPartitionId(ls_id, tablet_id)))) { + LOG_WARN("fail to push back ls tablet id", KR(ret)); + } + } + + } + return ret; +} + +int ObTableLoadStoreTableCtx::init_table_data_desc() +{ + int ret = OB_SUCCESS; + table_data_desc_.rowkey_column_num_ = + (!schema_->is_heap_table_ ? schema_->rowkey_column_count_ : 0); + table_data_desc_.column_count_ = + (!schema_->is_heap_table_ ? schema_->store_column_count_ + : schema_->store_column_count_ - 1); + + table_data_desc_.sstable_index_block_size_ = + ObDirectLoadSSTableIndexBlock::DEFAULT_INDEX_BLOCK_SIZE; + table_data_desc_.sstable_data_block_size_ = ObDirectLoadSSTableDataBlock::DEFAULT_DATA_BLOCK_SIZE; + if (!GCTX.is_shared_storage_mode()) { + table_data_desc_.external_data_block_size_ = ObDirectLoadDataBlock::SN_DEFAULT_DATA_BLOCK_SIZE; + } else { + table_data_desc_.external_data_block_size_ = ObDirectLoadDataBlock::SS_DEFAULT_DATA_BLOCK_SIZE; + } + + table_data_desc_.extra_buf_size_ = ObDirectLoadTableDataDesc::DEFAULT_EXTRA_BUF_SIZE; + table_data_desc_.compressor_type_ = store_ctx_->ctx_->param_.compressor_type_; + table_data_desc_.is_heap_table_ = schema_->is_heap_table_; + table_data_desc_.session_count_ = store_ctx_->ctx_->param_.session_count_; + table_data_desc_.exe_mode_ = store_ctx_->ctx_->param_.exe_mode_; + + int64_t wa_mem_limit = 0; + if (table_data_desc_.exe_mode_ == ObTableLoadExeMode::MAX_TYPE) { + if (OB_FAIL(ObTableLoadService::get_memory_limit(wa_mem_limit))) { + LOG_WARN("failed to get work area memory limit", KR(ret), K(store_ctx_->ctx_->param_.tenant_id_)); + } else if (wa_mem_limit < ObDirectLoadMemContext::MIN_MEM_LIMIT) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("wa_mem_limit is too small", KR(ret), K(wa_mem_limit)); + } else { + table_data_desc_.merge_count_per_round_ = + min(wa_mem_limit / table_data_desc_.sstable_data_block_size_ / store_ctx_->ctx_->param_.session_count_, + ObDirectLoadSSTableScanMerge::MAX_SSTABLE_COUNT); + table_data_desc_.max_mem_chunk_count_ = 128; + int64_t mem_chunk_size = wa_mem_limit / table_data_desc_.max_mem_chunk_count_; + if (mem_chunk_size <= ObDirectLoadExternalMultiPartitionRowChunk::MIN_MEMORY_LIMIT) { + mem_chunk_size = ObDirectLoadExternalMultiPartitionRowChunk::MIN_MEMORY_LIMIT; + table_data_desc_.max_mem_chunk_count_ = wa_mem_limit / mem_chunk_size; + } + table_data_desc_.mem_chunk_size_ = mem_chunk_size; + table_data_desc_.heap_table_mem_chunk_size_ = wa_mem_limit / store_ctx_->ctx_->param_.session_count_; + } + if (OB_SUCC(ret)) { + if (table_data_desc_.is_heap_table_) { + int64_t bucket_cnt = + wa_mem_limit / (store_ctx_->ctx_->param_.session_count_ * MACRO_BLOCK_WRITER_MEM_SIZE); + if ((ls_partition_ids_.count() <= bucket_cnt) || !need_sort_) { + is_fast_heap_table_ = true; + } else { + is_multiple_mode_ = true; + } + } else { + int64_t bucket_cnt = + wa_mem_limit / store_ctx_->ctx_->param_.session_count_ / + (table_data_desc_.sstable_index_block_size_ + table_data_desc_.sstable_data_block_size_); + is_multiple_mode_ = need_sort_ || ls_partition_ids_.count() > bucket_cnt || is_index_table_; + } + } + } else { + wa_mem_limit = store_ctx_->ctx_->param_.avail_memory_; + if (!is_index_table_ && (store_ctx_->ctx_->param_.exe_mode_ == ObTableLoadExeMode::FAST_HEAP_TABLE || + store_ctx_->ctx_->param_.exe_mode_ == ObTableLoadExeMode::GENERAL_TABLE_COMPACT)) { + is_fast_heap_table_ = (store_ctx_->ctx_->param_.exe_mode_ == ObTableLoadExeMode::FAST_HEAP_TABLE); + } else { + is_multiple_mode_ = true; + } + table_data_desc_.merge_count_per_round_ = + min(wa_mem_limit / table_data_desc_.sstable_data_block_size_ / store_ctx_->ctx_->param_.session_count_, + ObDirectLoadSSTableScanMerge::MAX_SSTABLE_COUNT); + table_data_desc_.max_mem_chunk_count_ = + wa_mem_limit / ObDirectLoadExternalMultiPartitionRowChunk::MIN_MEMORY_LIMIT; + } + if (OB_SUCC(ret)) { + lob_id_table_data_desc_ = table_data_desc_; + lob_id_table_data_desc_.rowkey_column_num_ = 1; + lob_id_table_data_desc_.column_count_ = 1; + lob_id_table_data_desc_.is_heap_table_ = false; + } + return ret; +} + +int ObTableLoadStoreTableCtx::init_insert_table_ctx() +{ + int ret = OB_SUCCESS; + ObDirectLoadInsertTableParam insert_table_param; + insert_table_param.table_id_ = table_id_; + insert_table_param.schema_version_ = store_ctx_->ctx_->ddl_param_.schema_version_; + insert_table_param.snapshot_version_ = store_ctx_->ctx_->ddl_param_.snapshot_version_; + insert_table_param.ddl_task_id_ = store_ctx_->ctx_->ddl_param_.task_id_; + insert_table_param.data_version_ = store_ctx_->ctx_->ddl_param_.data_version_; + insert_table_param.parallel_ = store_ctx_->ctx_->param_.session_count_; + insert_table_param.reserved_parallel_ = is_fast_heap_table_ ? store_ctx_->ctx_->param_.session_count_ : 0; + insert_table_param.rowkey_column_count_ = schema_->rowkey_column_count_; + insert_table_param.column_count_ = schema_->store_column_count_; + insert_table_param.lob_column_count_ = schema_->lob_column_idxs_.count(); + insert_table_param.is_partitioned_table_ = schema_->is_partitioned_table_; + insert_table_param.is_heap_table_ = schema_->is_heap_table_; + insert_table_param.is_column_store_ = schema_->is_column_store_; + insert_table_param.online_opt_stat_gather_ = store_ctx_->ctx_->param_.online_opt_stat_gather_; + insert_table_param.is_incremental_ = ObDirectLoadMethod::is_incremental(store_ctx_->ctx_->param_.method_); + insert_table_param.trans_param_ = store_ctx_->trans_param_; + insert_table_param.datum_utils_ = &(schema_->datum_utils_); + insert_table_param.col_descs_ = &(schema_->column_descs_); + insert_table_param.cmp_funcs_ = &(schema_->cmp_funcs_); + insert_table_param.online_sample_percent_ = store_ctx_->ctx_->param_.online_sample_percent_; + if (OB_ISNULL(insert_table_ctx_ = + OB_NEWx(ObDirectLoadInsertTableContext, (&allocator_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObDirectLoadInsertTableContext", KR(ret)); + } else if (OB_FAIL(insert_table_ctx_->init(insert_table_param, ls_partition_ids_, target_ls_partition_ids_))) { + LOG_WARN("fail to init insert table ctx", KR(ret)); + } + if (OB_FAIL(ret)) { + if (OB_NOT_NULL(insert_table_ctx_)) { + insert_table_ctx_->~ObDirectLoadInsertTableContext(); + allocator_.free(insert_table_ctx_); + insert_table_ctx_ = nullptr; + } + } + return ret; +} + +int ObTableLoadStoreTableCtx::init_row_handler() +{ + int ret = OB_SUCCESS; + if (!is_index_table_) { + ObTableLoadDataRowHandler *main_row_handler = nullptr; + if (OB_ISNULL(main_row_handler = OB_NEWx(ObTableLoadDataRowHandler, (&allocator_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObTableLoadDataRowHandler", KR(ret)); + } else if (OB_FAIL(main_row_handler->init(store_ctx_->ctx_->param_, store_ctx_->result_info_, + store_ctx_->error_row_handler_, + &(store_ctx_->index_store_table_ctx_map_)))) { + LOG_WARN("fail to init row handler", KR(ret), KPC(store_ctx_), K(is_index_table), + K(schema_->index_table_count_), KP(project_)); + } else { + row_handler_ = main_row_handler; + } + } else { + ObTableLoadIndexRowHandler *index_row_handler = nullptr; + if (OB_ISNULL(index_row_handler = OB_NEWx(ObTableLoadIndexRowHandler, (&allocator_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObTableLoadIndexRowHandler", KR(ret)); + } else { + row_handler_ = index_row_handler; + } + } + if (OB_FAIL(ret)) { + if (OB_NOT_NULL(row_handler_)) { + row_handler_->~ObDirectLoadDMLRowHandler(); + allocator_.free(row_handler_); + row_handler_ = nullptr; + } + } + return ret; +} + +int ObTableLoadStoreTableCtx::init_merger() +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(merger_ = OB_NEWx(ObTableLoadMerger, (&allocator_), store_ctx_, this))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObTableLoadMerger", KR(ret)); + } else if (OB_FAIL(merger_->init())) { + LOG_WARN("fail to init ObTableLoadMerger", KR(ret)); + } + if (OB_FAIL(ret)) { + if (OB_NOT_NULL(merger_)) { + merger_->~ObTableLoadMerger(); + allocator_.free(merger_); + merger_ = nullptr; + } + } + return ret; +} + +int ObTableLoadStoreTableCtx::init( + ObTableID table_id, bool is_index_table, ObTableLoadStoreCtx *store_ctx, + const table::ObTableLoadArray &partition_id_array, + const table::ObTableLoadArray &target_partition_id_array) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObTableLoadStoreTableCtx has been inited", KR(ret)); + } else if (OB_ISNULL(store_ctx) || OB_INVALID_ID == table_id) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), KP(store_ctx), K(table_id)); + } else if (OB_FAIL(index_table_builder_map_.create(1024, "TLD_IdxBd", "TLD_IdxBd", MTL_ID()))) { + LOG_WARN("fail to create index_table_builder_map_", KR(ret)); + } else { + table_id_ = table_id; + is_index_table_ = is_index_table; + store_ctx_ = store_ctx; + if (is_index_table) { + need_sort_ = true; + } else { + need_sort_ = (ObTableLoadExeMode::MEM_COMPACT == store_ctx_->ctx_->param_.exe_mode_) || + (ObTableLoadExeMode::MULTIPLE_HEAP_TABLE_COMPACT == store_ctx_->ctx_->param_.exe_mode_); + } + if (OB_FAIL(init_table_load_schema())) { + LOG_WARN("fail to init table load schema", KR(ret)); + } else if (OB_FAIL(init_index_projector())) { + LOG_WARN("fail to init index projector", KR(ret)); + } else if (OB_FAIL(init_ls_partition_ids(partition_id_array, target_partition_id_array))) { + LOG_WARN("fail to init ls_partition_ids", KR(ret)); + } else if (OB_FAIL(init_table_data_desc())) { + LOG_WARN("fail to init table data desc", KR(ret)); + } else if (OB_FAIL(init_insert_table_ctx())) { + LOG_WARN("fail to init insert table ctx", KR(ret)); + } else if (OB_FAIL(init_row_handler())) { + LOG_WARN("fail to init row handler", KR(ret)); + } else if (OB_FAIL(init_merger())) { + LOG_WARN("fail to init merger", KR(ret)); + } else { + is_inited_ = true; + } + } + return ret; +} + +int ObTableLoadStoreTableCtx::close_index_table_builder() +{ + int ret = OB_SUCCESS; + FOREACH_X(item, index_table_builder_map_, OB_SUCC(ret)) { + if (item->second != nullptr) { + if (OB_FAIL(item->second->close())) { + LOG_WARN("fail to close table", KR(ret)); + } + } + } + return ret; +} + +int ObTableLoadStoreTableCtx::get_index_table_builder(ObIDirectLoadPartitionTableBuilder *&table_builder) +{ + int ret = OB_SUCCESS; + table_builder = nullptr; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), KP(this)); + } else { + const int64_t part_id = get_tid_cache(); + if (OB_FAIL(index_table_builder_map_.get_refactored(get_tid_cache(), table_builder))) { + if (OB_UNLIKELY(OB_HASH_NOT_EXIST != ret)) { + LOG_WARN("fail to get table builder", KR(ret), K(get_tid_cache())); + } else { + ret = OB_SUCCESS; + ObDirectLoadExternalMultiPartitionTableBuildParam builder_param; + builder_param.table_data_desc_ = table_data_desc_; + builder_param.datum_utils_ = &(schema_->datum_utils_); + builder_param.file_mgr_ = store_ctx_->tmp_file_mgr_; + builder_param.extra_buf_ = reinterpret_cast(1); // unuse, delete in future + builder_param.extra_buf_size_ = 4096; // unuse, delete in future + { + ObMutexGuard guard(mutex_); + // use mutext to avoid thread unsafe allocator; + ObDirectLoadExternalMultiPartitionTableBuilder *new_builder = nullptr; + if (OB_ISNULL(new_builder = OB_NEWx( + ObDirectLoadExternalMultiPartitionTableBuilder, &allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObDirectLoadExternalMultiPartitionTableBuilder", KR(ret)); + } else if (OB_FAIL(new_builder->init(builder_param))) { + LOG_WARN("fail to init new_builder", KR(ret)); + } else if (OB_FAIL(index_table_builder_map_.set_refactored(part_id, new_builder))) { + LOG_WARN("fail to set table_builder_map_", KR(ret), K(part_id)); + } else { + table_builder = new_builder; + } + if (OB_FAIL(ret)) { + if (nullptr != new_builder) { + new_builder->~ObDirectLoadExternalMultiPartitionTableBuilder(); + allocator_.free(new_builder); + new_builder = nullptr; + } + } + } + } + } + } + return ret; +} + +} +} \ No newline at end of file diff --git a/src/observer/table_load/ob_table_load_store_table_ctx.h b/src/observer/table_load/ob_table_load_store_table_ctx.h new file mode 100644 index 000000000..04640efa3 --- /dev/null +++ b/src/observer/table_load/ob_table_load_store_table_ctx.h @@ -0,0 +1,87 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#pragma once + +#include "lib/hash/ob_hashmap.h" +#include "ob_tablet_id.h" +#include "storage/direct_load/ob_direct_load_table_data_desc.h" +#include "share/table/ob_table_load_define.h" + +namespace oceanbase +{ +namespace storage +{ +class ObDirectLoadInsertTableContext; +class ObIDirectLoadPartitionTableBuilder; +class ObDirectLoadDMLRowHandler; +} +namespace observer +{ +class ObTableLoadSchema; +class ObTableLoadMerger; +class ObTableLoadIndexTableProjector; +class ObTableLoadStoreCtx; +class ObTableLoadStoreTableCtx +{ +private: + typedef common::hash::ObHashMap TABLE_BUILDER_MAP; +public: + static const int64_t MACRO_BLOCK_WRITER_MEM_SIZE = 10 * 1024LL * 1024LL; + ObTableLoadStoreTableCtx(); + ~ObTableLoadStoreTableCtx(); + int init( + ObTableID table_id, bool is_index_table, ObTableLoadStoreCtx *store_ctx, + const table::ObTableLoadArray &partition_id_array, + const table::ObTableLoadArray &target_partition_id_array); + TABLE_BUILDER_MAP& get_index_table_builder_map() { return index_table_builder_map_; } + int close_index_table_builder(); + int get_index_table_builder(ObIDirectLoadPartitionTableBuilder *&table_builder); + TO_STRING_KV(K_(is_inited)); + bool is_valid() { + return is_inited_; + } +private: + int init_table_load_schema(); + int init_index_projector(); + int init_ls_partition_ids( + const table::ObTableLoadArray &partition_id_array, + const table::ObTableLoadArray &target_partition_id_array); + int init_table_data_desc(); + int init_insert_table_ctx(); + int init_row_handler(); + int init_merger(); +public: + common::ObArenaAllocator allocator_; + ObTableID table_id_; + bool is_index_table_; + ObTableLoadStoreCtx *store_ctx_; + bool need_sort_; + ObTableLoadSchema *schema_; + ObTableLoadIndexTableProjector *project_; + common::ObArray ls_partition_ids_; + common::ObArray target_ls_partition_ids_; + storage::ObDirectLoadTableDataDesc table_data_desc_; + storage::ObDirectLoadTableDataDesc lob_id_table_data_desc_; + bool is_fast_heap_table_; + bool is_multiple_mode_; + storage::ObDirectLoadInsertTableContext *insert_table_ctx_; + ObDirectLoadDMLRowHandler * row_handler_; + ObTableLoadMerger* merger_; + TABLE_BUILDER_MAP index_table_builder_map_; +private: + lib::ObMutex mutex_; + bool is_inited_; +}; + +} +} \ No newline at end of file diff --git a/src/observer/table_load/ob_table_load_store_trans_px_writer.cpp b/src/observer/table_load/ob_table_load_store_trans_px_writer.cpp index 33ebf5f68..42e3395d2 100644 --- a/src/observer/table_load/ob_table_load_store_trans_px_writer.cpp +++ b/src/observer/table_load/ob_table_load_store_trans_px_writer.cpp @@ -17,6 +17,7 @@ #include "observer/table_load/ob_table_load_store_trans.h" #include "observer/table_load/ob_table_load_table_ctx.h" #include "observer/table_load/ob_table_load_trans_store.h" +#include "observer/table_load/ob_table_load_store_table_ctx.h" namespace oceanbase { @@ -130,8 +131,8 @@ int ObTableLoadStoreTransPXWriter::check_tablet(const ObTabletID &tablet_id) LOG_WARN("unexpected store ctx is null", KR(ret), KPC(this)); } else { bool tablet_found = false; - for (int64_t i = 0; i < store_ctx_->ls_partition_ids_.count(); ++i) { - const ObTableLoadLSIdAndPartitionId &ls_part_id = store_ctx_->ls_partition_ids_.at(i); + for (int64_t i = 0; i < store_ctx_->data_store_table_ctx_->ls_partition_ids_.count(); ++i) { + const ObTableLoadLSIdAndPartitionId &ls_part_id = store_ctx_->data_store_table_ctx_->ls_partition_ids_.at(i); if (ls_part_id.part_tablet_id_.tablet_id_ == tablet_id) { tablet_found = true; break; @@ -139,7 +140,7 @@ int ObTableLoadStoreTransPXWriter::check_tablet(const ObTabletID &tablet_id) } if (OB_UNLIKELY(!tablet_found)) { ret = OB_TABLET_NOT_EXIST; - LOG_WARN("tablet id not found", KR(ret), K(tablet_id), K(store_ctx_->ls_partition_ids_)); + LOG_WARN("tablet id not found", KR(ret), K(tablet_id), K(store_ctx_->data_store_table_ctx_->ls_partition_ids_)); } } return ret; diff --git a/src/observer/table_load/ob_table_load_table_compactor.cpp b/src/observer/table_load/ob_table_load_table_compactor.cpp index 457a03191..9106f6bbf 100644 --- a/src/observer/table_load/ob_table_load_table_compactor.cpp +++ b/src/observer/table_load/ob_table_load_table_compactor.cpp @@ -23,6 +23,7 @@ #include "observer/table_load/ob_table_load_multiple_heap_table_compactor.h" #include "observer/table_load/ob_table_load_trans_store.h" #include "observer/table_load/ob_table_load_parallel_merge_table_compactor.h" +#include "observer/table_load/ob_table_load_store_table_ctx.h" namespace oceanbase { @@ -111,6 +112,10 @@ void ObTableLoadTableCompactResult::release_all_table_data() /** * ObTableLoadTableCompactConfig */ +ObTableLoadTableCompactConfig::~ObTableLoadTableCompactConfig() +{ + +} int ObTableLoadTableCompactConfigMainTable::handle_table_compact_success() { @@ -176,11 +181,50 @@ int ObTableLoadTableCompactConfigMainTable::init(ObTableLoadStoreCtx *store_ctx, return ret; } -ObTableLoadTableCompactConfigLobIdTable::ObTableLoadTableCompactConfigLobIdTable() : merger_(nullptr) + +ObTableLoadTableCompactConfigIndexTable::ObTableLoadTableCompactConfigIndexTable() : merger_(nullptr) +{ +} + +ObTableLoadTableCompactConfigIndexTable::~ObTableLoadTableCompactConfigIndexTable() { } +int ObTableLoadTableCompactConfigIndexTable::init(ObTableLoadMerger &merger) +{ + int ret = OB_SUCCESS; + merger_ = &merger; + is_sort_lobid_ = false; + return ret; +} + +int ObTableLoadTableCompactConfigIndexTable::handle_table_compact_success() +{ +// notify merger + return merger_->handle_table_compact_success(); +} + +int ObTableLoadTableCompactConfigIndexTable::get_tables(common::ObIArray &table_array, + common::ObIAllocator &allocator) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(merger_) || OB_ISNULL(merger_->store_table_ctx_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("merger is null", KR(ret)); + } + FOREACH_X(item, merger_->store_table_ctx_->get_index_table_builder_map(), OB_SUCC(ret)) { + if (OB_FAIL(item->second->get_tables(table_array, allocator))) { + LOG_WARN("fail to get tables", KR(ret)); + } + } + return ret; +} + +ObTableLoadTableCompactConfigLobIdTable::ObTableLoadTableCompactConfigLobIdTable() : merger_(nullptr) +{ + +} ObTableLoadTableCompactConfigLobIdTable::~ObTableLoadTableCompactConfigLobIdTable() { @@ -225,10 +269,10 @@ ObTableLoadTableCompactCtx::~ObTableLoadTableCompactCtx() { } -int ObTableLoadTableCompactCtx::init(ObTableLoadStoreCtx *store_ctx, ObTableLoadTableCompactConfig *compact_config) +int ObTableLoadTableCompactCtx::init(ObTableLoadStoreCtx *store_ctx, ObTableLoadStoreTableCtx * store_table_ctx, ObTableLoadTableCompactConfig *compact_config) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(nullptr == store_ctx || nullptr == compact_config)) { + if (OB_UNLIKELY(nullptr == store_ctx || nullptr == store_table_ctx || nullptr == compact_config)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", KR(ret), KP(store_ctx), KP(compact_config)); } else { @@ -236,6 +280,7 @@ int ObTableLoadTableCompactCtx::init(ObTableLoadStoreCtx *store_ctx, ObTableLoad LOG_WARN("fail to init result", KR(ret)); } else { store_ctx_ = store_ctx; + store_table_ctx_ = store_table_ctx; compact_config_ = compact_config; } } @@ -258,8 +303,8 @@ int ObTableLoadTableCompactCtx::new_compactor(ObTableLoadTableCompactorHandle &c if (compact_config_->is_sort_lobid_) { compactor = OB_NEW(ObTableLoadMemCompactor, attr); } else { - if (store_ctx_->is_multiple_mode_) { - if (store_ctx_->table_data_desc_.is_heap_table_) { + if (store_table_ctx_->is_multiple_mode_) { + if (store_table_ctx_->table_data_desc_.is_heap_table_) { compactor = OB_NEW(ObTableLoadMultipleHeapTableCompactor, attr); } else { compactor = OB_NEW(ObTableLoadMemCompactor, attr); diff --git a/src/observer/table_load/ob_table_load_table_compactor.h b/src/observer/table_load/ob_table_load_table_compactor.h index 2b879d767..d185db212 100644 --- a/src/observer/table_load/ob_table_load_table_compactor.h +++ b/src/observer/table_load/ob_table_load_table_compactor.h @@ -25,6 +25,7 @@ namespace observer class ObTableLoadStoreCtx; class ObTableLoadMerger; class ObTableLoadTableCompactCtx; +class ObTableLoadStoreTableCtx; struct ObTableLoadTableCompactTabletResult : public common::LinkHashValue { @@ -56,6 +57,7 @@ class ObTableLoadTableCompactConfig { public: ObTableLoadTableCompactConfig() : is_sort_lobid_(false) {} + virtual ~ObTableLoadTableCompactConfig(); virtual int handle_table_compact_success() = 0; virtual int get_tables(common::ObIArray &table_array, common::ObIAllocator &allocator) = 0; @@ -66,8 +68,8 @@ public: class ObTableLoadTableCompactConfigMainTable : public ObTableLoadTableCompactConfig { public: + ~ObTableLoadTableCompactConfigMainTable() override {} int init(ObTableLoadStoreCtx *store_ctx, ObTableLoadMerger &merger); - int handle_table_compact_success() override; int get_tables(common::ObIArray &table_array, common::ObIAllocator &allocator) override; @@ -76,6 +78,19 @@ private: ObTableLoadMerger *merger_; }; +class ObTableLoadTableCompactConfigIndexTable : public ObTableLoadTableCompactConfig +{ +public: + ObTableLoadTableCompactConfigIndexTable(); + ~ObTableLoadTableCompactConfigIndexTable() override; + int init(ObTableLoadMerger &merger); + int handle_table_compact_success() override; + int get_tables(common::ObIArray &table_array, + common::ObIAllocator &allocator) override; +private: + ObTableLoadMerger *merger_; +}; + class ObTableLoadTableCompactConfigLobIdTable : public ObTableLoadTableCompactConfig { public: @@ -144,7 +159,7 @@ class ObTableLoadTableCompactCtx public: ObTableLoadTableCompactCtx(); ~ObTableLoadTableCompactCtx(); - int init(ObTableLoadStoreCtx *store_ctx, ObTableLoadTableCompactConfig *compact_config); + int init(ObTableLoadStoreCtx *store_ctx, ObTableLoadStoreTableCtx * store_table_ctx, ObTableLoadTableCompactConfig *compact_config); bool is_valid() const; int start(); void stop(); @@ -157,6 +172,7 @@ private: public: ObTableLoadStoreCtx *store_ctx_; + ObTableLoadStoreTableCtx * store_table_ctx_; ObTableLoadTableCompactConfig *compact_config_; mutable obsys::ObRWLock rwlock_; ObTableLoadTableCompactorHandle compactor_handle_; diff --git a/src/observer/table_load/ob_table_load_table_ctx.h b/src/observer/table_load/ob_table_load_table_ctx.h index 035b638e2..586bde4a5 100644 --- a/src/observer/table_load/ob_table_load_table_ctx.h +++ b/src/observer/table_load/ob_table_load_table_ctx.h @@ -83,7 +83,7 @@ private: public: ObTableLoadParam param_; ObTableLoadDDLParam ddl_param_; - ObTableLoadSchema schema_; + ObTableLoadSchema schema_; // origin table load schema ObTableLoadCoordinatorCtx *coordinator_ctx_; // 只在控制节点构造 ObTableLoadStoreCtx *store_ctx_; // 只在数据节点构造 sql::ObLoadDataGID gid_; diff --git a/src/observer/table_load/ob_table_load_trans_bucket_writer.cpp b/src/observer/table_load/ob_table_load_trans_bucket_writer.cpp index 88ace5245..5c5d914e2 100644 --- a/src/observer/table_load/ob_table_load_trans_bucket_writer.cpp +++ b/src/observer/table_load/ob_table_load_trans_bucket_writer.cpp @@ -399,7 +399,7 @@ int ObTableLoadTransBucketWriter::write_for_partitioned(SessionContext &session_ } else if (OB_FAIL(coordinator_ctx_->partition_calc_.get_part_key(obj_rows.at(i), part_key))) { LOG_WARN("fail to get part key", KR(ret)); } else if (OB_FAIL(coordinator_ctx_->partition_calc_.cast_part_key(part_key, allocator))) { - if (OB_FAIL(error_row_handler->handle_error_row(ret, part_key))) { + if (OB_FAIL(error_row_handler->handle_error_row(ret))) { LOG_WARN("failed to handle error row", K(ret), K(part_key)); } else { ret = OB_SUCCESS; @@ -422,7 +422,7 @@ int ObTableLoadTransBucketWriter::write_for_partitioned(SessionContext &session_ bool need_write = false; if (OB_UNLIKELY(!partition_id.is_valid())) { ret = OB_NO_PARTITION_FOR_GIVEN_VALUE; - if (OB_FAIL(error_row_handler->handle_error_row(ret, part_keys.at(i)))) { + if (OB_FAIL(error_row_handler->handle_error_row(ret))) { LOG_WARN("failed to handle error row", K(ret), K(part_keys.at(i))); } else { ret = OB_SUCCESS; diff --git a/src/observer/table_load/ob_table_load_trans_store.cpp b/src/observer/table_load/ob_table_load_trans_store.cpp index 34e4159f5..7f99c360a 100644 --- a/src/observer/table_load/ob_table_load_trans_store.cpp +++ b/src/observer/table_load/ob_table_load_trans_store.cpp @@ -16,11 +16,14 @@ #include "observer/omt/ob_tenant_timezone_mgr.h" #include "observer/table_load/ob_table_load_autoinc_nextval.h" #include "observer/table_load/ob_table_load_error_row_handler.h" +#include "observer/table_load/ob_table_load_data_row_handler.h" +#include "storage/direct_load/ob_direct_load_dml_row_handler.h" #include "observer/table_load/ob_table_load_stat.h" #include "observer/table_load/ob_table_load_store_ctx.h" #include "observer/table_load/ob_table_load_table_ctx.h" #include "observer/table_load/ob_table_load_trans_ctx.h" #include "observer/table_load/ob_table_load_utils.h" +#include "observer/table_load/ob_table_load_store_table_ctx.h" #include "sql/engine/cmd/ob_load_data_utils.h" #include "sql/resolver/expr/ob_raw_expr_util.h" #include "sql/ob_sql_utils.h" @@ -157,8 +160,8 @@ int ObTableLoadTransStoreWriter::init() ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", KR(ret), KPC(trans_store_)); } else { - table_data_desc_ = &store_ctx_->table_data_desc_; - collation_type_ = trans_ctx_->ctx_->schema_.collation_type_; + table_data_desc_ = &store_ctx_->data_store_table_ctx_->table_data_desc_; + collation_type_ = store_ctx_->data_store_table_ctx_->schema_->collation_type_; if (OB_FAIL(ObSQLUtils::get_default_cast_mode(store_ctx_->ctx_->session_info_, cast_mode_))) { LOG_WARN("fail to get_default_cast_mode", KR(ret)); } else if (OB_FAIL(init_session_ctx_array())) { @@ -175,7 +178,7 @@ int ObTableLoadTransStoreWriter::init() int ObTableLoadTransStoreWriter::init_column_schemas_and_lob_info() { int ret = OB_SUCCESS; - const ObIArray &column_descs = store_ctx_->ctx_->schema_.column_descs_; + const ObIArray &column_descs = store_ctx_->data_store_table_ctx_->schema_->column_descs_; const ObTableSchema *table_schema = nullptr; if (OB_FAIL(ObTableLoadSchema::get_table_schema(param_.tenant_id_, param_.table_id_, schema_guard_, table_schema))) { @@ -215,12 +218,12 @@ int ObTableLoadTransStoreWriter::init_session_ctx_array() } ObDirectLoadTableStoreParam param; param.table_data_desc_ = *table_data_desc_; - param.datum_utils_ = &(trans_ctx_->ctx_->schema_.datum_utils_); - param.file_mgr_ = trans_ctx_->ctx_->store_ctx_->tmp_file_mgr_; - param.is_multiple_mode_ = trans_ctx_->ctx_->store_ctx_->is_multiple_mode_; - param.is_fast_heap_table_ = trans_ctx_->ctx_->store_ctx_->is_fast_heap_table_; - param.insert_table_ctx_ = trans_ctx_->ctx_->store_ctx_->insert_table_ctx_; - param.dml_row_handler_ = trans_ctx_->ctx_->store_ctx_->error_row_handler_; + param.datum_utils_ = &(store_ctx_->data_store_table_ctx_->schema_->datum_utils_); + param.file_mgr_ = store_ctx_->tmp_file_mgr_; + param.is_multiple_mode_ = store_ctx_->data_store_table_ctx_->is_multiple_mode_; + param.is_fast_heap_table_ = store_ctx_->data_store_table_ctx_->is_fast_heap_table_; + param.insert_table_ctx_ = store_ctx_->data_store_table_ctx_->insert_table_ctx_; + param.dml_row_handler_ = store_ctx_->data_store_table_ctx_->row_handler_; for (int64_t i = 0; OB_SUCC(ret) && i < session_count; ++i) { SessionContext *session_ctx = session_ctx_array_ + i; if (param_.px_mode_) { @@ -405,7 +408,7 @@ int ObTableLoadTransStoreWriter::cast_row(ObArenaAllocator &cast_allocator, if (OB_FAIL(ret)) { ObTableLoadErrorRowHandler *error_row_handler = trans_ctx_->ctx_->store_ctx_->error_row_handler_; - if (OB_FAIL(error_row_handler->handle_error_row(ret, row))) { + if (OB_FAIL(error_row_handler->handle_error_row(ret))) { LOG_WARN("failed to handle error row", K(ret), K(row)); } else { ret = OB_EAGAIN; @@ -553,12 +556,13 @@ int ObTableLoadTransStoreWriter::write_row_to_table_store(ObDirectLoadTableStore if (OB_FAIL(ret)) { ObTableLoadErrorRowHandler *error_row_handler = trans_ctx_->ctx_->store_ctx_->error_row_handler_; + ObDirectLoadDMLRowHandler *data_row_handler = trans_ctx_->ctx_->store_ctx_->data_store_table_ctx_->row_handler_; if (OB_LIKELY(OB_ERR_PRIMARY_KEY_DUPLICATE == ret)) { - if (OB_FAIL(error_row_handler->handle_update_row(datum_row))) { + if (OB_FAIL(data_row_handler->handle_update_row(datum_row))) { LOG_WARN("fail to handle update row", KR(ret), K(datum_row)); } } else if (OB_LIKELY(OB_ROWKEY_ORDER_ERROR == ret)) { - if (OB_FAIL(error_row_handler->handle_error_row(ret, datum_row))) { + if (OB_FAIL(error_row_handler->handle_error_row(ret))) { LOG_WARN("fail to handle error row", KR(ret), K(tablet_id), K(datum_row)); } } diff --git a/src/storage/direct_load/ob_direct_load_conflict_check.cpp b/src/storage/direct_load/ob_direct_load_conflict_check.cpp index 1547c0542..0a66168d3 100644 --- a/src/storage/direct_load/ob_direct_load_conflict_check.cpp +++ b/src/storage/direct_load/ob_direct_load_conflict_check.cpp @@ -48,8 +48,8 @@ bool ObDirectLoadConflictCheckParam::is_valid() const { return tablet_id_.is_valid() && store_column_count_ > 0 && table_data_desc_.is_valid() && nullptr != origin_table_ && nullptr != range_ && range_->is_valid() && - nullptr != col_descs_ && nullptr != lob_column_idxs_ && nullptr != builder_ && - nullptr != datum_utils_ && lob_meta_datum_utils_ != nullptr && + nullptr != col_descs_ && nullptr != lob_column_idxs_ && nullptr != builder_ && + nullptr != datum_utils_ && nullptr != lob_meta_datum_utils_ && nullptr != dml_row_handler_ && (lob_column_idxs_->empty() || tablet_id_in_lob_id_.is_valid()); } @@ -177,7 +177,7 @@ int ObDirectLoadConflictCheck::handle_get_next_row_finish( } if (OB_SUCC(ret)) { if (cmp_ret == 0) { - if (OB_FAIL(param_.dml_row_handler_->handle_update_row(*origin_row_, *load_row, datum_row))) { + if (OB_FAIL(param_.dml_row_handler_->handle_update_row(param_.tablet_id_, *origin_row_, *load_row, datum_row))) { LOG_WARN("fail to handle update row", KR(ret), KP(origin_row_), KP(load_row)); } else { if (datum_row == load_row) { @@ -191,12 +191,17 @@ int ObDirectLoadConflictCheck::handle_get_next_row_finish( origin_row_ = nullptr; } else { datum_row = load_row; - if (OB_FAIL(param_.dml_row_handler_->handle_insert_row(*datum_row))) { - LOG_WARN("fail to handle insert row", KR(ret), KP(datum_row)); + if (datum_row->row_flag_.is_delete()) { + if (OB_FAIL(param_.dml_row_handler_->handle_delete_row(param_.tablet_id_, *datum_row))) { + LOG_WARN("fail to handle insert row", KR(ret), KP(datum_row)); + } + } else { + if (OB_FAIL(param_.dml_row_handler_->handle_insert_row(param_.tablet_id_, *datum_row))) { + LOG_WARN("fail to handle insert row", KR(ret), KP(datum_row)); + } } } } - return ret; } diff --git a/src/storage/direct_load/ob_direct_load_conflict_check.h b/src/storage/direct_load/ob_direct_load_conflict_check.h index 5c1170ea7..ab6f137f6 100644 --- a/src/storage/direct_load/ob_direct_load_conflict_check.h +++ b/src/storage/direct_load/ob_direct_load_conflict_check.h @@ -26,7 +26,6 @@ namespace storage { class ObDirectLoadSSTable; class ObDirectLoadMultipleSSTable; - struct ObDirectLoadConflictCheckParam { public: diff --git a/src/storage/direct_load/ob_direct_load_data_fuse.cpp b/src/storage/direct_load/ob_direct_load_data_fuse.cpp index 8d5f224bf..9d01e6a45 100644 --- a/src/storage/direct_load/ob_direct_load_data_fuse.cpp +++ b/src/storage/direct_load/ob_direct_load_data_fuse.cpp @@ -228,8 +228,14 @@ int ObDirectLoadDataFuse::inner_get_next_row(const ObDatumRow *&datum_row) if (OB_FAIL(rows_merger_.pop())) { LOG_WARN("fail to pop item", KR(ret)); } else if (item->iter_idx_ == LOAD_IDX) { - if (OB_FAIL(param_.dml_row_handler_->handle_insert_row(*datum_row))) { - LOG_WARN("fail to handle insert row", KR(ret), KPC(datum_row)); + if (datum_row->row_flag_.is_delete()) { + if (OB_FAIL(param_.dml_row_handler_->handle_delete_row(param_.tablet_id_, *datum_row))) { + LOG_WARN("fail to handle insert row", KR(ret), KP(datum_row)); + } + } else { + if (OB_FAIL(param_.dml_row_handler_->handle_insert_row(param_.tablet_id_, *datum_row))) { + LOG_WARN("fail to handle insert row", KR(ret), KP(datum_row)); + } } } } @@ -253,7 +259,7 @@ int ObDirectLoadDataFuse::inner_get_next_row(const ObDatumRow *&datum_row) } } if (OB_SUCC(ret)) { - if (OB_FAIL(param_.dml_row_handler_->handle_update_row(*old_row, *new_row, datum_row))) { + if (OB_FAIL(param_.dml_row_handler_->handle_update_row(param_.tablet_id_, *old_row, *new_row, datum_row))) { LOG_WARN("fail to handle update row", KR(ret), KPC(old_row), KPC(new_row)); } } diff --git a/src/storage/direct_load/ob_direct_load_data_insert.cpp b/src/storage/direct_load/ob_direct_load_data_insert.cpp index 4018e1f29..f02fa8c30 100644 --- a/src/storage/direct_load/ob_direct_load_data_insert.cpp +++ b/src/storage/direct_load/ob_direct_load_data_insert.cpp @@ -89,8 +89,16 @@ int ObDirectLoadDataInsert::get_next_row(const ObDatumRow *&datum_row) } else if (OB_UNLIKELY(datum_row->count_ != param_.store_column_count_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected column count", KR(ret), K(datum_row->count_), K(param_.store_column_count_)); - } else if (OB_FAIL(param_.dml_row_handler_->handle_insert_row(*datum_row))) { - LOG_WARN("fail to handle insert row", KR(ret), KPC(datum_row)); + } else { + if (datum_row->row_flag_.is_delete()) { + if (OB_FAIL(param_.dml_row_handler_->handle_delete_row(param_.tablet_id_, *datum_row))) { + LOG_WARN("fail to handle insert row", KR(ret), KP(datum_row)); + } + } else { + if (OB_FAIL(param_.dml_row_handler_->handle_insert_row(param_.tablet_id_, *datum_row))) { + LOG_WARN("fail to handle insert row", KR(ret), KP(datum_row)); + } + } } return ret; diff --git a/src/storage/direct_load/ob_direct_load_dml_row_handler.h b/src/storage/direct_load/ob_direct_load_dml_row_handler.h index 2a269542b..d838b286f 100644 --- a/src/storage/direct_load/ob_direct_load_dml_row_handler.h +++ b/src/storage/direct_load/ob_direct_load_dml_row_handler.h @@ -26,7 +26,10 @@ public: ObDirectLoadDMLRowHandler() = default; virtual ~ObDirectLoadDMLRowHandler() = default; // handle rows direct insert into sstable - virtual int handle_insert_row(const blocksstable::ObDatumRow &row) = 0; + virtual int handle_insert_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) = 0; + virtual int handle_delete_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) = 0; + // only used for heap table + virtual int handle_insert_row_with_multi_version(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) = 0; // handle rows with the same primary key in the imported data virtual int handle_update_row(const blocksstable::ObDatumRow &row) = 0; virtual int handle_update_row(common::ObArray &rows, @@ -34,7 +37,8 @@ public: virtual int handle_update_row(common::ObArray &rows, const ObDirectLoadMultipleDatumRow *&row) = 0; // handle rows with the same primary key between the imported data and the original data - virtual int handle_update_row(const blocksstable::ObDatumRow &old_row, + virtual int handle_update_row(const ObTabletID tablet_id, + const blocksstable::ObDatumRow &old_row, const blocksstable::ObDatumRow &new_row, const blocksstable::ObDatumRow *&result_row) = 0; DECLARE_PURE_VIRTUAL_TO_STRING; diff --git a/src/storage/direct_load/ob_direct_load_external_multi_partition_row.cpp b/src/storage/direct_load/ob_direct_load_external_multi_partition_row.cpp index d817f48e5..c67f73cfa 100644 --- a/src/storage/direct_load/ob_direct_load_external_multi_partition_row.cpp +++ b/src/storage/direct_load/ob_direct_load_external_multi_partition_row.cpp @@ -90,7 +90,7 @@ OB_DEF_SERIALIZE_SIZE_SIMPLE(ObDirectLoadExternalMultiPartitionRow) */ ObDirectLoadConstExternalMultiPartitionRow::ObDirectLoadConstExternalMultiPartitionRow() - : buf_size_(0), buf_(nullptr) + : is_deleted_(false), buf_size_(0), buf_(nullptr) { } @@ -104,6 +104,7 @@ void ObDirectLoadConstExternalMultiPartitionRow::reset() rowkey_datum_array_.reset(); seq_no_.reset(); buf_size_ = 0; + is_deleted_ = false; buf_ = nullptr; } @@ -116,6 +117,7 @@ ObDirectLoadConstExternalMultiPartitionRow &ObDirectLoadConstExternalMultiPartit rowkey_datum_array_ = other.rowkey_datum_array_; buf_size_ = other.buf_size_; seq_no_ = other.seq_no_; + is_deleted_ = other.is_deleted_; buf_ = other.buf_; } return *this; @@ -128,6 +130,7 @@ ObDirectLoadConstExternalMultiPartitionRow &ObDirectLoadConstExternalMultiPartit rowkey_datum_array_ = other.external_row_.rowkey_datum_array_; buf_size_ = other.external_row_.buf_size_; seq_no_ = other.external_row_.seq_no_; + is_deleted_ = other.external_row_.is_deleted_; buf_ = other.external_row_.buf_; return *this; } @@ -158,6 +161,7 @@ int ObDirectLoadConstExternalMultiPartitionRow::deep_copy( } else { buf_size_ = src.buf_size_; seq_no_ = src.seq_no_; + is_deleted_ = src.is_deleted_; buf_ = buf + pos; MEMCPY(buf + pos, src.buf_, buf_size_); pos += buf_size_; diff --git a/src/storage/direct_load/ob_direct_load_external_multi_partition_row.h b/src/storage/direct_load/ob_direct_load_external_multi_partition_row.h index 35b3a97dd..e4f507ef2 100644 --- a/src/storage/direct_load/ob_direct_load_external_multi_partition_row.h +++ b/src/storage/direct_load/ob_direct_load_external_multi_partition_row.h @@ -63,6 +63,7 @@ public: common::ObTabletID tablet_id_; ObDirectLoadConstDatumArray rowkey_datum_array_; table::ObTableLoadSequenceNo seq_no_; + bool is_deleted_; int64_t buf_size_; const char *buf_; }; diff --git a/src/storage/direct_load/ob_direct_load_external_multi_partition_table.cpp b/src/storage/direct_load/ob_direct_load_external_multi_partition_table.cpp index d7165e999..d7804a896 100644 --- a/src/storage/direct_load/ob_direct_load_external_multi_partition_table.cpp +++ b/src/storage/direct_load/ob_direct_load_external_multi_partition_table.cpp @@ -110,7 +110,7 @@ int ObDirectLoadExternalMultiPartitionTableBuilder::append_row(const ObTabletID OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_append_row_time_us); row_.tablet_id_ = tablet_id; if (OB_FAIL(row_.external_row_.from_datums(datum_row.storage_datums_, datum_row.count_, - param_.table_data_desc_.rowkey_column_num_, seq_no))) { + param_.table_data_desc_.rowkey_column_num_, seq_no, datum_row.row_flag_.is_delete()))) { LOG_WARN("fail to from datums", KR(ret)); } else if (OB_FAIL(external_writer_.write_item(row_))) { LOG_WARN("fail to write item", KR(ret)); diff --git a/src/storage/direct_load/ob_direct_load_external_row.cpp b/src/storage/direct_load/ob_direct_load_external_row.cpp index cef75c669..0ca6bb99e 100644 --- a/src/storage/direct_load/ob_direct_load_external_row.cpp +++ b/src/storage/direct_load/ob_direct_load_external_row.cpp @@ -33,6 +33,7 @@ void ObDirectLoadExternalRow::reset() { rowkey_datum_array_.reset(); seq_no_.reset(); + is_deleted_ = false; buf_size_ = 0; buf_ = nullptr; allocator_.reset(); @@ -42,6 +43,7 @@ void ObDirectLoadExternalRow::reuse() { rowkey_datum_array_.reuse(); seq_no_.reset(); + is_deleted_ = false; buf_size_ = 0; buf_ = nullptr; allocator_.reuse(); @@ -73,6 +75,7 @@ int ObDirectLoadExternalRow::deep_copy(const ObDirectLoadExternalRow &src, char } else { buf_size_ = src.buf_size_; seq_no_ = src.seq_no_; + is_deleted_ = src.is_deleted_; buf_ = buf + pos; MEMCPY(buf + pos, src.buf_, buf_size_); pos += buf_size_; @@ -82,7 +85,8 @@ int ObDirectLoadExternalRow::deep_copy(const ObDirectLoadExternalRow &src, char } int ObDirectLoadExternalRow::from_datums(ObStorageDatum *datums, int64_t column_count, - int64_t rowkey_column_count, const ObTableLoadSequenceNo &seq_no) + int64_t rowkey_column_count, const ObTableLoadSequenceNo &seq_no, + const bool is_deleted) { OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, transfer_external_row_time_us); int ret = OB_SUCCESS; @@ -110,6 +114,7 @@ int ObDirectLoadExternalRow::from_datums(ObStorageDatum *datums, int64_t column_ buf_ = buf; buf_size_ = buf_size; seq_no_ = seq_no; + is_deleted_ = is_deleted; } } } @@ -165,7 +170,7 @@ OB_DEF_SERIALIZE_SIMPLE(ObDirectLoadExternalRow) { OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_row_serialize_time_us); int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_ENCODE, rowkey_datum_array_, seq_no_, buf_size_); + LST_DO_CODE(OB_UNIS_ENCODE, rowkey_datum_array_, seq_no_, is_deleted_, buf_size_); if (OB_SUCC(ret) && OB_NOT_NULL(buf_)) { MEMCPY(buf + pos, buf_, buf_size_); pos += buf_size_; @@ -178,7 +183,7 @@ OB_DEF_DESERIALIZE_SIMPLE(ObDirectLoadExternalRow) OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_row_deserialize_time_us); int ret = OB_SUCCESS; reuse(); - LST_DO_CODE(OB_UNIS_DECODE, rowkey_datum_array_, seq_no_, buf_size_); + LST_DO_CODE(OB_UNIS_DECODE, rowkey_datum_array_, seq_no_, is_deleted_, buf_size_); if (OB_SUCC(ret)) { buf_ = buf + pos; pos += buf_size_; @@ -190,7 +195,7 @@ OB_DEF_SERIALIZE_SIZE_SIMPLE(ObDirectLoadExternalRow) { OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_row_serialize_time_us); int64_t len = 0; - LST_DO_CODE(OB_UNIS_ADD_LEN, rowkey_datum_array_, seq_no_, buf_size_); + LST_DO_CODE(OB_UNIS_ADD_LEN, rowkey_datum_array_, seq_no_, is_deleted_, buf_size_); len += buf_size_; return len; } diff --git a/src/storage/direct_load/ob_direct_load_external_row.h b/src/storage/direct_load/ob_direct_load_external_row.h index ad9a177d7..5ce9b1531 100644 --- a/src/storage/direct_load/ob_direct_load_external_row.h +++ b/src/storage/direct_load/ob_direct_load_external_row.h @@ -32,20 +32,24 @@ public: int deep_copy(const ObDirectLoadExternalRow &src, char *buf, const int64_t len, int64_t &pos); // not deep copy int from_datums(blocksstable::ObStorageDatum *datums, int64_t column_count, - int64_t rowkey_column_count, const table::ObTableLoadSequenceNo &seq_no); + int64_t rowkey_column_count, const table::ObTableLoadSequenceNo &seq_no, + const bool is_deleted); int to_datums(blocksstable::ObStorageDatum *datums, int64_t column_count) const; int get_rowkey(blocksstable::ObDatumRowkey &rowkey) const; bool is_valid() const { return rowkey_datum_array_.is_valid() && seq_no_.is_valid() && buf_size_ > 0 && nullptr != buf_; } + + bool is_deleted() const { return is_deleted_; } OB_INLINE int64_t get_raw_size() const { return buf_size_; } - TO_STRING_KV(K_(rowkey_datum_array), K_(seq_no), K_(buf_size), KP_(buf)); + TO_STRING_KV(K_(rowkey_datum_array), K_(seq_no), K_(is_deleted), K_(buf_size), KP_(buf)); public: common::ObArenaAllocator allocator_; ObDirectLoadDatumArray rowkey_datum_array_; table::ObTableLoadSequenceNo seq_no_; + bool is_deleted_;//default is false int64_t buf_size_; const char *buf_; }; diff --git a/src/storage/direct_load/ob_direct_load_external_table_builder.cpp b/src/storage/direct_load/ob_direct_load_external_table_builder.cpp index dbef108e6..304290d26 100644 --- a/src/storage/direct_load/ob_direct_load_external_table_builder.cpp +++ b/src/storage/direct_load/ob_direct_load_external_table_builder.cpp @@ -104,7 +104,7 @@ int ObDirectLoadExternalTableBuilder::append_row(const ObTabletID &tablet_id, } else { OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_append_row_time_us); if (OB_FAIL(external_row_.from_datums(datum_row.storage_datums_, datum_row.count_, - build_param_.table_data_desc_.rowkey_column_num_, seq_no))) { + build_param_.table_data_desc_.rowkey_column_num_, seq_no, datum_row.row_flag_.is_delete()))) { LOG_WARN("fail to from datums", KR(ret)); } else if (OB_FAIL(external_writer_.write_item(external_row_))) { LOG_WARN("fail to write item", KR(ret)); diff --git a/src/storage/direct_load/ob_direct_load_fast_heap_table_builder.cpp b/src/storage/direct_load/ob_direct_load_fast_heap_table_builder.cpp index ba6766825..01b0401f3 100644 --- a/src/storage/direct_load/ob_direct_load_fast_heap_table_builder.cpp +++ b/src/storage/direct_load/ob_direct_load_fast_heap_table_builder.cpp @@ -220,7 +220,7 @@ int ObDirectLoadFastHeapTableBuilder::append_row(const ObTabletID &tablet_id, } } if (OB_SUCC(ret)) { - if (OB_FAIL(param_.dml_row_handler_->handle_insert_row(datum_row_))) { + if (OB_FAIL(param_.dml_row_handler_->handle_insert_row_with_multi_version(param_.tablet_id_, datum_row_))) { LOG_WARN("fail to handle insert row", KR(ret), K_(datum_row)); } } diff --git a/src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp b/src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp index 67c6b1653..9fc8a7acc 100644 --- a/src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp +++ b/src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp @@ -776,7 +776,7 @@ int ObDirectLoadInsertTableContext::init( MTL(ObTenantDirectLoadMgr *)->alloc_execution_context_id(ddl_ctrl_.context_id_))) { LOG_WARN("alloc execution context id failed", K(ret)); } else if (OB_FAIL(create_all_tablet_contexts(ls_partition_ids, target_ls_partition_ids))) { - LOG_WARN("fail to create all tablet contexts", KR(ret)); + LOG_WARN("fail to create all tablet contexts", KR(ret), K(ls_partition_ids), K(target_ls_partition_ids)); } else if (param_.online_opt_stat_gather_ && sql_stat_map_.create(1024, "TLD_SqlStatMap", "TLD_SqlStatMap", MTL_ID())) { LOG_WARN("fail to create sql stat map", KR(ret)); @@ -800,7 +800,7 @@ int ObDirectLoadInsertTableContext::create_all_tablet_contexts( if (OB_ISNULL(tablet_ctx = OB_NEWx(ObDirectLoadInsertTabletContext, (&allocator_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to new ObDirectLoadInsertTabletContext", KR(ret)); - } else if (OB_FAIL(tablet_ctx->init(this, ls_id, origin_tablet_id, tablet_id))) { + } else if (OB_FAIL(tablet_ctx->init(this, ls_id, origin_tablet_id, tablet_id))){ LOG_WARN("fail to init fast heap table tablet ctx", KR(ret)); } else if (OB_FAIL(tablet_ctx_map_.set_refactored(origin_tablet_id, tablet_ctx))) { LOG_WARN("fail to set tablet ctx map", KR(ret)); diff --git a/src/storage/direct_load/ob_direct_load_insert_table_ctx.h b/src/storage/direct_load/ob_direct_load_insert_table_ctx.h index 762b5a873..051ec375b 100644 --- a/src/storage/direct_load/ob_direct_load_insert_table_ctx.h +++ b/src/storage/direct_load/ob_direct_load_insert_table_ctx.h @@ -140,6 +140,8 @@ public: const ObLobId &get_min_insert_lob_id() const { return min_insert_lob_id_; } + OB_INLINE bool is_incremental() const {return nullptr != param_ ? param_->is_incremental_ : false;}; + public: int init(ObDirectLoadInsertTableContext *table_ctx, const share::ObLSID &ls_id, @@ -252,7 +254,7 @@ public: OB_INLINE bool need_rescan() const { return (!param_.is_incremental_ && param_.is_column_store_); } OB_INLINE bool need_del_lob() const { return (param_.is_incremental_ && param_.lob_column_count_ > 0); } - + OB_INLINE TABLET_CTX_MAP &get_tablet_ctx_map() { return tablet_ctx_map_; } int64_t get_sql_stat_column_count() const; int get_sql_statistics(table::ObTableLoadSqlStatistics *&sql_statistics); int update_sql_statistics(table::ObTableLoadSqlStatistics &sql_statistics, diff --git a/src/storage/direct_load/ob_direct_load_lob_meta_row_iter.h b/src/storage/direct_load/ob_direct_load_lob_meta_row_iter.h index 0ff169021..656f9ac91 100644 --- a/src/storage/direct_load/ob_direct_load_lob_meta_row_iter.h +++ b/src/storage/direct_load/ob_direct_load_lob_meta_row_iter.h @@ -28,7 +28,18 @@ class ObDirectLoadLobIdConflictHandler : public ObDirectLoadDMLRowHandler public: ObDirectLoadLobIdConflictHandler() {} virtual ~ObDirectLoadLobIdConflictHandler() {} - int handle_insert_row(const blocksstable::ObDatumRow &row) override { return OB_SUCCESS; } + int handle_insert_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) + { + return OB_ERR_UNEXPECTED; + } + int handle_delete_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) + { + return OB_ERR_UNEXPECTED; + } + int handle_insert_row_with_multi_version(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) + { + return OB_ERR_UNEXPECTED; + } int handle_update_row(const blocksstable::ObDatumRow &row) override { return OB_ERR_UNEXPECTED; } int handle_update_row(common::ObArray &rows, const ObDirectLoadExternalRow *&row) override @@ -40,7 +51,7 @@ public: { return OB_ERR_UNEXPECTED; } - int handle_update_row(const blocksstable::ObDatumRow &old_row, + int handle_update_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &old_row, const blocksstable::ObDatumRow &new_row, const blocksstable::ObDatumRow *&result_row) override { diff --git a/src/storage/direct_load/ob_direct_load_mem_dump.cpp b/src/storage/direct_load/ob_direct_load_mem_dump.cpp index 81d037899..7f3539531 100644 --- a/src/storage/direct_load/ob_direct_load_mem_dump.cpp +++ b/src/storage/direct_load/ob_direct_load_mem_dump.cpp @@ -278,6 +278,7 @@ int ObDirectLoadMemDump::dump_tables() } } if (OB_SUCC(ret)) { + datum_row.row_flag_.set_flag(external_row->is_deleted_?ObDmlFlag::DF_DELETE : ObDmlFlag::DF_INSERT); if (OB_FAIL(external_row->to_datums(datum_row.storage_datums_, datum_row.count_))) { LOG_WARN("fail to transfer dataum row", KR(ret)); } else if (OB_FAIL(table_builder->append_row(external_row->tablet_id_, external_row->seq_no_, datum_row))) { diff --git a/src/storage/direct_load/ob_direct_load_merge_ctx.cpp b/src/storage/direct_load/ob_direct_load_merge_ctx.cpp index fc0bd477b..c08e25762 100644 --- a/src/storage/direct_load/ob_direct_load_merge_ctx.cpp +++ b/src/storage/direct_load/ob_direct_load_merge_ctx.cpp @@ -13,6 +13,7 @@ #define USING_LOG_PREFIX STORAGE #include "storage/direct_load/ob_direct_load_merge_ctx.h" +#include "observer/table_load/ob_table_load_store_ctx.h" #include "share/ob_tablet_autoincrement_service.h" #include "storage/direct_load/ob_direct_load_external_table.h" #include "storage/direct_load/ob_direct_load_multiple_heap_table.h" diff --git a/src/storage/direct_load/ob_direct_load_merge_ctx.h b/src/storage/direct_load/ob_direct_load_merge_ctx.h index 466744ebd..93177f9fb 100644 --- a/src/storage/direct_load/ob_direct_load_merge_ctx.h +++ b/src/storage/direct_load/ob_direct_load_merge_ctx.h @@ -71,7 +71,8 @@ public: KP_(insert_table_ctx), KP_(dml_row_handler), KP_(file_mgr), - K_(trans_param)); + K_(trans_param), + K_(index_table_count)); public: uint64_t table_id_; uint64_t lob_meta_table_id_; @@ -95,6 +96,7 @@ public: ObDirectLoadDMLRowHandler *dml_row_handler_; ObDirectLoadTmpFileManager *file_mgr_; ObDirectLoadTransParam trans_param_; + int index_table_count_; }; class ObDirectLoadMergeCtx @@ -162,8 +164,9 @@ public: { return param_.is_incremental_ && (param_.insert_mode_ == ObDirectLoadInsertMode::NORMAL || - (param_.insert_mode_ == ObDirectLoadInsertMode::INC_REPLACE && param_.lob_column_idxs_->count() > 0)); + (param_.insert_mode_ == ObDirectLoadInsertMode::INC_REPLACE && (param_.lob_column_idxs_->count() > 0 || param_.index_table_count_ > 0))); } + const ObDirectLoadMergeParam &get_param() const { return param_; } const common::ObTabletID &get_tablet_id() const { return tablet_id_; } const common::ObIArray &get_tasks() const diff --git a/src/storage/direct_load/ob_direct_load_multiple_datum_row.cpp b/src/storage/direct_load/ob_direct_load_multiple_datum_row.cpp index 8d69fa5d7..30b912ec7 100644 --- a/src/storage/direct_load/ob_direct_load_multiple_datum_row.cpp +++ b/src/storage/direct_load/ob_direct_load_multiple_datum_row.cpp @@ -24,7 +24,7 @@ using namespace blocksstable; using namespace table; ObDirectLoadMultipleDatumRow::ObDirectLoadMultipleDatumRow() - : allocator_("TLD_MultiRow"), buf_size_(0), buf_(nullptr) + : allocator_("TLD_MultiRow"), is_deleted_(false), buf_size_(0), buf_(nullptr) { allocator_.set_tenant_id(MTL_ID()); } @@ -37,6 +37,7 @@ void ObDirectLoadMultipleDatumRow::reset() { rowkey_.reset(); seq_no_.reset(); + is_deleted_ = false; buf_size_ = 0; buf_ = nullptr; allocator_.reset(); @@ -46,6 +47,7 @@ void ObDirectLoadMultipleDatumRow::reuse() { rowkey_.reuse(); seq_no_.reset(); + is_deleted_ = false; buf_size_ = 0; buf_ = nullptr; allocator_.reuse(); @@ -77,6 +79,7 @@ int ObDirectLoadMultipleDatumRow::deep_copy(const ObDirectLoadMultipleDatumRow & } else { buf_size_ = src.buf_size_; seq_no_ = src.seq_no_; + is_deleted_ = src.is_deleted_; buf_ = buf + pos; MEMCPY(buf + pos, src.buf_, buf_size_); pos += buf_size_; @@ -86,7 +89,7 @@ int ObDirectLoadMultipleDatumRow::deep_copy(const ObDirectLoadMultipleDatumRow & } int ObDirectLoadMultipleDatumRow::from_datums(const ObTabletID &tablet_id, ObStorageDatum *datums, - int64_t column_count, int64_t rowkey_column_count, const ObTableLoadSequenceNo &seq_no) + int64_t column_count, int64_t rowkey_column_count, const ObTableLoadSequenceNo &seq_no, const bool is_deleted) { OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, transfer_external_row_time_us); int ret = OB_SUCCESS; @@ -98,6 +101,7 @@ int ObDirectLoadMultipleDatumRow::from_datums(const ObTabletID &tablet_id, ObSto } else { reuse(); seq_no_ = seq_no; + is_deleted_ = is_deleted; ObDirectLoadDatumArray serialize_datum_array; if (OB_FAIL(rowkey_.assign(tablet_id, datums, rowkey_column_count))) { LOG_WARN("fail to assign rowkey", KR(ret)); @@ -171,7 +175,7 @@ OB_DEF_SERIALIZE_SIMPLE(ObDirectLoadMultipleDatumRow) { OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_row_serialize_time_us); int ret = OB_SUCCESS; - LST_DO_CODE(OB_UNIS_ENCODE, rowkey_, seq_no_, buf_size_); + LST_DO_CODE(OB_UNIS_ENCODE, rowkey_, seq_no_, is_deleted_, buf_size_); if (OB_SUCC(ret) && OB_NOT_NULL(buf_)) { MEMCPY(buf + pos, buf_, buf_size_); pos += buf_size_; @@ -184,7 +188,7 @@ OB_DEF_DESERIALIZE_SIMPLE(ObDirectLoadMultipleDatumRow) OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_row_deserialize_time_us); int ret = OB_SUCCESS; reuse(); - LST_DO_CODE(OB_UNIS_DECODE, rowkey_, seq_no_, buf_size_); + LST_DO_CODE(OB_UNIS_DECODE, rowkey_, seq_no_, is_deleted_, buf_size_); if (OB_SUCC(ret)) { buf_ = buf + pos; pos += buf_size_; @@ -196,7 +200,7 @@ OB_DEF_SERIALIZE_SIZE_SIMPLE(ObDirectLoadMultipleDatumRow) { OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_row_serialize_time_us); int64_t len = 0; - LST_DO_CODE(OB_UNIS_ADD_LEN, rowkey_, seq_no_, buf_size_); + LST_DO_CODE(OB_UNIS_ADD_LEN, rowkey_, seq_no_, is_deleted_, buf_size_); len += buf_size_; return len; } diff --git a/src/storage/direct_load/ob_direct_load_multiple_datum_row.h b/src/storage/direct_load/ob_direct_load_multiple_datum_row.h index ada48ac2e..8c34e937d 100644 --- a/src/storage/direct_load/ob_direct_load_multiple_datum_row.h +++ b/src/storage/direct_load/ob_direct_load_multiple_datum_row.h @@ -33,7 +33,7 @@ public: // not deep copy int from_datums(const common::ObTabletID &tablet_id, blocksstable::ObStorageDatum *datums, int64_t column_count, int64_t rowkey_column_count, - const table::ObTableLoadSequenceNo &seq_no); + const table::ObTableLoadSequenceNo &seq_no, const bool is_deleted); int to_datums(blocksstable::ObStorageDatum *datums, int64_t column_count) const; OB_INLINE bool is_valid() const { @@ -46,6 +46,7 @@ public: common::ObArenaAllocator allocator_; ObDirectLoadMultipleDatumRowkey rowkey_; table::ObTableLoadSequenceNo seq_no_; + bool is_deleted_; int64_t buf_size_; const char *buf_; }; diff --git a/src/storage/direct_load/ob_direct_load_multiple_sstable_builder.cpp b/src/storage/direct_load/ob_direct_load_multiple_sstable_builder.cpp index 33498744c..01a5e8627 100644 --- a/src/storage/direct_load/ob_direct_load_multiple_sstable_builder.cpp +++ b/src/storage/direct_load/ob_direct_load_multiple_sstable_builder.cpp @@ -165,7 +165,7 @@ int ObDirectLoadMultipleSSTableBuilder::append_row(const ObTabletID &tablet_id, LOG_WARN("invalid args", KR(ret), K(param_), K(datum_row)); } else { if (OB_FAIL(row_.from_datums(tablet_id, datum_row.storage_datums_, datum_row.count_, - param_.table_data_desc_.rowkey_column_num_, seq_no))) { + param_.table_data_desc_.rowkey_column_num_, seq_no, datum_row.row_flag_.is_delete()))) { LOG_WARN("fail to from datum row", KR(ret)); } else if (OB_FAIL(append_row(row_))) { LOG_WARN("fail to append row", KR(ret), K(row_)); diff --git a/src/storage/direct_load/ob_direct_load_multiple_sstable_scan_merge.cpp b/src/storage/direct_load/ob_direct_load_multiple_sstable_scan_merge.cpp index 4ef0b631b..e5d4c8336 100644 --- a/src/storage/direct_load/ob_direct_load_multiple_sstable_scan_merge.cpp +++ b/src/storage/direct_load/ob_direct_load_multiple_sstable_scan_merge.cpp @@ -286,6 +286,7 @@ int ObDirectLoadMultipleSSTableScanMerge::get_next_row(const ObDatumRow *&datum_ multiple_datum_row->to_datums(datum_row_.storage_datums_, datum_row_.count_))) { LOG_WARN("fail to transfer datum row", KR(ret)); } else { + datum_row_.row_flag_.set_flag(multiple_datum_row->is_deleted_? ObDmlFlag::DF_DELETE : ObDmlFlag::DF_INSERT); datum_row = &datum_row_; } } diff --git a/src/storage/direct_load/ob_direct_load_partition_merge_task.cpp b/src/storage/direct_load/ob_direct_load_partition_merge_task.cpp index 28db1d2f6..b49f304c2 100644 --- a/src/storage/direct_load/ob_direct_load_partition_merge_task.cpp +++ b/src/storage/direct_load/ob_direct_load_partition_merge_task.cpp @@ -301,6 +301,8 @@ int ObDirectLoadPartitionRangeMergeTask::RowIterator::inner_get_next_row(ObDatum i < datum_row->count_; ++i, ++j) { datum_row_.storage_datums_[j] = datum_row->storage_datums_[i]; } + datum_row_.row_flag_.set_flag(datum_row->row_flag_.is_delete() ? blocksstable::DF_DELETE : blocksstable::DF_INSERT, + datum_row->row_flag_.is_delete() || !insert_tablet_ctx_->is_incremental() ? DF_TYPE_NORMAL : DF_TYPE_INSERT_DELETE); result_row = &datum_row_; } } @@ -550,6 +552,8 @@ int ObDirectLoadPartitionRangeMultipleMergeTask::RowIterator::inner_get_next_row i < datum_row->count_; ++i, ++j) { datum_row_.storage_datums_[j] = datum_row->storage_datums_[i]; } + datum_row_.row_flag_.set_flag(datum_row->row_flag_.is_delete() ? blocksstable::DF_DELETE : blocksstable::DF_INSERT, + datum_row->row_flag_.is_delete() || !insert_tablet_ctx_->is_incremental() ? DF_TYPE_NORMAL : DF_TYPE_INSERT_DELETE); result_row = &datum_row_; } } @@ -716,6 +720,7 @@ int ObDirectLoadPartitionHeapTableMergeTask::RowIterator::init( deserialize_datum_cnt_ = merge_param.store_column_count_ - merge_param.rowkey_column_num_; pk_interval_ = pk_interval; dml_row_handler_ = merge_param.dml_row_handler_; + tablet_id_ = tablet_id; is_inited_ = true; } } @@ -747,7 +752,7 @@ int ObDirectLoadPartitionHeapTableMergeTask::RowIterator::inner_get_next_row( result_row = &datum_row_; } if (OB_SUCC(ret)) { - if (OB_FAIL(dml_row_handler_->handle_insert_row(*result_row))) { + if (OB_FAIL(dml_row_handler_->handle_insert_row_with_multi_version(tablet_id_, *result_row))) { LOG_WARN("fail to handle insert row", KR(ret), KPC(result_row)); } } @@ -873,6 +878,7 @@ int ObDirectLoadPartitionHeapTableMultipleMergeTask::RowIterator::init( deserialize_datum_cnt_ = merge_param.store_column_count_ - merge_param.rowkey_column_num_; pk_interval_ = pk_interval; dml_row_handler_ = merge_param.dml_row_handler_; + tablet_id_ = tablet_id; is_inited_ = true; } } @@ -905,7 +911,7 @@ int ObDirectLoadPartitionHeapTableMultipleMergeTask::RowIterator::inner_get_next result_row = &datum_row_; } if (OB_SUCC(ret)) { - if (OB_FAIL(dml_row_handler_->handle_insert_row(*result_row))) { + if (OB_FAIL(dml_row_handler_->handle_insert_row_with_multi_version(tablet_id_, *result_row))) { LOG_WARN("fail to handle insert row", KR(ret), KPC(result_row)); } } @@ -1107,6 +1113,8 @@ int ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::RowIterator::inner i < datum_row->count_; ++i, ++j) { datum_row_.storage_datums_[j] = datum_row->storage_datums_[i]; } + datum_row_.row_flag_.set_flag(datum_row->row_flag_.is_delete() ? blocksstable::DF_DELETE : blocksstable::DF_INSERT, + datum_row->row_flag_.is_delete() || !insert_tablet_ctx_->is_incremental() ? DF_TYPE_NORMAL : DF_TYPE_INSERT_DELETE); result_row = &datum_row_; } } @@ -1137,7 +1145,7 @@ int ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::RowIterator::inner result_row = &datum_row_; } if (OB_SUCC(ret) && nullptr != result_row) { - if (OB_FAIL(dml_row_handler_->handle_insert_row(*result_row))) { + if (OB_FAIL(dml_row_handler_->handle_insert_row_with_multi_version(tablet_id_, *result_row))) { LOG_WARN("fail to handle insert row", KR(ret), KPC(result_row)); } } diff --git a/src/storage/direct_load/ob_direct_load_partition_merge_task.h b/src/storage/direct_load/ob_direct_load_partition_merge_task.h index 8308f4d3f..0f8ecbe30 100644 --- a/src/storage/direct_load/ob_direct_load_partition_merge_task.h +++ b/src/storage/direct_load/ob_direct_load_partition_merge_task.h @@ -189,6 +189,7 @@ private: int64_t deserialize_datum_cnt_; share::ObTabletCacheInterval pk_interval_; ObDirectLoadDMLRowHandler *dml_row_handler_; + ObTabletID tablet_id_; }; private: ObDirectLoadExternalTable *external_table_; @@ -229,6 +230,7 @@ private: int64_t deserialize_datum_cnt_; share::ObTabletCacheInterval pk_interval_; ObDirectLoadDMLRowHandler *dml_row_handler_; + ObTabletID tablet_id_; }; private: ObDirectLoadMultipleHeapTable *heap_table_; diff --git a/src/storage/direct_load/ob_direct_load_sstable_builder.cpp b/src/storage/direct_load/ob_direct_load_sstable_builder.cpp index 369e5df90..c62084937 100644 --- a/src/storage/direct_load/ob_direct_load_sstable_builder.cpp +++ b/src/storage/direct_load/ob_direct_load_sstable_builder.cpp @@ -102,7 +102,7 @@ int ObDirectLoadSSTableBuilder::append_row(const ObTabletID &tablet_id, const ta if (OB_FAIL(check_rowkey_order(key))) { LOG_WARN("fail to check rowkey order", KR(ret), K(datum_row)); } else if (OB_FAIL(external_row.from_datums(datum_row.storage_datums_, datum_row.count_, - param_.table_data_desc_.rowkey_column_num_, seq_no))) { + param_.table_data_desc_.rowkey_column_num_, seq_no, datum_row.row_flag_.is_delete()))) { LOG_WARN("fail to from datum row", KR(ret)); } else if (OB_FAIL(data_block_writer_.append_row(external_row))) { LOG_WARN("fail to append row to data block writer", KR(ret), K(external_row)); diff --git a/src/storage/direct_load/ob_direct_load_sstable_scan_merge.cpp b/src/storage/direct_load/ob_direct_load_sstable_scan_merge.cpp index 48af4a83a..13ccde4be 100644 --- a/src/storage/direct_load/ob_direct_load_sstable_scan_merge.cpp +++ b/src/storage/direct_load/ob_direct_load_sstable_scan_merge.cpp @@ -289,6 +289,7 @@ int ObDirectLoadSSTableScanMerge::get_next_row(const ObDatumRow *&datum_row) } else if (OB_FAIL(external_row->to_datums(datum_row_.storage_datums_, datum_row_.count_))) { LOG_WARN("fail to transfer datum row", KR(ret)); } else { + datum_row_.row_flag_.set_flag(external_row->is_deleted() ? DF_DELETE : DF_INSERT); datum_row = &datum_row_; } } diff --git a/src/storage/direct_load/ob_direct_load_sstable_scanner.cpp b/src/storage/direct_load/ob_direct_load_sstable_scanner.cpp index bce1164ac..ab888893c 100644 --- a/src/storage/direct_load/ob_direct_load_sstable_scanner.cpp +++ b/src/storage/direct_load/ob_direct_load_sstable_scanner.cpp @@ -558,6 +558,7 @@ int ObDirectLoadSSTableScanner::get_next_row(const ObDatumRow *&datum_row) } else if (OB_FAIL(external_row->to_datums(datum_row_.storage_datums_, datum_row_.count_))) { LOG_WARN("fail to transfer datum row", KR(ret)); } else { + datum_row_.row_flag_.set_flag(external_row->is_deleted() ? DF_DELETE: DF_INSERT); datum_row = &datum_row_; } }