hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FEAT MERGE]inc direct load support local index

Browse Source 
Co-authored-by: medcll <527998250@qq.com>
Co-authored-by: coolfishchen <coolfishchen@gmail.com>
Co-authored-by: suz-yang <suz.yang@foxmail.com>
This commit is contained in:
obdev 2024-09-30 07:16:12 +00:00 committed by ob-robot
parent 003f27aca1
commit 13f9f3dea8
69 changed files with 2298 additions and 661 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/observer/CMakeLists.txt
View File

@ -273,6 +273,12 @@ ob_set_subtarget(ob_server table_load
table_load/ob_table_load_multiple_heap_table_compactor.cpp
table_load/ob_table_load_redef_table.cpp
table_load/ob_table_load_stat.cpp
table_load/ob_table_load_index_table_projector.cpp
table_load/ob_table_load_store_table_ctx.cpp
table_load/ob_table_load_merger_manager.cpp
table_load/ob_table_load_index_row_handler.cpp
table_load/ob_table_load_data_row_handler.cpp
table_load/ob_table_load_open_insert_table_ctx_manager.cpp
table_load/ob_table_load_pre_sorter.cpp
table_load/ob_table_load_mem_chunk_manager.cpp
table_load/ob_table_load_pre_sort_writer.cpp

31
src/observer/table_load/ob_table_load_coordinator.cpp
View File

@ -295,6 +295,7 @@ int ObTableLoadCoordinator::gen_apply_arg(ObDirectLoadResourceApplyArg &apply_ar
} else {
bool include_cur_addr = false;
bool need_sort = ctx_->param_.need_sort_;
bool main_need_sort = ctx_->param_.need_sort_;
int64_t total_partitions = 0;
ObArray<int64_t> partitions;
int64_t store_server_count = all_leader_info_array.count();
@ -374,41 +375,47 @@ int ObTableLoadCoordinator::gen_apply_arg(ObDirectLoadResourceApplyArg &apply_ar
// 直接写宏块需要的内存，对于非排序模式，每个分区各自写宏块，所以要乘分区数
min_unsort_memory = MACROBLOCK_BUFFER_SIZE * partitions[i] * write_session_count;
if (min_unsort_memory <= memory_limit) {
need_sort = false;
main_need_sort = false;
unit.memory_size_ = min_unsort_memory;
} else {
need_sort = ctx_->param_.need_sort_; // allow forced non-sorting
main_need_sort = ctx_->param_.need_sort_; // allow forced non-sorting
unit.memory_size_ = MIN(ObTableLoadAssignedMemoryManager::MIN_SORT_MEMORY_PER_TASK, memory_limit);
}
} else {
// 取写宏块或写临时文件需要内存的最小值，对于非排序模式，每个分区各自写临时文件，所以要乘分区数
min_unsort_memory = SSTABLE_BUFFER_SIZE * partitions[i] * unit.thread_count_;
if (need_sort) {
if (main_need_sort) {
unit.memory_size_ = MIN(unit.thread_count_ * ObDirectLoadExternalMultiPartitionRowChunk::MIN_MEMORY_LIMIT * 4, memory_limit);
} else {
// hint指定不排序，如果不排序内存大于内存上限，要改成走排序模式，一般是分区数较大的场景
if (min_unsort_memory < memory_limit) {
unit.memory_size_ = MAX(min_unsort_memory, MACROBLOCK_BUFFER_SIZE * write_session_count);
} else {
need_sort = true;
main_need_sort = true;
unit.memory_size_ = MIN(unit.thread_count_ * ObDirectLoadExternalMultiPartitionRowChunk::MIN_MEMORY_LIMIT * 4, memory_limit);
}
}
}
if (need_sort) {
// 只要有一个节点走排序模式，所有节点统一走排序模式
if (main_need_sort) {
break;
}
}
if (need_sort) {
// 排序模式，所有节点都分配固定的最小排序内存
ObSchemaGetterGuard schema_guard;
const ObTableSchema *table_schema = nullptr;
if (OB_FAIL(ObTableLoadSchema::get_table_schema(tenant_id, ctx_->ddl_param_.dest_table_id_, schema_guard, table_schema))) {
LOG_WARN("fail to get table schema", KR(ret), K(tenant_id), K(ctx_->ddl_param_.dest_table_id_));
} else if (main_need_sort || (ObDirectLoadMethod::is_incremental(ctx_->param_.method_) &&
table_schema->get_simple_index_infos().count() > 0)) {
need_sort = true;
for (int64_t i = 0; i < store_server_count; i++) {
ObDirectLoadResourceUnit &unit = apply_arg.apply_array_[i];
unit.memory_size_ = MIN(unit.thread_count_ * ObDirectLoadExternalMultiPartitionRowChunk::MIN_MEMORY_LIMIT * 4, memory_limit);
}
} else {
need_sort = false;
}
}
if (OB_SUCC(ret)) {
ObDirectLoadResourceOpRes apply_res;
if (OB_FAIL(ObTableLoadResourceService::apply_resource(apply_arg, apply_res))) {
@ -425,8 +432,8 @@ int ObTableLoadCoordinator::gen_apply_arg(ObDirectLoadResourceApplyArg &apply_ar
ctx_->param_.session_count_ = coordinator_session_count;
ctx_->param_.write_session_count_ = write_session_count;
ctx_->param_.exe_mode_ = (ctx_->schema_.is_heap_table_ ?
(need_sort ? ObTableLoadExeMode::MULTIPLE_HEAP_TABLE_COMPACT : ObTableLoadExeMode::FAST_HEAP_TABLE) :
(need_sort ? ObTableLoadExeMode::MEM_COMPACT : ObTableLoadExeMode::GENERAL_TABLE_COMPACT));
(main_need_sort ? ObTableLoadExeMode::MULTIPLE_HEAP_TABLE_COMPACT : ObTableLoadExeMode::FAST_HEAP_TABLE) :
(main_need_sort ? ObTableLoadExeMode::MEM_COMPACT : ObTableLoadExeMode::GENERAL_TABLE_COMPACT));
ctx_->job_stat_->parallel_ = coordinator_session_count;
if (OB_FAIL(ObTableLoadService::add_assigned_task(apply_arg))) {
LOG_WARN("fail to add_assigned_task", KR(ret));
@ -1709,7 +1716,7 @@ public:
LOG_WARN("column count doesn't match value count", KR(ret), K(src_obj_row),
K(ctx_->param_.column_count_));
ObNewRow new_row(src_obj_row.cells_, src_obj_row.count_);
if (OB_FAIL(error_row_handler->handle_error_row(ret, new_row))) {
if (OB_FAIL(error_row_handler->handle_error_row(ret))) {
LOG_WARN("fail to handle error row", KR(ret));
}
} else if (OB_FAIL(src_obj_row.project(idx_array, out_obj_row))) {

293
src/observer/table_load/ob_table_load_data_row_handler.cpp Normal file
View File

@ -0,0 +1,293 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SERVER
#include "observer/table_load/ob_table_load_data_row_handler.h"
#include "observer/table_load/ob_table_load_error_row_handler.h"
#include "observer/table_load/ob_table_load_store_table_ctx.h"
#include "observer/table_load/ob_table_load_schema.h"
#include "observer/table_load/ob_table_load_index_table_projector.h"
#include "storage/direct_load/ob_direct_load_i_table.h"
namespace oceanbase
{
namespace observer
{
ObTableLoadDataRowHandler::ObTableLoadDataRowHandler()
: error_row_handler_(nullptr),
index_store_table_ctx_map_(nullptr),
result_info_(nullptr),
is_inited_(false)
{
}
ObTableLoadDataRowHandler::~ObTableLoadDataRowHandler()
{
}
int ObTableLoadDataRowHandler::init(
const ObTableLoadParam &param, table::ObTableLoadResultInfo &result_info,
ObTableLoadErrorRowHandler *error_row_handler,
hash::ObHashMap<ObTableID, ObTableLoadStoreTableCtx *> *index_store_table_ctx_map)
{
int ret = OB_SUCCESS;
if (IS_INIT) {
int ret = OB_INIT_TWICE;
LOG_WARN("ObTableLoadDataRowHandler init twice", KR(ret), KP(this));
} else if (OB_ISNULL(error_row_handler) || OB_ISNULL(index_store_table_ctx_map)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("allocator is null", KR(ret), KP(error_row_handler), KP(index_store_table_ctx_map));
} else {
error_row_handler_ = error_row_handler;
index_store_table_ctx_map_ = index_store_table_ctx_map;
result_info_ = &result_info;
dup_action_ = param.dup_action_;
is_inited_ = true;
}
return ret;
}
int ObTableLoadDataRowHandler::handle_insert_row(const ObTabletID tablet_id,
const blocksstable::ObDatumRow &row)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadDataRowHandler not init", KR(ret), KP(this));
} else {
FOREACH_X(iter, *index_store_table_ctx_map_, OB_SUCC(ret))
{
ObTabletID index_tablet_id;
blocksstable::ObDatumRow index_row;
ObIDirectLoadPartitionTableBuilder *builder = nullptr;
if (OB_FAIL(
iter->second->project_->projector(tablet_id, row, false, index_tablet_id, index_row))) {
LOG_WARN("fail to projector", KR(ret), K(tablet_id), K(row));
} else {
index_row.row_flag_.set_flag(blocksstable::DF_INSERT,
blocksstable::ObDmlRowFlagType::DF_TYPE_INSERT_DELETE);
if (OB_FAIL(iter->second->get_index_table_builder(builder))) {
LOG_WARN("get builder failed", KR(ret));
} else if (OB_FAIL(builder->append_row(index_tablet_id, 0, index_row))) {
LOG_WARN("add row failed", KR(ret), K(index_row));
}
}
}
if (OB_SUCC(ret)) {
ATOMIC_INC(&result_info_->rows_affected_);
}
}
return ret;
}
int ObTableLoadDataRowHandler::handle_insert_row_with_multi_version(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadDataRowHandler not init", KR(ret), KP(this));
} else {
FOREACH_X(iter, *index_store_table_ctx_map_, OB_SUCC(ret))
{
ObTabletID index_tablet_id;
blocksstable::ObDatumRow index_row;
ObIDirectLoadPartitionTableBuilder *builder = nullptr;
if (OB_FAIL(
iter->second->project_->projector(tablet_id, row, true, index_tablet_id, index_row))) {
LOG_WARN("fail to projector", KR(ret), K(tablet_id), K(row));
} else {
index_row.row_flag_.set_flag(blocksstable::DF_INSERT,
blocksstable::ObDmlRowFlagType::DF_TYPE_INSERT_DELETE);
if (OB_FAIL(iter->second->get_index_table_builder(builder))) {
LOG_WARN("get builder failed", KR(ret));
} else if (OB_FAIL(builder->append_row(index_tablet_id, 0, index_row))) {
LOG_WARN("add row failed", KR(ret), K(index_row));
}
}
}
if (OB_SUCC(ret)) {
ATOMIC_INC(&result_info_->rows_affected_);
}
}
return ret;
}
int ObTableLoadDataRowHandler::handle_update_row(const blocksstable::ObDatumRow &row)
{
UNUSED(row);
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else {
if (ObLoadDupActionType::LOAD_STOP_ON_DUP == dup_action_) {
if (OB_FAIL(error_row_handler_->handle_error_row(OB_ERR_PRIMARY_KEY_DUPLICATE))) {
LOG_WARN("fail to handle error row", KR(ret));
}
} else if (ObLoadDupActionType::LOAD_REPLACE == dup_action_) {
ATOMIC_AAF(&result_info_->rows_affected_, 2);
ATOMIC_INC(&result_info_->deleted_);
} else if (ObLoadDupActionType::LOAD_IGNORE == dup_action_) {
ATOMIC_INC(&result_info_->skipped_);
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected dup action", KR(ret), K_(dup_action));
}
}
return ret;
}
int ObTableLoadDataRowHandler::handle_update_row(const ObTabletID tablet_id,
const blocksstable::ObDatumRow &old_row,
const blocksstable::ObDatumRow &new_row,
const blocksstable::ObDatumRow *&result_row)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadDataRowHandler not init", KR(ret), KP(this));
} else {
if (ObLoadDupActionType::LOAD_STOP_ON_DUP == dup_action_) {
if (OB_FAIL(error_row_handler_->handle_error_row(OB_ERR_PRIMARY_KEY_DUPLICATE))) {
LOG_WARN("fail to handle error row", KR(ret));
} else {
result_row = &old_row;
}
} else if (ObLoadDupActionType::LOAD_IGNORE == dup_action_) {
result_row = &old_row;
ATOMIC_INC(&result_info_->skipped_);
} else if (ObLoadDupActionType::LOAD_REPLACE == dup_action_) {
result_row = &new_row;
ATOMIC_INC(&result_info_->deleted_);
ATOMIC_AAF(&result_info_->rows_affected_, 2);
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected dup action", KR(ret), K_(dup_action));
}
if (OB_SUCC(ret) && result_row == &new_row) {
FOREACH_X(iter, *index_store_table_ctx_map_, OB_SUCC(ret)) {
ObTabletID index_tablet_id;
blocksstable::ObDatumRow index_row_old;
blocksstable::ObDatumRow index_row_new;
ObIDirectLoadPartitionTableBuilder *builder = nullptr;
if (OB_FAIL(iter->second->project_->projector(tablet_id, old_row, false, index_tablet_id,
index_row_old))) {
LOG_WARN("fail to projector", KR(ret), K(tablet_id), K(old_row));
} else if (OB_FAIL(iter->second->project_->projector(tablet_id, new_row, false,
index_tablet_id, index_row_new))) {
LOG_WARN("fail to projector", KR(ret), K(tablet_id), K(new_row));
} else if (OB_FAIL(iter->second->get_index_table_builder(builder))) {
LOG_WARN("get builder failed", KR(ret));
} else {
ObDatumRowkey old_key(index_row_old.storage_datums_,
iter->second->schema_->rowkey_column_count_);
ObDatumRowkey new_key(index_row_new.storage_datums_,
iter->second->schema_->rowkey_column_count_);
int cmp_ret;
if (OB_FAIL(old_key.compare(new_key, iter->second->schema_->datum_utils_, cmp_ret))) {
LOG_WARN("fail to compare", KR(ret));
} else {
// when cmp_ret == 0, insert new row is equal to delete old row and insert new row.
if (0 != cmp_ret) {
index_row_old.row_flag_.set_flag(blocksstable::DF_DELETE,
blocksstable::ObDmlRowFlagType::DF_TYPE_NORMAL);
if (OB_FAIL(builder->append_row(index_tablet_id, 0, index_row_old))) {
LOG_WARN("add row failed", KR(ret), K(index_row_old));
}
}
}
if (OB_SUCC(ret)) {
index_row_new.row_flag_.set_flag(blocksstable::DF_INSERT,
blocksstable::ObDmlRowFlagType::DF_TYPE_INSERT_DELETE);
if (OB_FAIL(builder->append_row(index_tablet_id, 0, index_row_new))) {
LOG_WARN("add row failed", KR(ret), K(index_row_new));
}
}
}
}
}
}
return ret;
}
int ObTableLoadDataRowHandler::handle_update_row(common::ObArray<const ObDirectLoadExternalRow *> &rows,
const ObDirectLoadExternalRow *&row)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_UNLIKELY(rows.count() < 2)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret));
} else {
int64_t duplicate_row_count = rows.count() - 1;
lib::ob_sort(rows.begin(), rows.end(), ObTableLoadDataRowHandler::external_row_compare);
if (ObLoadDupActionType::LOAD_STOP_ON_DUP == dup_action_) {
if (OB_FAIL(error_row_handler_->handle_error_row(OB_ERR_PRIMARY_KEY_DUPLICATE, duplicate_row_count))) {
LOG_WARN("fail to handle error row", KR(ret));
} else {
row = rows.at(0);
}
} else if (ObLoadDupActionType::LOAD_REPLACE == dup_action_) {
ATOMIC_AAF(&result_info_->rows_affected_, 2 * duplicate_row_count);
ATOMIC_AAF(&result_info_->deleted_, duplicate_row_count);
row = rows.at(duplicate_row_count);
} else if (ObLoadDupActionType::LOAD_IGNORE == dup_action_) {
ATOMIC_AAF(&result_info_->skipped_, duplicate_row_count);
row = rows.at(0);
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected dup action", KR(ret), K_(dup_action));
}
}
return ret;
}
int ObTableLoadDataRowHandler::handle_update_row(
common::ObArray<const ObDirectLoadMultipleDatumRow *> &rows,
const ObDirectLoadMultipleDatumRow *&row)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_UNLIKELY(rows.count() < 2)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret));
} else {
int64_t duplicate_row_count = rows.count() - 1;
lib::ob_sort(rows.begin(), rows.end(), ObTableLoadDataRowHandler::multiple_external_row_compare);
if (ObLoadDupActionType::LOAD_STOP_ON_DUP == dup_action_) {
if (OB_FAIL(error_row_handler_->handle_error_row(OB_ERR_PRIMARY_KEY_DUPLICATE, duplicate_row_count))) {
LOG_WARN("fail to handle error row", KR(ret));
} else {
row = rows.at(0);
}
} else if (ObLoadDupActionType::LOAD_REPLACE == dup_action_) {
ATOMIC_AAF(&result_info_->rows_affected_, 2 * duplicate_row_count);
ATOMIC_AAF(&result_info_->deleted_, duplicate_row_count);
row = rows.at(duplicate_row_count);
} else if (ObLoadDupActionType::LOAD_IGNORE == dup_action_) {
ATOMIC_AAF(&result_info_->skipped_, duplicate_row_count);
row = rows.at(0);
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected dup action", KR(ret), K_(dup_action));
}
}
return ret;
}
} // namespace observer
} // namespace oceanbase

76
src/observer/table_load/ob_table_load_data_row_handler.h Normal file
View File

@ -0,0 +1,76 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include "storage/blocksstable/ob_datum_row.h"
#include "sql/engine/cmd/ob_load_data_utils.h"
#include "storage/direct_load/ob_direct_load_dml_row_handler.h"
namespace oceanbase
{
namespace table
{
class ObTableLoadResultInfo;
} // namespace table
namespace observer
{
class ObTableLoadErrorRowHandler;
class ObTableLoadParam;
class ObTableLoadStoreTableCtx;
class ObTableLoadDataRowHandler : public ObDirectLoadDMLRowHandler
{
private:
static bool external_row_compare(const ObDirectLoadExternalRow *lhs,
const ObDirectLoadExternalRow *rhs)
{
return lhs->seq_no_ < rhs->seq_no_;
}
static bool multiple_external_row_compare(const ObDirectLoadMultipleDatumRow *lhs,
const ObDirectLoadMultipleDatumRow *rhs)
{
return lhs->seq_no_ < rhs->seq_no_;
}
public:
ObTableLoadDataRowHandler();
virtual ~ObTableLoadDataRowHandler();
int init(const ObTableLoadParam &param, table::ObTableLoadResultInfo &result_info,
ObTableLoadErrorRowHandler *error_row_handler,
hash::ObHashMap<ObTableID, ObTableLoadStoreTableCtx *> *index_store_table_ctx_map);
int handle_insert_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) override;
int handle_delete_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row)
{
return OB_ERR_UNEXPECTED;
}
int handle_insert_row_with_multi_version(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) override;
int handle_update_row(const blocksstable::ObDatumRow &row);
int handle_update_row(common::ObArray<const ObDirectLoadExternalRow *> &rows,
const ObDirectLoadExternalRow *&row);
int handle_update_row(common::ObArray<const ObDirectLoadMultipleDatumRow *> &rows,
const ObDirectLoadMultipleDatumRow *&row) override;
int handle_update_row(const ObTabletID tablet_id,
const blocksstable::ObDatumRow &old_row,
const blocksstable::ObDatumRow &new_row,
const blocksstable::ObDatumRow *&result_row);
TO_STRING_KV(KPC_(error_row_handler), K_(result_info), K_(dup_action), K_(is_inited));
private:
ObTableLoadErrorRowHandler * error_row_handler_;
hash::ObHashMap<ObTableID, ObTableLoadStoreTableCtx* > *index_store_table_ctx_map_;
table::ObTableLoadResultInfo *result_info_;
sql::ObLoadDupActionType dup_action_;
bool is_inited_;
};
}
}

200
src/observer/table_load/ob_table_load_error_row_handler.cpp
View File

@ -57,190 +57,12 @@ int ObTableLoadErrorRowHandler::init(const ObTableLoadParam &param,
return ret;
}
int ObTableLoadErrorRowHandler::handle_insert_row(const blocksstable::ObDatumRow &row)
{
UNUSED(row);
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else {
ATOMIC_INC(&result_info_->rows_affected_);
}
return ret;
}
int ObTableLoadErrorRowHandler::handle_update_row(const ObDatumRow &row)
{
UNUSED(row);
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else {
if (ObLoadDupActionType::LOAD_STOP_ON_DUP == dup_action_) {
if (0 == max_error_row_count_) {
ret = OB_ERR_PRIMARY_KEY_DUPLICATE;
} else {
ObMutexGuard guard(mutex_);
if (error_row_count_ >= max_error_row_count_) {
ret = OB_ERR_TOO_MANY_ROWS;
LOG_WARN("error row count reaches its maximum value", KR(ret), K_(max_error_row_count),
K_(error_row_count));
} else {
++error_row_count_;
}
}
ATOMIC_INC(&job_stat_->detected_error_rows_);
} else if (ObLoadDupActionType::LOAD_REPLACE == dup_action_) {
ATOMIC_AAF(&result_info_->rows_affected_, 2);
ATOMIC_INC(&result_info_->deleted_);
} else if (ObLoadDupActionType::LOAD_IGNORE == dup_action_) {
ATOMIC_INC(&result_info_->skipped_);
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected dup action", KR(ret), K_(dup_action));
}
}
return ret;
}
int ObTableLoadErrorRowHandler::handle_update_row(
common::ObArray<const ObDirectLoadExternalRow *> &rows, const ObDirectLoadExternalRow *&row)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_UNLIKELY(rows.count() < 2)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret));
} else {
int64_t duplicate_row_count = rows.count() - 1;
lib::ob_sort(rows.begin(), rows.end(),
[](const ObDirectLoadExternalRow *lhs, const ObDirectLoadExternalRow *rhs) {
return lhs->seq_no_ < rhs->seq_no_;
});
if (ObLoadDupActionType::LOAD_STOP_ON_DUP == dup_action_) {
if (0 == max_error_row_count_) {
ret = OB_ERR_PRIMARY_KEY_DUPLICATE;
} else {
ObMutexGuard guard(mutex_);
error_row_count_ += duplicate_row_count;
if (error_row_count_ >= max_error_row_count_) {
ret = OB_ERR_TOO_MANY_ROWS;
LOG_WARN("error row count reaches its maximum value", KR(ret), K_(max_error_row_count),
K_(error_row_count));
}
}
ATOMIC_AAF(&job_stat_->detected_error_rows_, duplicate_row_count);
row = rows.at(0);
} else if (ObLoadDupActionType::LOAD_REPLACE == dup_action_) {
ATOMIC_AAF(&result_info_->rows_affected_, 2 * duplicate_row_count);
ATOMIC_AAF(&result_info_->deleted_, duplicate_row_count);
row = rows.at(duplicate_row_count);
} else if (ObLoadDupActionType::LOAD_IGNORE == dup_action_) {
ATOMIC_AAF(&result_info_->skipped_, duplicate_row_count);
row = rows.at(0);
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected dup action", KR(ret), K_(dup_action));
}
}
return ret;
}
int ObTableLoadErrorRowHandler::handle_update_row(
common::ObArray<const ObDirectLoadMultipleDatumRow *> &rows,
const ObDirectLoadMultipleDatumRow *&row)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_UNLIKELY(rows.count() < 2)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret));
} else {
int64_t duplicate_row_count = rows.count() - 1;
lib::ob_sort(rows.begin(), rows.end(),
[](const ObDirectLoadMultipleDatumRow *lhs, const ObDirectLoadMultipleDatumRow *rhs) {
return lhs->seq_no_ < rhs->seq_no_;
});
if (ObLoadDupActionType::LOAD_STOP_ON_DUP == dup_action_) {
if (0 == max_error_row_count_) {
ret = OB_ERR_PRIMARY_KEY_DUPLICATE;
} else {
error_row_count_ += duplicate_row_count;
ObMutexGuard guard(mutex_);
if (error_row_count_ >= max_error_row_count_) {
ret = OB_ERR_TOO_MANY_ROWS;
LOG_WARN("error row count reaches its maximum value", KR(ret), K_(max_error_row_count),
K_(error_row_count));
}
}
ATOMIC_AAF(&job_stat_->detected_error_rows_, duplicate_row_count);
row = rows.at(0);
} else if (ObLoadDupActionType::LOAD_REPLACE == dup_action_) {
ATOMIC_AAF(&result_info_->rows_affected_, 2 * duplicate_row_count);
ATOMIC_AAF(&result_info_->deleted_, duplicate_row_count);
row = rows.at(duplicate_row_count);
} else if (ObLoadDupActionType::LOAD_IGNORE == dup_action_) {
ATOMIC_AAF(&result_info_->skipped_, duplicate_row_count);
row = rows.at(0);
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected dup action", KR(ret), K_(dup_action));
}
}
return ret;
}
int ObTableLoadErrorRowHandler::handle_update_row(const ObDatumRow &old_row,
const ObDatumRow &new_row,
const ObDatumRow *&result_row)
int ObTableLoadErrorRowHandler::handle_error_row(int error_code)
{
int ret = OB_SUCCESS;
result_row = nullptr;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else {
if (ObLoadDupActionType::LOAD_STOP_ON_DUP == dup_action_) {
if (0 == max_error_row_count_) {
ret = OB_ERR_PRIMARY_KEY_DUPLICATE;
} else {
ObMutexGuard guard(mutex_);
if (error_row_count_ >= max_error_row_count_) {
ret = OB_ERR_TOO_MANY_ROWS;
LOG_WARN("error row count reaches its maximum value", KR(ret), K_(max_error_row_count),
K_(error_row_count));
} else {
++error_row_count_;
}
}
if (OB_SUCC(ret)) {
result_row = &old_row;
}
ATOMIC_INC(&job_stat_->detected_error_rows_);
} else if (ObLoadDupActionType::LOAD_IGNORE == dup_action_) {
result_row = &old_row;
ATOMIC_INC(&result_info_->skipped_);
} else if (ObLoadDupActionType::LOAD_REPLACE == dup_action_) {
result_row = &new_row;
ATOMIC_INC(&result_info_->deleted_);
ATOMIC_AAF(&result_info_->rows_affected_, 2);
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected dup action", KR(ret), K_(dup_action));
}
}
return ret;
}
int ObTableLoadErrorRowHandler::handle_error_row(int error_code, const ObNewRow &row)
{
UNUSED(row);
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
@ -261,9 +83,8 @@ int ObTableLoadErrorRowHandler::handle_error_row(int error_code, const ObNewRow
return ret;
}
int ObTableLoadErrorRowHandler::handle_error_row(int error_code, const ObDatumRow &row)
int ObTableLoadErrorRowHandler::handle_error_row(int error_code, int64_t duplicate_row_count)
{
UNUSED(row);
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
@ -271,15 +92,18 @@ int ObTableLoadErrorRowHandler::handle_error_row(int error_code, const ObDatumRo
} else if (max_error_row_count_ == 0) {
ret = error_code;
} else {
ObMutexGuard guard(mutex_);
if (error_row_count_ >= max_error_row_count_) {
ret = OB_ERR_TOO_MANY_ROWS;
LOG_WARN("error row count reaches its maximum value", KR(ret), K_(max_error_row_count),
K_(error_row_count));
if (0 == max_error_row_count_) {
ret = OB_ERR_PRIMARY_KEY_DUPLICATE;
} else {
++error_row_count_;
ObMutexGuard guard(mutex_);
error_row_count_ += duplicate_row_count;
if (error_row_count_ > max_error_row_count_) {
ret = OB_ERR_TOO_MANY_ROWS;
LOG_WARN("error row count reaches its maximum value", KR(ret), K_(max_error_row_count),
K_(error_row_count));
}
}
ATOMIC_INC(&job_stat_->detected_error_rows_);
ATOMIC_AAF(&job_stat_->detected_error_rows_, duplicate_row_count);
}
return ret;
}

16
src/observer/table_load/ob_table_load_error_row_handler.h
View File

@ -15,7 +15,6 @@
#include "common/row/ob_row.h"
#include "sql/engine/cmd/ob_load_data_utils.h"
#include "storage/blocksstable/ob_datum_row.h"
#include "storage/direct_load/ob_direct_load_dml_row_handler.h"
namespace oceanbase
{
@ -29,24 +28,15 @@ class ObTableLoadStoreCtx;
class ObTableLoadCoordinatorCtx;
class ObTableLoadParam;
class ObTableLoadErrorRowHandler : public ObDirectLoadDMLRowHandler
class ObTableLoadErrorRowHandler
{
public:
ObTableLoadErrorRowHandler();
virtual ~ObTableLoadErrorRowHandler();
int init(const ObTableLoadParam &param, table::ObTableLoadResultInfo &result_info,
sql::ObLoadDataStat *job_stat);
int handle_insert_row(const blocksstable::ObDatumRow &row) override;
int handle_update_row(const blocksstable::ObDatumRow &row) override;
int handle_update_row(common::ObArray<const ObDirectLoadExternalRow *> &rows,
const ObDirectLoadExternalRow *&row) override;
int handle_update_row(common::ObArray<const ObDirectLoadMultipleDatumRow *> &rows,
const ObDirectLoadMultipleDatumRow *&row) override;
int handle_update_row(const blocksstable::ObDatumRow &old_row,
const blocksstable::ObDatumRow &new_row,
const blocksstable::ObDatumRow *&result_row) override;
int handle_error_row(int error_code, const common::ObNewRow &row);
int handle_error_row(int error_code, const blocksstable::ObDatumRow &row);
int handle_error_row(int error_code);
int handle_error_row(int error_code, int64_t duplicate_row_count);
uint64_t get_error_row_count() const;
TO_STRING_KV(K_(dup_action), K_(max_error_row_count), K_(error_row_count));
private:

14
src/observer/table_load/ob_table_load_general_table_compactor.cpp
View File

@ -22,6 +22,7 @@
#include "observer/table_load/ob_table_load_trans_store.h"
#include "storage/direct_load/ob_direct_load_external_table_compactor.h"
#include "storage/direct_load/ob_direct_load_sstable_compactor.h"
#include "observer/table_load/ob_table_load_store_table_ctx.h"
namespace oceanbase
{
@ -206,6 +207,7 @@ int ObTableLoadGeneralTableCompactor::CompactorTaskIter::get_next_compactor_task
ObTableLoadGeneralTableCompactor::ObTableLoadGeneralTableCompactor()
: store_ctx_(nullptr),
store_table_ctx_(nullptr),
param_(nullptr),
allocator_("TLD_GeneralTC"),
running_thread_count_(0),
@ -225,6 +227,7 @@ void ObTableLoadGeneralTableCompactor::reset()
{
abort_unless(compacting_list_.is_empty());
store_ctx_ = nullptr;
store_table_ctx_ = nullptr;
param_ = nullptr;
for (int64_t i = 0; i < all_compactor_array_.count(); ++i) {
ObIDirectLoadTabletTableCompactor *compactor = all_compactor_array_.at(i);
@ -242,6 +245,7 @@ int ObTableLoadGeneralTableCompactor::inner_init()
{
int ret = OB_SUCCESS;
store_ctx_ = compact_ctx_->store_ctx_;
store_table_ctx_ = compact_ctx_->store_table_ctx_;
param_ = &store_ctx_->ctx_->param_;
return ret;
}
@ -362,10 +366,10 @@ int ObTableLoadGeneralTableCompactor::create_tablet_table_compactor(
{
int ret = OB_SUCCESS;
table_compactor = nullptr;
if (store_ctx_->ctx_->schema_.is_heap_table_) {
if (store_table_ctx_->schema_->is_heap_table_) {
ObDirectLoadExternalTableCompactParam compact_param;
compact_param.tablet_id_ = tablet_id;
compact_param.table_data_desc_ = store_ctx_->table_data_desc_;
compact_param.table_data_desc_ = store_table_ctx_->table_data_desc_;
ObDirectLoadExternalTableCompactor *external_table_compactor = nullptr;
if (OB_ISNULL(external_table_compactor =
OB_NEWx(ObDirectLoadExternalTableCompactor, (&allocator_)))) {
@ -375,11 +379,11 @@ int ObTableLoadGeneralTableCompactor::create_tablet_table_compactor(
LOG_WARN("fail to init external table compactor", KR(ret));
}
table_compactor = external_table_compactor;
} else if (!param_->need_sort_) {
} else if (!store_table_ctx_->need_sort_) {
ObDirectLoadSSTableCompactParam compact_param;
compact_param.tablet_id_ = tablet_id;
compact_param.table_data_desc_ = store_ctx_->table_data_desc_;
compact_param.datum_utils_ = &(store_ctx_->ctx_->schema_.datum_utils_);
compact_param.table_data_desc_ = store_table_ctx_->table_data_desc_;
compact_param.datum_utils_ = &(store_table_ctx_->schema_->datum_utils_);
ObDirectLoadSSTableCompactor *sstable_compactor = nullptr;
if (OB_ISNULL(sstable_compactor = OB_NEWx(ObDirectLoadSSTableCompactor, (&allocator_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;

1
src/observer/table_load/ob_table_load_general_table_compactor.h
View File

@ -75,6 +75,7 @@ private:
int build_result();
private:
ObTableLoadStoreCtx *store_ctx_;
ObTableLoadStoreTableCtx * store_table_ctx_;
const ObTableLoadParam *param_;
common::ObArenaAllocator allocator_;
common::ObArray<storage::ObIDirectLoadTabletTableCompactor *> all_compactor_array_;

42
src/observer/table_load/ob_table_load_index_row_handler.cpp Normal file
View File

@ -0,0 +1,42 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SERVER
#include "observer/table_load/ob_table_load_index_row_handler.h"
namespace oceanbase
{
namespace observer
{
using namespace blocksstable;
ObTableLoadIndexRowHandler::ObTableLoadIndexRowHandler() {}
ObTableLoadIndexRowHandler::~ObTableLoadIndexRowHandler() {}
int ObTableLoadIndexRowHandler::handle_insert_row(const ObTabletID tablet_id,
const blocksstable::ObDatumRow &row)
{
int ret = OB_SUCCESS;
//do nothing
return ret;
}
int ObTableLoadIndexRowHandler::handle_delete_row(const ObTabletID tablet_id,
const blocksstable::ObDatumRow &row)
{
int ret = OB_SUCCESS;
// do nothing
return ret;
}
} // namespace observer
} // namespace oceanbase

57
src/observer/table_load/ob_table_load_index_row_handler.h Normal file
View File

@ -0,0 +1,57 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include "storage/blocksstable/ob_datum_row.h"
#include "storage/direct_load/ob_direct_load_dml_row_handler.h"
namespace oceanbase
{
namespace observer
{
class ObTableLoadStoreTableCtx;
class ObTableLoadIndexRowHandler : public ObDirectLoadDMLRowHandler
{
public:
ObTableLoadIndexRowHandler();
virtual ~ObTableLoadIndexRowHandler();
int handle_insert_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row);
int handle_delete_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row);
int handle_insert_row_with_multi_version(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row)
{
return OB_ERR_UNEXPECTED;
}
int handle_update_row(const blocksstable::ObDatumRow &row) override { return OB_ERR_UNEXPECTED; };
int handle_update_row(common::ObArray<const ObDirectLoadExternalRow *> &rows,
const ObDirectLoadExternalRow *&row) override
{
return OB_ERR_UNEXPECTED;
};
int handle_update_row(common::ObArray<const ObDirectLoadMultipleDatumRow *> &rows,
const ObDirectLoadMultipleDatumRow *&row) override
{
return OB_ERR_UNEXPECTED;
};
int handle_update_row(const ObTabletID tablet_id,
const blocksstable::ObDatumRow &old_row,
const blocksstable::ObDatumRow &new_row,
const blocksstable::ObDatumRow *&result_row) override
{
return OB_ERR_UNEXPECTED;
};
TO_STRING_EMPTY();
};
} // namespace observer
} // namespace oceanbase

196
src/observer/table_load/ob_table_load_index_table_projector.cpp Normal file
View File

@ -0,0 +1,196 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX STORAGE
#include "observer/table_load/ob_table_load_index_table_projector.h"
#include "lib/oblog/ob_log_module.h"
#include "share/rc/ob_tenant_base.h"
#include "storage/ob_i_store.h"
namespace oceanbase
{
namespace observer
{
ObTableLoadIndexTableProjector::~ObTableLoadIndexTableProjector()
{
row_projector_.reset();
tablet_projector_.destroy();
index_tablet_id_to_part_id_map_.destroy();
}
int ObTableLoadIndexTableProjector::init(const share::schema::ObTableSchema *data_table_schema,
const share::schema::ObTableSchema *index_table_schema)
{
int ret = OB_SUCCESS;
if (IS_INIT) {
ret = OB_INIT_TWICE;
LOG_WARN("ObTableLoadIndexTableProjector init twice", KR(ret));
} else if (OB_FAIL(tablet_projector_.create(1024, "TLD_ITP", "TLD_ITP", MTL_ID()))) {
LOG_WARN("fail to create tablet projector", KR(ret));
} else if (OB_FAIL(
index_tablet_id_to_part_id_map_.create(1024, "TLD_ITP", "TLD_ITP", MTL_ID()))) {
LOG_WARN("fail to create index tablet id to part id map", KR(ret));
} else if (OB_FAIL(build_projector(data_table_schema, index_table_schema))) {
LOG_WARN("fail to build projector", KR(ret), KPC(data_table_schema), KPC(index_table_schema));
} else {
is_inited_ = true;
}
return ret;
}
int ObTableLoadIndexTableProjector::get_index_tablet_id_and_part_id_by_data_tablet_id(
const ObTabletID &data_tablet_id, ObTabletID &index_tablet_id, ObObjectID &part_id)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadIndexTableProjector not init", KR(ret));
} else if (OB_FAIL(tablet_projector_.get_refactored(data_tablet_id, index_tablet_id))) {
LOG_WARN("fail to get index tablet id", KR(ret), K(data_tablet_id));
} else if (OB_FAIL(index_tablet_id_to_part_id_map_.get_refactored(index_tablet_id, part_id))) {
LOG_WARN("fail to get index tablet id", KR(ret), K(index_tablet_id));
}
return ret;
}
int ObTableLoadIndexTableProjector::build_projector(
const share::schema::ObTableSchema *data_table_schema,
const share::schema::ObTableSchema *index_table_schema)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(data_table_schema) || OB_ISNULL(index_table_schema)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), KP(data_table_schema), KP(index_table_schema));
} else if (OB_FAIL(build_row_projector(data_table_schema, index_table_schema))) {
LOG_WARN("fail to build row projector", KR(ret));
} else if (OB_FAIL(build_tablet_projector(data_table_schema, index_table_schema))) {
LOG_WARN("fail to build tablet projector", KR(ret));
} else {
column_num_ = index_table_schema->get_column_count();
main_table_rowkey_column_num_ = data_table_schema->get_rowkey_column_num();
}
return ret;
}
int ObTableLoadIndexTableProjector::build_row_projector(
const share::schema::ObTableSchema *data_table_schema,
const share::schema::ObTableSchema *index_table_schema)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(data_table_schema) || OB_ISNULL(index_table_schema)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), KP(data_table_schema), KP(index_table_schema));
} else {
common::ObArray<share::schema::ObColDesc> main_column_descs;
common::ObArray<share::schema::ObColDesc> index_column_descs;
if (OB_FAIL(data_table_schema->get_column_ids(main_column_descs, false))) {
LOG_WARN("fail to get column ids", KR(ret));
} else if (OB_FAIL(index_table_schema->get_column_ids(index_column_descs, false))) {
LOG_WARN("fail to get column ids", KR(ret));
} else {
FOREACH_X(iter, index_column_descs, OB_SUCC(ret)) {
share::schema::ObColDesc index_col_desc = *iter;
for (int64_t i = 0; OB_SUCC(ret) && i < main_column_descs.count(); i++) {
if (index_col_desc.col_id_ == main_column_descs.at(i).col_id_) {
if (OB_FAIL(row_projector_.push_back(i))) {
LOG_WARN("fail to push back", KR(ret), K(i));
}
}
}
}
}
}
return ret;
}
int ObTableLoadIndexTableProjector::build_tablet_projector(
const share::schema::ObTableSchema *data_table_schema,
const share::schema::ObTableSchema *index_table_schema)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(data_table_schema) || OB_ISNULL(index_table_schema)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), KP(data_table_schema), KP(index_table_schema));
} else {
ObArray<ObTabletID> tablet_ids;
if (OB_FAIL(data_table_schema->get_tablet_ids(tablet_ids))) {
LOG_WARN("fail to get tablet ids", KR(ret));
} else {
int64_t main_part_idx = OB_INVALID_ID;
int64_t main_subpart_idx = OB_INVALID_ID;
ObObjectID index_part_id = OB_INVALID_ID;
ObObjectID index_subpart_id = OB_INVALID_ID;
ObTabletID index_tablet_id(ObTabletID::INVALID_TABLET_ID);
for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); i++) {
if (data_table_schema->is_partitioned_table() &&
OB_FAIL(data_table_schema->get_part_idx_by_tablet(tablet_ids.at(i), main_part_idx,
main_subpart_idx))) {
LOG_WARN("fail to get part idx by tablet", KR(ret), K(tablet_ids.at(i)));
} else if (OB_FAIL(index_table_schema->get_part_id_and_tablet_id_by_idx(
main_part_idx, main_subpart_idx, index_part_id, index_subpart_id,
index_tablet_id))) {
LOG_WARN("fail to get index tablet id", KR(ret), K(main_part_idx), K(main_subpart_idx));
} else if (OB_FAIL(tablet_projector_.set_refactored(tablet_ids.at(i), index_tablet_id))) {
LOG_WARN("fail to add tablet projector", KR(ret), K(index_tablet_id),
K(tablet_ids.at(i)));
} else if (OB_FAIL(index_tablet_id_to_part_id_map_.set_refactored(index_tablet_id,
index_part_id))) {
LOG_WARN("fail to add index tablet id to part id map", KR(ret), K(index_tablet_id),
K(index_part_id));
}
}
}
}
return ret;
}
int ObTableLoadIndexTableProjector::projector(const ObTabletID &data_tablet_id,
const blocksstable::ObDatumRow &origin_datum_row,
const bool &have_multiversion_col,
ObTabletID &index_tablet_id,
blocksstable::ObDatumRow &out_datum_row)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadIndexTableProjector not init", KR(ret));
} else if (OB_FAIL(out_datum_row.init(column_num_))) {
LOG_WARN("fail to init index datum row", KR(ret));
} else if (OB_FAIL(tablet_projector_.get_refactored(data_tablet_id, index_tablet_id))) {
LOG_WARN("fail to get index id", KR(ret), K(data_tablet_id));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < row_projector_.size(); i++) {
if (have_multiversion_col && row_projector_.at(i) >= main_table_rowkey_column_num_) {
if (row_projector_.at(i) + storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt() >=
origin_datum_row.get_column_count()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("fail to get datum", KR(ret), K(row_projector_.at(i)), K(origin_datum_row));
} else {
out_datum_row.storage_datums_[i] =
origin_datum_row
.storage_datums_[row_projector_.at(i) +
storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt()];
}
} else {
if (row_projector_.at(i) >= origin_datum_row.get_column_count()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("fail to get datum", KR(ret), K(row_projector_.at(i)), K(origin_datum_row));
} else {
out_datum_row.storage_datums_[i] = origin_datum_row.storage_datums_[row_projector_.at(i)];
}
}
}
}
return ret;
}
} // namespace observer
} // namespace oceanbase

52
src/observer/table_load/ob_table_load_index_table_projector.h Normal file
View File

@ -0,0 +1,52 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include "lib/ob_define.h"
#include "lib/container/ob_array.h"
#include "lib/hash/ob_hashmap.h"
#include "ob_tablet_id.h"
#include "share/schema/ob_table_schema.h"
#include "storage/blocksstable/ob_datum_row.h"
namespace oceanbase
{
namespace observer
{
class ObTableLoadIndexTableProjector
{
public:
ObTableLoadIndexTableProjector()
: column_num_(0), main_table_rowkey_column_num_(0), is_inited_(false)
{
}
~ObTableLoadIndexTableProjector();
int init(const share::schema::ObTableSchema *data_table_schema, const share::schema::ObTableSchema *index_table_schema);
int projector(const ObTabletID &data_tablet_id, const blocksstable::ObDatumRow &origin_datum_row,
const bool &have_multiversion_col, ObTabletID &index_tablet_id,
blocksstable::ObDatumRow &out_datum_row);
int get_index_tablet_id_and_part_id_by_data_tablet_id(const ObTabletID &data_tablet_id, ObTabletID &index_tablet_id, ObObjectID &part_id);
private:
int build_projector(const share::schema::ObTableSchema *data_table_schema, const share::schema::ObTableSchema *index_table_schema);
int build_tablet_projector(const share::schema::ObTableSchema *data_table_schema, const share::schema::ObTableSchema *index_table_schema);
int build_row_projector(const share::schema::ObTableSchema *data_table_schema, const share::schema::ObTableSchema *index_table_schema);
private:
common::ObArray<uint64_t> row_projector_;
common::hash::ObHashMap<ObTabletID, ObTabletID, common::hash::NoPthreadDefendMode> tablet_projector_;//main tablet id -> index tablet id
common::hash::ObHashMap<ObTabletID, ObObjectID> index_tablet_id_to_part_id_map_;
int64_t column_num_;
int64_t main_table_rowkey_column_num_;
bool is_inited_;
};
}
}

27
src/observer/table_load/ob_table_load_mem_compactor.cpp
View File

@ -21,6 +21,7 @@
#include "observer/table_load/ob_table_load_task.h"
#include "observer/table_load/ob_table_load_task_scheduler.h"
#include "observer/table_load/ob_table_load_trans_store.h"
#include "observer/table_load/ob_table_load_store_table_ctx.h"
#include "storage/direct_load/ob_direct_load_external_table.h"
#include "storage/direct_load/ob_direct_load_mem_loader.h"
#include "storage/direct_load/ob_direct_load_mem_sample.h"
@ -232,6 +233,7 @@ private:
ObTableLoadMemCompactor::ObTableLoadMemCompactor()
: store_ctx_(nullptr),
store_table_ctx_(nullptr),
param_(nullptr),
allocator_("TLD_MemC"),
finish_task_count_(0),
@ -249,6 +251,7 @@ ObTableLoadMemCompactor::~ObTableLoadMemCompactor()
void ObTableLoadMemCompactor::reset()
{
store_ctx_ = nullptr;
store_table_ctx_ = nullptr;
param_ = nullptr;
finish_task_count_ = 0;
@ -268,6 +271,7 @@ int ObTableLoadMemCompactor::inner_init()
int ret = OB_SUCCESS;
const uint64_t tenant_id = MTL_ID();
store_ctx_ = compact_ctx_->store_ctx_;
store_table_ctx_ = compact_ctx_->store_table_ctx_;
param_ = &(store_ctx_->ctx_->param_);
if (OB_UNLIKELY(param_->session_count_ < 2)) {
// 排序至少需要两个线程
@ -282,21 +286,21 @@ int ObTableLoadMemCompactor::inner_init()
}
if (compact_ctx_->compact_config_->is_sort_lobid_) {
mem_ctx_.table_data_desc_ = store_ctx_->lob_id_table_data_desc_;
mem_ctx_.datum_utils_ = &(store_ctx_->ctx_->schema_.lob_meta_datum_utils_);
mem_ctx_.table_data_desc_ = store_table_ctx_->lob_id_table_data_desc_;
mem_ctx_.datum_utils_ = &(store_table_ctx_->schema_->lob_meta_datum_utils_);
mem_ctx_.need_sort_ = true;
mem_ctx_.column_count_ = 1;
} else {
mem_ctx_.table_data_desc_ = store_ctx_->table_data_desc_;
mem_ctx_.datum_utils_ = &(store_ctx_->ctx_->schema_.datum_utils_);
mem_ctx_.need_sort_ = param_->need_sort_;
mem_ctx_.column_count_ = (store_ctx_->ctx_->schema_.is_heap_table_
? store_ctx_->ctx_->schema_.store_column_count_ - 1
: store_ctx_->ctx_->schema_.store_column_count_);
mem_ctx_.table_data_desc_ = store_table_ctx_->table_data_desc_;
mem_ctx_.datum_utils_ = &(store_table_ctx_->schema_->datum_utils_);
mem_ctx_.need_sort_ = store_table_ctx_->need_sort_;
mem_ctx_.column_count_ = (store_table_ctx_->schema_->is_heap_table_
? store_table_ctx_->schema_->store_column_count_ - 1
: store_table_ctx_->schema_->store_column_count_);
}
mem_ctx_.mem_load_task_count_ = param_->session_count_;
mem_ctx_.dml_row_handler_ = store_ctx_->error_row_handler_;
mem_ctx_.dml_row_handler_ = store_table_ctx_->row_handler_;
mem_ctx_.file_mgr_ = store_ctx_->tmp_file_mgr_;
mem_ctx_.dup_action_ = param_->dup_action_;
}
@ -311,7 +315,7 @@ int ObTableLoadMemCompactor::inner_init()
if (OB_SUCC(ret)) {
if (OB_FAIL(mem_ctx_.init())) {
LOG_WARN("fail to init compactor ctx", KR(ret));
} else if (OB_FAIL(parallel_merge_ctx_.init(store_ctx_, mem_ctx_.table_data_desc_))) {
} else if (OB_FAIL(parallel_merge_ctx_.init(store_ctx_, store_table_ctx_, mem_ctx_.table_data_desc_))) {
LOG_WARN("fail to init parallel merge ctx", KR(ret));
}
}
@ -540,7 +544,8 @@ int ObTableLoadMemCompactor::start_compact()
{
int ret = OB_SUCCESS;
if (store_ctx_->enable_pre_sort_
&& !store_ctx_->insert_table_ctx_->need_del_lob()) {
&& !store_table_ctx_->insert_table_ctx_->need_del_lob()
&& !store_table_ctx_->is_index_table_) {
// do nothing
} else {
if (OB_FAIL(start_load())) {

1
src/observer/table_load/ob_table_load_mem_compactor.h
View File

@ -86,6 +86,7 @@ private:
int64_t get_compact_task_count() const;
private:
ObTableLoadStoreCtx *store_ctx_;
ObTableLoadStoreTableCtx *store_table_ctx_;
const ObTableLoadParam *param_;
common::ObArenaAllocator allocator_; //需要最后析构
int64_t finish_task_count_ CACHE_ALIGNED;

137
src/observer/table_load/ob_table_load_merger.cpp
View File

@ -14,6 +14,7 @@
#include "observer/table_load/ob_table_load_merger.h"
#include "observer/table_load/ob_table_load_error_row_handler.h"
#include "observer/table_load/ob_table_load_data_row_handler.h"
#include "observer/table_load/ob_table_load_service.h"
#include "observer/table_load/ob_table_load_stat.h"
#include "observer/table_load/ob_table_load_store_ctx.h"
@ -24,6 +25,8 @@
#include "storage/direct_load/ob_direct_load_fast_heap_table.h"
#include "storage/direct_load/ob_direct_load_multi_map.h"
#include "storage/direct_load/ob_direct_load_range_splitter.h"
#include "observer/table_load/ob_table_load_store_table_ctx.h"
#include "observer/table_load/ob_table_load_merger_manager.h"
#include "storage/blocksstable/ob_sstable.h"
namespace oceanbase
@ -246,20 +249,29 @@ private:
* ObTableLoadMerger
*/
ObTableLoadMerger::ObTableLoadMerger(ObTableLoadStoreCtx *store_ctx)
: store_ctx_(store_ctx),
ObTableLoadMerger::ObTableLoadMerger(ObTableLoadStoreCtx *store_ctx, ObTableLoadStoreTableCtx *store_table_ctx)
: store_table_ctx_(store_table_ctx),
allocator_("TLD_TLM"),
store_ctx_(store_ctx),
param_(store_ctx->ctx_->param_),
table_compact_config_(nullptr),
running_thread_count_(0),
has_error_(false),
is_stop_(false),
is_inited_(false)
{
allocator_.set_tenant_id(MTL_ID());
}
ObTableLoadMerger::~ObTableLoadMerger()
{
abort_unless(merging_list_.is_empty());
abort_unless(rescan_list_.is_empty());
if (OB_NOT_NULL(table_compact_config_)) {
table_compact_config_->~ObTableLoadTableCompactConfig();
allocator_.free(table_compact_config_);
table_compact_config_ = nullptr;
}
}
int ObTableLoadMerger::init()
@ -269,12 +281,39 @@ int ObTableLoadMerger::init()
ret = OB_INIT_TWICE;
LOG_WARN("ObTableLoadMerger init twice", KR(ret), KP(this));
} else {
if (OB_FAIL(table_compact_config_.init(store_ctx_, *this))) {
LOG_WARN("fail to init table_compact_config_", KR(ret));
if (store_table_ctx_->is_index_table_) {
ObTableLoadTableCompactConfigIndexTable *index_table_compact_config = nullptr;
if (OB_ISNULL(index_table_compact_config = OB_NEWx(ObTableLoadTableCompactConfigIndexTable, &allocator_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to allocate memory for table compact config", KR(ret));
} else if (OB_FAIL((index_table_compact_config->init(*this)))) {
LOG_WARN("fail to init table compact config", KR(ret));
} else {
table_compact_config_ = index_table_compact_config;
}
} else {
ObTableLoadTableCompactConfigMainTable *main_table_compact_config = nullptr;
if (OB_ISNULL(main_table_compact_config = OB_NEWx(ObTableLoadTableCompactConfigMainTable, &allocator_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to allocate memory for table compact config", KR(ret));
} else if (OB_FAIL((main_table_compact_config->init(store_ctx_, *this)))) {
LOG_WARN("fail to init table compact config", KR(ret));
} else {
table_compact_config_ = main_table_compact_config;
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(lob_id_compact_config_.init(*this))) {
LOG_WARN("fail to init lob id compact config", KR(ret));
} else if (OB_FAIL(table_compact_ctx_.init(store_ctx_, &table_compact_config_))) {
} else if (OB_FAIL(table_compact_ctx_.init(store_ctx_, store_table_ctx_, table_compact_config_))) {
LOG_WARN("fail to init table compact ctx", KR(ret));
}
if (OB_FAIL(ret)) {
if (OB_NOT_NULL(table_compact_config_)) {
table_compact_config_->~ObTableLoadTableCompactConfig();
allocator_.free(table_compact_config_);
table_compact_config_ = nullptr;
}
} else {
is_inited_ = true;
}
@ -289,7 +328,7 @@ int ObTableLoadMerger::start()
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadMerger not init", KR(ret), KP(this));
} else {
if (store_ctx_->is_fast_heap_table_) {
if (store_table_ctx_->is_fast_heap_table_) {
if (OB_FAIL(build_merge_ctx())) {
LOG_WARN("fail to build merge ctx", KR(ret));
} else if (OB_FAIL(start_merge())) {
@ -361,31 +400,31 @@ int ObTableLoadMerger::build_merge_ctx()
{
int ret = OB_SUCCESS;
ObDirectLoadMergeParam merge_param;
merge_param.table_id_ = param_.table_id_;
merge_param.lob_meta_table_id_ = store_ctx_->ctx_->schema_.lob_meta_table_id_;
merge_param.target_table_id_ = store_ctx_->ctx_->ddl_param_.dest_table_id_;
merge_param.rowkey_column_num_ = store_ctx_->ctx_->schema_.rowkey_column_count_;
merge_param.store_column_count_ = store_ctx_->ctx_->schema_.store_column_count_;
merge_param.table_id_ = store_table_ctx_->table_id_;
merge_param.lob_meta_table_id_ = store_table_ctx_->schema_->lob_meta_table_id_;
merge_param.target_table_id_ = store_table_ctx_->table_id_;
merge_param.rowkey_column_num_ = store_table_ctx_->schema_->rowkey_column_count_;
merge_param.store_column_count_ = store_table_ctx_->schema_->store_column_count_;
merge_param.fill_cg_thread_cnt_ = param_.session_count_;
merge_param.lob_column_idxs_ = &(store_ctx_->ctx_->schema_.lob_column_idxs_);
merge_param.table_data_desc_ = store_ctx_->table_data_desc_;
merge_param.datum_utils_ = &(store_ctx_->ctx_->schema_.datum_utils_);
merge_param.lob_column_idxs_ = &(store_ctx_->ctx_->schema_.lob_column_idxs_);
merge_param.col_descs_ = &(store_ctx_->ctx_->schema_.column_descs_);
merge_param.lob_id_table_data_desc_ = store_ctx_->lob_id_table_data_desc_;
merge_param.lob_meta_datum_utils_ = &(store_ctx_->ctx_->schema_.lob_meta_datum_utils_);
merge_param.lob_meta_col_descs_ = &(store_ctx_->ctx_->schema_.lob_meta_column_descs_);
merge_param.is_heap_table_ = store_ctx_->ctx_->schema_.is_heap_table_;
merge_param.is_fast_heap_table_ = store_ctx_->is_fast_heap_table_;
merge_param.lob_column_idxs_ = &(store_table_ctx_->schema_->lob_column_idxs_);
merge_param.table_data_desc_ = store_table_ctx_->table_data_desc_;
merge_param.datum_utils_ = &(store_table_ctx_->schema_->datum_utils_);
merge_param.col_descs_ = &(store_table_ctx_->schema_->column_descs_);
merge_param.lob_id_table_data_desc_ = store_table_ctx_->lob_id_table_data_desc_;
merge_param.lob_meta_datum_utils_ = &(store_table_ctx_->schema_->lob_meta_datum_utils_);
merge_param.lob_meta_col_descs_ = &(store_table_ctx_->schema_->lob_meta_column_descs_);
merge_param.is_heap_table_ = store_table_ctx_->schema_->is_heap_table_;
merge_param.is_fast_heap_table_ = store_table_ctx_->is_fast_heap_table_;
merge_param.is_incremental_ = ObDirectLoadMethod::is_incremental(param_.method_);
merge_param.insert_mode_ = param_.insert_mode_;
merge_param.insert_table_ctx_ = store_ctx_->insert_table_ctx_;
merge_param.dml_row_handler_ = store_ctx_->error_row_handler_;
merge_param.insert_table_ctx_ = store_table_ctx_->insert_table_ctx_;
merge_param.dml_row_handler_ = store_table_ctx_->row_handler_;
merge_param.file_mgr_ = store_ctx_->tmp_file_mgr_;
merge_param.trans_param_ = store_ctx_->trans_param_;
if (OB_FAIL(merge_ctx_.init(store_ctx_->ctx_, merge_param, store_ctx_->ls_partition_ids_))) {
merge_param.index_table_count_ = store_table_ctx_->schema_->index_table_count_;
if (OB_FAIL(merge_ctx_.init(store_ctx_->ctx_, merge_param, store_table_ctx_->ls_partition_ids_))) {
LOG_WARN("fail to init merge ctx", KR(ret));
} else if (store_ctx_->is_multiple_mode_) {
} else if (store_table_ctx_->is_multiple_mode_) {
const ObIArray<ObDirectLoadTabletMergeCtx *> &tablet_merge_ctxs =
merge_ctx_.get_tablet_merge_ctxs();
ObArray<ObIDirectLoadPartitionTable *> empty_table_array;
@ -407,7 +446,7 @@ int ObTableLoadMerger::build_merge_ctx()
} else {
table_array = &empty_table_array;
}
if (!merge_param.is_heap_table_ && !table_array->empty()) {
if (!store_table_ctx_->schema_->is_heap_table_ && !table_array->empty()) {
// for optimize split range is too slow
ObArray<ObDirectLoadMultipleSSTable *> multiple_sstable_array;
ObDirectLoadMultipleMergeRangeSplitter range_splitter;
@ -422,8 +461,8 @@ int ObTableLoadMerger::build_merge_ctx()
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(range_splitter.init(multiple_sstable_array, merge_param.table_data_desc_,
merge_param.datum_utils_, *merge_param.col_descs_))) {
if (OB_FAIL(range_splitter.init(multiple_sstable_array, store_table_ctx_->table_data_desc_,
&(store_table_ctx_->schema_->datum_utils_), (store_table_ctx_->schema_->column_descs_)))) {
LOG_WARN("fail to init range splitter", KR(ret));
}
}
@ -434,7 +473,7 @@ int ObTableLoadMerger::build_merge_ctx()
LOG_WARN("fail to build merge task for multiple pk table", KR(ret));
}
}
} else if (merge_param.is_heap_table_ && !table_array->empty() &&
} else if (store_table_ctx_->schema_->is_heap_table_ && !table_array->empty() &&
tablet_merge_ctxs.count() > param_.session_count_ * 2) {
// for optimize the super multi-partition heap table space serious enlargement
for (int64_t i = 0; OB_SUCC(ret) && i < tablet_merge_ctxs.count(); ++i) {
@ -448,8 +487,8 @@ int ObTableLoadMerger::build_merge_ctx()
for (int64_t i = 0; OB_SUCC(ret) && i < tablet_merge_ctxs.count(); ++i) {
ObDirectLoadTabletMergeCtx *tablet_merge_ctx = tablet_merge_ctxs.at(i);
if (OB_FAIL(tablet_merge_ctx->build_merge_task(
*table_array, store_ctx_->ctx_->schema_.column_descs_, param_.session_count_,
store_ctx_->is_multiple_mode_))) {
*table_array, store_table_ctx_->schema_->column_descs_, param_.session_count_,
store_table_ctx_->is_multiple_mode_))) {
LOG_WARN("fail to build merge task", KR(ret));
}
}
@ -475,8 +514,8 @@ int ObTableLoadMerger::build_merge_ctx()
}
if (OB_SUCC(ret)) {
if (OB_FAIL(tablet_merge_ctx->build_merge_task(
*table_array, store_ctx_->ctx_->schema_.column_descs_, param_.session_count_,
store_ctx_->is_multiple_mode_))) {
*table_array, store_table_ctx_->schema_->column_descs_, param_.session_count_,
store_table_ctx_->is_multiple_mode_))) {
LOG_WARN("fail to build merge task", KR(ret));
}
}
@ -551,21 +590,21 @@ int ObTableLoadMerger::build_del_lob_ctx(bool &need_del_lob)
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(range_splitter.init(multiple_sstable_array, store_ctx_->lob_id_table_data_desc_,
&(store_ctx_->ctx_->schema_.lob_meta_datum_utils_),
store_ctx_->ctx_->schema_.lob_meta_column_descs_))) {
if (OB_FAIL(range_splitter.init(multiple_sstable_array, store_table_ctx_->lob_id_table_data_desc_,
&(store_table_ctx_->schema_->lob_meta_datum_utils_),
store_table_ctx_->schema_->lob_meta_column_descs_))) {
LOG_WARN("fail to init range splitter", KR(ret));
}
}
for (int64_t i = 0; OB_SUCC(ret) && i < tablet_merge_ctxs.count(); ++i) {
ObDirectLoadTabletMergeCtx *tablet_merge_ctx = tablet_merge_ctxs.at(i);
ObDirectLoadInsertTabletContext *insert_tablet_ctx = nullptr;
if (OB_FAIL(store_ctx_->insert_table_ctx_->get_tablet_context(
tablet_merge_ctx->get_tablet_id(), insert_tablet_ctx))) {
if (OB_FAIL(store_table_ctx_->insert_table_ctx_->get_tablet_context(
tablet_merge_ctx->get_tablet_id(), insert_tablet_ctx))) {
LOG_WARN("fail to get tablet ctx", KR(ret), K(tablet_merge_ctx->get_tablet_id()));
} else if (OB_FAIL(tablet_merge_ctx->build_del_lob_task(
multiple_sstable_array, range_splitter, param_.session_count_,
insert_tablet_ctx->get_min_insert_lob_id()))) {
multiple_sstable_array, range_splitter, param_.session_count_,
insert_tablet_ctx->get_min_insert_lob_id()))) {
LOG_WARN("fail to build del lob task for multiple pk table", KR(ret));
}
}
@ -790,23 +829,23 @@ int ObTableLoadMerger::handle_merge_thread_finish(int ret_code)
// release tmpfile
// TODO(suzhi.yt) release all tables and merge tasks
table_compact_ctx_.result_.release_all_table_data();
if (store_ctx_->insert_table_ctx_->need_rescan()) {
if (store_table_ctx_->insert_table_ctx_->need_rescan()) {
if (OB_FAIL(build_rescan_ctx())) {
LOG_WARN("fail to build rescan ctx", KR(ret));
} else if (OB_FAIL(start_rescan())) {
LOG_WARN("fail to start rescan", KR(ret));
}
} else if (store_ctx_->insert_table_ctx_->need_del_lob()) {
} else if (store_table_ctx_->insert_table_ctx_->need_del_lob()) {
if (OB_FAIL(merge_ctx_.close_table_builder())) {
LOG_WARN("fail to close table builder", KR(ret));
} else if (OB_FAIL(lob_id_compact_ctx_.init(store_ctx_, &lob_id_compact_config_))) {
} else if (OB_FAIL(lob_id_compact_ctx_.init(store_ctx_, store_table_ctx_, &lob_id_compact_config_))) {
LOG_WARN("fail to init lob compact", KR(ret));
} else if (OB_FAIL(lob_id_compact_ctx_.start())) {
LOG_WARN("fail to start lob compact", KR(ret));
}
} else {
if (OB_FAIL(store_ctx_->set_status_merged())) {
LOG_WARN("fail to set store status merged", KR(ret));
if (OB_FAIL(store_ctx_->merger_manager_->handle_merge_finish())) {
LOG_WARN("fail to handle merge finish", KR(ret));
}
}
}
@ -825,8 +864,8 @@ int ObTableLoadMerger::handle_rescan_thread_finish(const int ret_code)
if (OB_UNLIKELY(is_stop_ || has_error_)) {
} else {
FLOG_INFO("LOAD RESCAN COMPLETED");
if (OB_FAIL(store_ctx_->set_status_merged())) {
LOG_WARN("fail to set store status merged", KR(ret));
if (OB_FAIL(store_ctx_->merger_manager_->handle_merge_finish())) {
LOG_WARN("fail to handle merge finish", KR(ret));
}
}
}
@ -845,8 +884,8 @@ int ObTableLoadMerger::handle_del_lob_thread_finish(int ret_code)
} else {
FLOG_INFO("LOAD DEL LOB COMPLETED");
lob_id_compact_ctx_.result_.release_all_table_data();
if (OB_FAIL(store_ctx_->set_status_merged())) {
LOG_WARN("fail to set store status merged", KR(ret));
if (OB_FAIL(store_ctx_->merger_manager_->handle_merge_finish())) {
LOG_WARN("fail to handle merge finish", KR(ret));
}
}
}

8
src/observer/table_load/ob_table_load_merger.h
View File

@ -28,6 +28,7 @@ namespace observer
{
class ObTableLoadParam;
class ObTableLoadStoreCtx;
class ObTableLoadStoreTableCtx;
class ObTableLoadMerger
{
@ -39,7 +40,7 @@ class ObTableLoadMerger
class RescanTaskCallback;
class DelLobTaskCallback;
public:
ObTableLoadMerger(ObTableLoadStoreCtx *store_ctx);
ObTableLoadMerger(ObTableLoadStoreCtx *store_ctx, ObTableLoadStoreTableCtx *store_table_ctx);
~ObTableLoadMerger();
int init();
int start();
@ -63,10 +64,13 @@ private:
int handle_rescan_thread_finish(const int ret_code);
void handle_del_lob_task_finish(ObDirectLoadPartitionDelLobTask *&del_lob_task);
int handle_del_lob_thread_finish(int ret_code);
public:
ObTableLoadStoreTableCtx * const store_table_ctx_;
private:
common::ObArenaAllocator allocator_;
ObTableLoadStoreCtx * const store_ctx_;
const ObTableLoadParam &param_;
ObTableLoadTableCompactConfigMainTable table_compact_config_;
ObTableLoadTableCompactConfig * table_compact_config_;
ObTableLoadTableCompactConfigLobIdTable lob_id_compact_config_;
ObTableLoadTableCompactCtx table_compact_ctx_;
ObTableLoadTableCompactCtx lob_id_compact_ctx_;

126
src/observer/table_load/ob_table_load_merger_manager.cpp Normal file
View File

@ -0,0 +1,126 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "observer/table_load/ob_table_load_merger_manager.h"
#include "observer/table_load/ob_table_load_store_ctx.h"
#include "observer/table_load/ob_table_load_merger.h"
#include "observer/table_load/ob_table_load_store_table_ctx.h"
#include "share/table/ob_table_load_sql_statistics.h"
#define USING_LOG_PREFIX SERVER
namespace oceanbase
{
namespace observer
{
void ObTableLoadMergerManager::stop()
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadMergerManager not init", KR(ret));
} else {
if (OB_ISNULL(store_ctx_->data_store_table_ctx_) || OB_ISNULL(store_ctx_->data_store_table_ctx_->merger_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected main store table ctx or merger is NULL", KR(ret));
} else {
store_ctx_->data_store_table_ctx_->merger_->stop();
}
FOREACH(it, store_ctx_->index_store_table_ctx_map_) {
if (OB_ISNULL(it->second) || OB_ISNULL(it->second->merger_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected index store table ctx or merger is NULL", KR(ret), K(it->first));
} else {
it->second->merger_->stop();
}
}
}
}
int ObTableLoadMergerManager::handle_merge_finish()
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadMergerManager not init", KR(ret));
} else {
if (index_merger_idx_ < index_store_array_.size()) {
if (OB_FAIL(index_store_array_.at(index_merger_idx_)->close_index_table_builder())) {
LOG_WARN("fail to close index table builder", KR(ret));
} else if (OB_FAIL(index_store_array_.at(index_merger_idx_)->merger_->start())) {
LOG_WARN("fail to start index ObTableLoadMerger", KR(ret), K(index_merger_idx_));
}
LOG_INFO ("merge index start", K(index_merger_idx_));
index_merger_idx_++;
} else {
if (OB_FAIL(store_ctx_->set_status_merged())) {
LOG_WARN("fail to set status merged", KR(ret));
}
}
}
return ret;
}
int ObTableLoadMergerManager::init()
{
int ret = OB_SUCCESS;
if (IS_INIT) {
ret = OB_INIT_TWICE;
LOG_WARN("ObTableLoadMergerManager init twice", KR(ret), KP(this));
} else {
FOREACH_X(it, store_ctx_->index_store_table_ctx_map_, OB_SUCC(ret)) {
if (OB_ISNULL(it->second)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected index store table ctx is NULL", KR(ret), K(it->first));
} else if (OB_FAIL(index_store_array_.push_back(it->second))) {
LOG_WARN("fail to push back index store table ctx", KR(ret), K(it->first));
}
}
if (OB_SUCC(ret)) {
LOG_INFO("INDEX MERGER COUNT", K(index_store_array_.size()));
is_inited_ = true;
}
}
return ret;
}
int ObTableLoadMergerManager::commit(table::ObTableLoadDmlStat &dml_stats, table::ObTableLoadSqlStatistics &sql_statistics)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadMergerManager not init", KR(ret));
} else if (OB_ISNULL(store_ctx_->data_store_table_ctx_) || OB_ISNULL(store_ctx_->data_store_table_ctx_->insert_table_ctx_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected main store table ctx or insert_table_ctx is NULL", KR(ret));
} else if (OB_FAIL(store_ctx_->data_store_table_ctx_->insert_table_ctx_->commit(dml_stats, sql_statistics))) {
LOG_WARN("fail to commit main ObTableLoadMerger", KR(ret));
}
return ret;
}
int ObTableLoadMergerManager::start()
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadMergerManager not init", KR(ret));
} else if (OB_ISNULL(store_ctx_->data_store_table_ctx_) || OB_ISNULL(store_ctx_->data_store_table_ctx_->merger_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected main store table ctx or merger is NULL", KR(ret));
} else if (OB_FAIL(store_ctx_->data_store_table_ctx_->merger_->start())) {
LOG_WARN("fail to start main ObTableLoadMerger", KR(ret));
}
return ret;
}
}
}

43
src/observer/table_load/ob_table_load_merger_manager.h Normal file
View File

@ -0,0 +1,43 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include "lib/container/ob_array.h"
namespace oceanbase
{
namespace table
{
class ObTableLoadDmlStat;
class ObTableLoadSqlStatistics;
} // namespace table
namespace observer
{
class ObTableLoadStoreTableCtx;
class ObTableLoadStoreCtx;
class ObTableLoadMergerManager
{
public:
ObTableLoadMergerManager(ObTableLoadStoreCtx *store_ctx) : store_ctx_(store_ctx), index_merger_idx_(0), is_inited_(false) {}
void stop();
int start();
int init();
int handle_merge_finish();
int commit(table::ObTableLoadDmlStat &dml_stats, table::ObTableLoadSqlStatistics &sql_statistics);
private:
ObTableLoadStoreCtx * store_ctx_;
common::ObArray<ObTableLoadStoreTableCtx*> index_store_array_;
int index_merger_idx_;
bool is_inited_;
};
}
}

18
src/observer/table_load/ob_table_load_multiple_heap_table_compactor.cpp
View File

@ -14,6 +14,7 @@
#include "observer/table_load/ob_table_load_multiple_heap_table_compactor.h"
#include "observer/table_load/ob_table_load_error_row_handler.h"
#include "observer/table_load/ob_table_load_data_row_handler.h"
#include "observer/table_load/ob_table_load_service.h"
#include "observer/table_load/ob_table_load_stat.h"
#include "observer/table_load/ob_table_load_store_ctx.h"
@ -23,6 +24,7 @@
#include "observer/table_load/ob_table_load_trans_store.h"
#include "storage/direct_load/ob_direct_load_multiple_heap_table_builder.h"
#include "storage/direct_load/ob_direct_load_multiple_heap_table_compactor.h"
#include "observer/table_load/ob_table_load_store_table_ctx.h"
namespace oceanbase
{
@ -295,6 +297,7 @@ private:
ObTableLoadMultipleHeapTableCompactor::ObTableLoadMultipleHeapTableCompactor()
: store_ctx_(nullptr),
store_table_ctx_(nullptr),
param_(nullptr),
allocator_("TLD_MemC"),
finish_task_count_(0)
@ -310,6 +313,7 @@ ObTableLoadMultipleHeapTableCompactor::~ObTableLoadMultipleHeapTableCompactor()
void ObTableLoadMultipleHeapTableCompactor::reset()
{
store_ctx_ = nullptr;
store_table_ctx_ = nullptr;
param_ = nullptr;
finish_task_count_ = 0;
mem_ctx_.reset();
@ -321,20 +325,20 @@ int ObTableLoadMultipleHeapTableCompactor::inner_init()
int ret = OB_SUCCESS;
const uint64_t tenant_id = MTL_ID();
store_ctx_ = compact_ctx_->store_ctx_;
store_table_ctx_ = compact_ctx_->store_table_ctx_;
param_ = &(store_ctx_->ctx_->param_);
mem_ctx_.mem_dump_task_count_ = param_->session_count_ / 3; //暂时先写成1/3，后续再优化
if (mem_ctx_.mem_dump_task_count_ == 0) {
mem_ctx_.mem_dump_task_count_ = 1;
}
mem_ctx_.table_data_desc_ = store_ctx_->table_data_desc_;
mem_ctx_.datum_utils_ = &(store_ctx_->ctx_->schema_.datum_utils_);
mem_ctx_.need_sort_ = param_->need_sort_;
mem_ctx_.table_data_desc_ = store_table_ctx_->table_data_desc_;
mem_ctx_.datum_utils_ = &(store_table_ctx_->schema_->datum_utils_);
mem_ctx_.need_sort_ = store_table_ctx_->need_sort_;
mem_ctx_.mem_load_task_count_ = param_->session_count_;
mem_ctx_.column_count_ =
(store_ctx_->ctx_->schema_.is_heap_table_ ? store_ctx_->ctx_->schema_.store_column_count_ - 1
: store_ctx_->ctx_->schema_.store_column_count_);
mem_ctx_.dml_row_handler_ = store_ctx_->error_row_handler_;
mem_ctx_.column_count_ = (store_table_ctx_->schema_->is_heap_table_ ? store_table_ctx_->schema_->store_column_count_ - 1
: store_table_ctx_->schema_->store_column_count_);
mem_ctx_.dml_row_handler_ = store_table_ctx_->row_handler_;
mem_ctx_.file_mgr_ = store_ctx_->tmp_file_mgr_;
mem_ctx_.dup_action_ = param_->dup_action_;

1
src/observer/table_load/ob_table_load_multiple_heap_table_compactor.h
View File

@ -72,6 +72,7 @@ private:
int create_heap_table_sorter(ObDirectLoadMultipleHeapTableSorter *&heap_table_sorter);
private:
ObTableLoadStoreCtx *store_ctx_;
ObTableLoadStoreTableCtx *store_table_ctx_;
const ObTableLoadParam *param_;
common::ObArenaAllocator allocator_; //需要最后析构
int64_t finish_task_count_ CACHE_ALIGNED;

89
src/observer/table_load/ob_table_load_open_insert_table_ctx_manager.cpp Normal file
View File

@ -0,0 +1,89 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "observer/table_load/ob_table_load_open_insert_table_ctx_manager.h"
#include "observer/table_load/ob_table_load_store_ctx.h"
#include "observer/table_load/ob_table_load_store_table_ctx.h"
#define USING_LOG_PREFIX SERVER
namespace oceanbase
{
namespace observer
{
int ObTableLoadOpenInsertTableCtxManager::init()
{
int ret = OB_SUCCESS;
if (IS_INIT) {
ret = OB_INIT_TWICE;
LOG_WARN("ObTableLoadOpenInsertTableCtxManager init twice", KR(ret), K(is_inited_));
} else {
common::hash::ObHashMap<common::ObTabletID, ObDirectLoadInsertTabletContext *> &data_insert_tablet_ctxs =
store_ctx_->data_store_table_ctx_->insert_table_ctx_->get_tablet_ctx_map();
FOREACH_X(iter, data_insert_tablet_ctxs, OB_SUCC(ret)) {
if (OB_ISNULL(iter->second)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("insert tablet ctx is null", KR(ret));
} else if (OB_FAIL(insert_tablet_ctxs_.push_back(iter->second))) {
LOG_WARN("fail to push back insert tablet ctx", KR(ret));
}
}
FOREACH_X(it, store_ctx_->index_store_table_ctx_map_, OB_SUCC(ret)) {
common::hash::ObHashMap<common::ObTabletID, ObDirectLoadInsertTabletContext *>
&index_insert_tablet_ctxs = it->second->insert_table_ctx_->get_tablet_ctx_map();
FOREACH_X(iter, index_insert_tablet_ctxs, OB_SUCC(ret))
{
if (OB_ISNULL(iter->second)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("insert tablet ctx is null", KR(ret));
} else if (OB_FAIL(insert_tablet_ctxs_.push_back(iter->second))) {
LOG_WARN("fail to push back insert tablet ctx", KR(ret));
}
}
}
if (OB_SUCC(ret)) {
is_inited_ = true;
}
}
return ret;
}
int ObTableLoadOpenInsertTableCtxManager::get_next_insert_tablet_ctx(
ObDirectLoadInsertTabletContext *&tablet_ctx)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadOpenInsertTableCtxManager not init", KR(ret), K(is_inited_));
} else {
ObMutexGuard guard(op_lock_);
if (insert_tablet_context_idx_ < insert_tablet_ctxs_.count()) {
tablet_ctx = insert_tablet_ctxs_[insert_tablet_context_idx_];
insert_tablet_context_idx_++;
} else {
ret = OB_ITER_END;
}
}
return ret;
}
void ObTableLoadOpenInsertTableCtxManager::handle_open_insert_tablet_ctx_finish(bool &is_finish)
{
ObMutexGuard guard(op_lock_);
is_finish = false;
opened_insert_tablet_count_++;
if (opened_insert_tablet_count_ == insert_tablet_ctxs_.count()) {
is_finish = true;
}
}
}
} // namespace oceanbase

43
src/observer/table_load/ob_table_load_open_insert_table_ctx_manager.h Normal file
View File

@ -0,0 +1,43 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include "storage/direct_load/ob_direct_load_insert_table_ctx.h"
namespace oceanbase
{
namespace observer
{
class ObTableLoadStoreCtx;
class ObTableLoadOpenInsertTableCtxManager
{
public:
ObTableLoadOpenInsertTableCtxManager(ObTableLoadStoreCtx *store_ctx)
: store_ctx_(store_ctx),
insert_tablet_context_idx_(0),
opened_insert_tablet_count_(0),
is_inited_(false)
{
}
int get_next_insert_tablet_ctx(ObDirectLoadInsertTabletContext *&tablet_ctx);
void handle_open_insert_tablet_ctx_finish(bool &is_finish);
int init();
private:
ObTableLoadStoreCtx * store_ctx_;
common::ObArray<ObDirectLoadInsertTabletContext *> insert_tablet_ctxs_;
int insert_tablet_context_idx_;
int opened_insert_tablet_count_;
lib::ObMutex op_lock_;
bool is_inited_;
};
}
} // namespace oceanbase

16
src/observer/table_load/ob_table_load_parallel_merge_ctx.cpp
View File

@ -14,6 +14,7 @@
#include "observer/table_load/ob_table_load_parallel_merge_ctx.h"
#include "observer/table_load/ob_table_load_error_row_handler.h"
#include "observer/table_load/ob_table_load_data_row_handler.h"
#include "observer/table_load/ob_table_load_service.h"
#include "observer/table_load/ob_table_load_stat.h"
#include "observer/table_load/ob_table_load_store_ctx.h"
@ -25,6 +26,7 @@
#include "storage/direct_load/ob_direct_load_multiple_sstable_scan_merge.h"
#include "storage/direct_load/ob_direct_load_multiple_sstable_scanner.h"
#include "storage/direct_load/ob_direct_load_range_splitter.h"
#include "observer/table_load/ob_table_load_store_table_ctx.h"
namespace oceanbase
{
@ -372,7 +374,7 @@ public:
ObDirectLoadMultipleSSTableScanMergeParam scan_merge_param;
scan_merge_param.table_data_desc_ = parallel_merge_ctx_->table_data_desc_;
scan_merge_param.datum_utils_ = &(ctx_->schema_.datum_utils_);
scan_merge_param.dml_row_handler_ = parallel_merge_ctx_->store_ctx_->error_row_handler_;
scan_merge_param.dml_row_handler_ = parallel_merge_ctx_->store_table_ctx_->row_handler_;
for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ctx_->merge_sstable_count_; ++i) {
ObDirectLoadMultipleSSTable *sstable = tablet_ctx_->sstables_.at(i);
if (OB_FAIL(sstable_array_.push_back(sstable))) {
@ -397,7 +399,7 @@ public:
} else {
ObDirectLoadMultipleSSTableBuildParam build_param;
build_param.tablet_id_ = tablet_ctx_->tablet_id_;
build_param.table_data_desc_ = parallel_merge_ctx_->store_ctx_->table_data_desc_;
build_param.table_data_desc_ = parallel_merge_ctx_->table_data_desc_;
build_param.datum_utils_ = &(ctx_->schema_.datum_utils_);
build_param.file_mgr_ = parallel_merge_ctx_->store_ctx_->tmp_file_mgr_;
build_param.extra_buf_ = extra_buf_;
@ -456,7 +458,7 @@ public:
int ret = OB_SUCCESS;
ObDirectLoadMultipleSSTableCompactParam compact_param;
compact_param.tablet_id_ = tablet_ctx_->tablet_id_;
compact_param.table_data_desc_ = parallel_merge_ctx_->store_ctx_->table_data_desc_;
compact_param.table_data_desc_ = parallel_merge_ctx_->table_data_desc_;
compact_param.datum_utils_ = &ctx_->schema_.datum_utils_;
if (OB_FAIL(compactor_.init(compact_param))) {
LOG_WARN("fail to init sstable compactor", KR(ret));
@ -539,6 +541,7 @@ private:
ObTableLoadParallelMergeCtx::ObTableLoadParallelMergeCtx()
: store_ctx_(nullptr),
store_table_ctx_(nullptr),
thread_count_(0),
cb_(nullptr),
allocator_("TLD_ParalMerge"),
@ -572,21 +575,22 @@ ObTableLoadParallelMergeCtx::~ObTableLoadParallelMergeCtx()
heavy_task_list_.reset();
}
int ObTableLoadParallelMergeCtx::init(ObTableLoadStoreCtx *store_ctx,
int ObTableLoadParallelMergeCtx::init(ObTableLoadStoreCtx *store_ctx, ObTableLoadStoreTableCtx *store_table_ctx,
const ObDirectLoadTableDataDesc &table_data_desc)
{
int ret = OB_SUCCESS;
if (IS_INIT) {
ret = OB_INIT_TWICE;
LOG_WARN("ObTableLoadParallelMergeCtx init twice", KR(ret), KP(this));
} else if (OB_UNLIKELY(nullptr == store_ctx || !table_data_desc.is_valid())) {
} else if (OB_UNLIKELY(nullptr == store_ctx || nullptr == store_table_ctx || !table_data_desc.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), KP(store_ctx), K(table_data_desc));
LOG_WARN("invalid args", KR(ret), KP(store_ctx), KP(store_table_ctx), K(table_data_desc));
} else {
if (OB_FAIL(tablet_ctx_map_.create(1024, "TLD_CptCtxMap", "TLD_CptCtxMap", MTL_ID()))) {
LOG_WARN("fail to create ctx map", KR(ret));
} else {
store_ctx_ = store_ctx;
store_table_ctx_ = store_table_ctx;
table_data_desc_ = table_data_desc;
thread_count_ = store_ctx_->task_scheduler_->get_thread_count();
is_inited_ = true;

4
src/observer/table_load/ob_table_load_parallel_merge_ctx.h
View File

@ -25,6 +25,7 @@ namespace oceanbase
namespace observer
{
class ObTableLoadStoreCtx;
class ObTableLoadStoreTableCtx;
class ObTableLoadTask;
class ObTableLoadParallelMergeSSTableCompare
@ -81,7 +82,7 @@ public:
typedef TabletCtxMap::const_iterator TabletCtxIterator;
ObTableLoadParallelMergeCtx();
~ObTableLoadParallelMergeCtx();
int init(ObTableLoadStoreCtx *store_ctx, const ObDirectLoadTableDataDesc &table_data_desc);
int init(ObTableLoadStoreCtx *store_ctx, ObTableLoadStoreTableCtx *store_table_ctx, const ObDirectLoadTableDataDesc &table_data_desc);
int add_tablet_sstable(storage::ObDirectLoadMultipleSSTable *sstable);
int start(ObTableLoadParallelMergeCb *cb);
void stop();
@ -105,6 +106,7 @@ private:
private:
ObTableLoadStoreCtx *store_ctx_;
ObDirectLoadTableDataDesc table_data_desc_;
ObTableLoadStoreTableCtx *store_table_ctx_;
int64_t thread_count_;
ObTableLoadParallelMergeCb *cb_;
common::ObArenaAllocator allocator_;

4
src/observer/table_load/ob_table_load_parallel_merge_table_compactor.cpp
View File

@ -77,8 +77,8 @@ int ObTableLoadParallelMergeTableCompactor::inner_init()
{
int ret = OB_SUCCESS;
const uint64_t tenant_id = MTL_ID();
if (OB_FAIL(parallel_merge_ctx_.init(compact_ctx_->store_ctx_,
compact_ctx_->store_ctx_->table_data_desc_))) {
if (OB_FAIL(parallel_merge_ctx_.init(compact_ctx_->store_ctx_, compact_ctx_->store_table_ctx_,
compact_ctx_->store_table_ctx_->table_data_desc_))) {
LOG_WARN("fail to init parallel merge ctx", KR(ret));
} else if (OB_FAIL(parallel_merge_cb_.init(this))) {
LOG_WARN("fail to init parallel merge cb", KR(ret));

12
src/observer/table_load/ob_table_load_pre_sort_writer.cpp
View File

@ -82,7 +82,7 @@ int ObTableLoadPreSortWriter::write(int32_t session_id,
if (OB_ISNULL(store_writer_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("store writer is nullptr", KR(ret));
} else if (OB_FAIL(datum_row.init(pre_sorter_->store_ctx_->table_data_desc_.column_count_))) {
} else if (OB_FAIL(datum_row.init(pre_sorter_->store_ctx_->data_store_table_ctx_->table_data_desc_.column_count_))) {
LOG_WARN("fail to init datum row", KR(ret));
} else if (OB_FAIL(store_writer_->cast_row(session_id, new_row, datum_row))) {
if (OB_UNLIKELY(OB_EAGAIN != ret)) {
@ -92,8 +92,9 @@ int ObTableLoadPreSortWriter::write(int32_t session_id,
}
} else if (OB_FAIL(external_row.external_row_.from_datums(datum_row.storage_datums_,
datum_row.count_,
pre_sorter_->store_ctx_->table_data_desc_.rowkey_column_num_,
row.obj_row_.seq_no_))) {
pre_sorter_->store_ctx_->data_store_table_ctx_->table_data_desc_.rowkey_column_num_,
row.obj_row_.seq_no_,
false))) {
LOG_WARN("fail to cast to external row", KR(ret));
} else if (OB_FAIL(append_row(external_row))) {
LOG_WARN("fail to append row", KR(ret));
@ -118,8 +119,9 @@ int ObTableLoadPreSortWriter::px_write(const ObTabletID &tablet_id,
// do nothing
} else if (OB_FAIL(external_row.external_row_.from_datums(row.storage_datums_,
row.count_,
pre_sorter_->store_ctx_->table_data_desc_.rowkey_column_num_,
seq_no))) {
pre_sorter_->store_ctx_->data_store_table_ctx_->table_data_desc_.rowkey_column_num_,
seq_no,
false))) {
LOG_WARN("fail to cast to external row", KR(ret));
} else if (OB_FAIL(append_row(external_row))) {
LOG_WARN("fail to append row", KR(ret));

30
src/observer/table_load/ob_table_load_pre_sorter.cpp
View File

@ -75,7 +75,7 @@ int ObTableLoadPreSorter::init()
LOG_WARN("fail to init task count", KR(ret), K(dump_task_count_), K(other_task_count_));
} else if (OB_FAIL(init_mem_ctx())) {
LOG_WARN("fail to init mem ctx", KR(ret));
} else if (OB_FAIL(parallel_merge_ctx_.init(store_ctx_, mem_ctx_.table_data_desc_))) {
} else if (OB_FAIL(parallel_merge_ctx_.init(store_ctx_, store_ctx_->data_store_table_ctx_, mem_ctx_.table_data_desc_))) {
LOG_WARN("fail to init parallel_merge_ctx_", KR(ret));
} else if (OB_FAIL(init_sample_task_scheduler())) {
LOG_WARN("fail to init sample task scheduler", KR(ret));
@ -107,14 +107,14 @@ int ObTableLoadPreSorter::init_task_count()
int ObTableLoadPreSorter::init_mem_ctx()
{
int ret = OB_SUCCESS;
mem_ctx_.table_data_desc_ = store_ctx_->table_data_desc_;
mem_ctx_.table_data_desc_ = store_ctx_->data_store_table_ctx_->table_data_desc_;
mem_ctx_.datum_utils_ = &(ctx_->schema_.datum_utils_);
mem_ctx_.need_sort_ = ctx_->param_.need_sort_;
mem_ctx_.column_count_ = (store_ctx_->ctx_->schema_.is_heap_table_
? store_ctx_->ctx_->schema_.store_column_count_ - 1
: store_ctx_->ctx_->schema_.store_column_count_);
mem_ctx_.mem_dump_task_count_ = dump_task_count_;
mem_ctx_.dml_row_handler_ = store_ctx_->error_row_handler_;
mem_ctx_.dml_row_handler_ = store_ctx_->data_store_table_ctx_->row_handler_;
mem_ctx_.dup_action_ = ctx_->param_.dup_action_;
mem_ctx_.file_mgr_ = store_ctx_->tmp_file_mgr_;
if (OB_FAIL(mem_ctx_.init())) {
@ -324,7 +324,7 @@ int ObTableLoadPreSorter::handle_parallel_merge_success()
int ObTableLoadPreSorter::build_parallel_merge_result()
{
int ret = OB_SUCCESS;
ObTableLoadMerger *merger = store_ctx_->merger_;
ObTableLoadMerger *merger = store_ctx_->data_store_table_ctx_->merger_;
const ObTableLoadParallelMergeCtx::TabletCtxMap &tablet_ctx_map =
parallel_merge_ctx_.get_tablet_ctx_map();
if (OB_ISNULL(merger)) {
@ -401,19 +401,19 @@ int ObTableLoadPreSorter::build_merge_param(ObDirectLoadMergeParam& merge_param)
merge_param.store_column_count_ = ctx_->schema_.store_column_count_;
merge_param.fill_cg_thread_cnt_ = ctx_->param_.session_count_;
merge_param.lob_column_idxs_ = &(ctx_->schema_.lob_column_idxs_);
merge_param.table_data_desc_ = store_ctx_->table_data_desc_;
merge_param.table_data_desc_ = store_ctx_->data_store_table_ctx_->table_data_desc_;
merge_param.datum_utils_ = &(ctx_->schema_.datum_utils_);
merge_param.lob_column_idxs_ = &(ctx_->schema_.lob_column_idxs_);
merge_param.col_descs_ = &(ctx_->schema_.column_descs_);
merge_param.lob_id_table_data_desc_ = store_ctx_->lob_id_table_data_desc_;
merge_param.lob_id_table_data_desc_ = store_ctx_->data_store_table_ctx_->lob_id_table_data_desc_;
merge_param.lob_meta_datum_utils_ = &(ctx_->schema_.lob_meta_datum_utils_);
merge_param.lob_meta_col_descs_ = &(ctx_->schema_.lob_meta_column_descs_);
merge_param.is_heap_table_ = ctx_->schema_.is_heap_table_;
merge_param.is_fast_heap_table_ = store_ctx_->is_fast_heap_table_;
merge_param.is_fast_heap_table_ = store_ctx_->data_store_table_ctx_->is_fast_heap_table_;
merge_param.is_incremental_ = ObDirectLoadMethod::is_incremental(ctx_->param_.method_);
merge_param.insert_mode_ = ctx_->param_.insert_mode_;
merge_param.insert_table_ctx_ = store_ctx_->insert_table_ctx_;
merge_param.dml_row_handler_ = store_ctx_->error_row_handler_;
merge_param.insert_table_ctx_ = store_ctx_->data_store_table_ctx_->insert_table_ctx_;
merge_param.dml_row_handler_ = store_ctx_->data_store_table_ctx_->row_handler_;
merge_param.file_mgr_ = store_ctx_->tmp_file_mgr_;
merge_param.trans_param_ = store_ctx_->trans_param_;
return ret;
@ -488,7 +488,7 @@ int ObTableLoadPreSorter::build_merge_ctx_for_multiple_mode(ObDirectLoadMergePar
ObDirectLoadTabletMergeCtx *tablet_merge_ctx = tablet_merge_ctxs.at(i);
if (OB_FAIL(tablet_merge_ctx->build_merge_task(
*table_array, ctx_->schema_.column_descs_, ctx_->param_.session_count_,
store_ctx_->is_multiple_mode_))) {
store_ctx_->data_store_table_ctx_->is_multiple_mode_))) {
LOG_WARN("fail to build merge task", KR(ret));
}
}
@ -522,7 +522,7 @@ int ObTableLoadPreSorter::build_merge_ctx_for_non_multiple_mode(ObDirectLoadMerg
if (OB_SUCC(ret)) {
if (OB_FAIL(tablet_merge_ctx->build_merge_task(
*table_array, ctx_->schema_.column_descs_, ctx_->param_.session_count_,
store_ctx_->is_multiple_mode_))) {
store_ctx_->data_store_table_ctx_->is_multiple_mode_))) {
LOG_WARN("fail to build merge task", KR(ret));
}
}
@ -537,7 +537,7 @@ int ObTableLoadPreSorter::build_merge_ctx()
{
int ret = OB_SUCCESS;
ObDirectLoadMergeParam merge_param;
ObTableLoadMerger *merger = store_ctx_->merger_;
ObTableLoadMerger *merger = store_ctx_->data_store_table_ctx_->merger_;
if (OB_ISNULL(merger)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("merger shoult not be nullptr", KR(ret));
@ -545,9 +545,9 @@ int ObTableLoadPreSorter::build_merge_ctx()
LOG_WARN("fail to build merge param", KR(ret));
} else {
ObTableLoadTableCompactResult &result = merger->table_compact_ctx_.result_;
if (OB_FAIL(merger->merge_ctx_.init(ctx_, merge_param, store_ctx_->ls_partition_ids_))) {
if (OB_FAIL(merger->merge_ctx_.init(ctx_, merge_param, store_ctx_->data_store_table_ctx_->ls_partition_ids_))) {
LOG_WARN("fail to init merge ctx", KR(ret));
} else if (store_ctx_->is_multiple_mode_) {
} else if (store_ctx_->data_store_table_ctx_->is_multiple_mode_) {
if (OB_FAIL(build_merge_ctx_for_multiple_mode(merge_param, merger, result))) {
LOG_WARN("fail to build merge ctx for multiple mode", KR(ret));
}
@ -567,7 +567,7 @@ int ObTableLoadPreSorter::build_merge_ctx()
int ObTableLoadPreSorter::start_merge()
{
int ret = OB_SUCCESS;
if (OB_FAIL(store_ctx_->merger_->start_merge())) {
if (OB_FAIL(store_ctx_->data_store_table_ctx_->merger_->start_merge())) {
LOG_WARN("fail to start merge", KR(ret));
}
return ret;

32
src/observer/table_load/ob_table_load_schema.cpp
View File

@ -380,6 +380,35 @@ int ObTableLoadSchema::check_has_lob_column(const ObTableSchema *table_schema, b
return ret;
}
int ObTableLoadSchema::check_has_non_local_index(share::schema::ObSchemaGetterGuard &schema_guard,
const share::schema::ObTableSchema *table_schema,
bool &bret)
{
int ret = OB_SUCCESS;
bret = false;
if (OB_UNLIKELY(nullptr == table_schema)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), KP(table_schema));
} else {
const ObIArray<ObAuxTableMetaInfo> &simple_index_infos = table_schema->get_simple_index_infos();
for (int64_t i = 0; OB_SUCC(ret) && i < simple_index_infos.count(); i++) {
const ObAuxTableMetaInfo &index_table_info = simple_index_infos.at(i);
const share::schema::ObTableSchema *index_table_schema = nullptr;
if (OB_FAIL(get_table_schema(schema_guard, MTL_ID(), index_table_info.table_id_,
index_table_schema))) {
LOG_WARN("fail to get table shema of index table", KR(ret), K(index_table_info.table_id_));
} else {
if (INDEX_TYPE_NORMAL_LOCAL != index_table_schema->get_index_type() &&
INDEX_TYPE_NORMAL_GLOBAL_LOCAL_STORAGE != index_table_schema->get_index_type()) {
bret = true;
break;
}
}
}
}
return ret;
}
int ObTableLoadSchema::get_tenant_optimizer_gather_stats_on_load(const uint64_t tenant_id,
bool &value)
{
@ -437,6 +466,7 @@ ObTableLoadSchema::ObTableLoadSchema()
part_level_(PARTITION_LEVEL_ZERO),
schema_version_(0),
lob_meta_table_id_(OB_INVALID_ID),
index_table_count_(0),
is_inited_(false)
{
allocator_.set_tenant_id(MTL_ID());
@ -465,6 +495,7 @@ void ObTableLoadSchema::reset()
part_level_ = PARTITION_LEVEL_ZERO;
schema_version_ = 0;
lob_meta_table_id_ = OB_INVALID_ID;
index_table_count_ = 0;
lob_column_idxs_.reset();
column_descs_.reset();
multi_version_column_descs_.reset();
@ -515,6 +546,7 @@ int ObTableLoadSchema::init_table_schema(const ObTableSchema *table_schema)
if (table_schema->has_lob_aux_table()) {
lob_meta_table_id_ = table_schema->get_aux_lob_meta_tid();
}
index_table_count_ = table_schema->get_simple_index_infos().count();
if (OB_FAIL(ObTableLoadUtils::deep_copy(table_schema->get_table_name_str(), table_name_,
allocator_))) {
LOG_WARN("fail to deep copy table name", KR(ret));

5
src/observer/table_load/ob_table_load_schema.h
View File

@ -75,7 +75,9 @@ public:
static int check_has_unused_column(const share::schema::ObTableSchema *table_schema, bool &bret);
static int check_has_roaringbitmap_column(const share::schema::ObTableSchema *table_schema, bool &bret);
static int check_has_lob_column(const share::schema::ObTableSchema *table_schema, bool &bret);
static int check_has_non_local_index(share::schema::ObSchemaGetterGuard &schema_guard,
const share::schema::ObTableSchema *table_schema,
bool &bret);
static int get_tenant_optimizer_gather_stats_on_load(const uint64_t tenant_id, bool &value);
public:
@ -112,6 +114,7 @@ public:
share::schema::ObPartitionLevel part_level_;
int64_t schema_version_;
uint64_t lob_meta_table_id_;
int64_t index_table_count_;
common::ObArray<int64_t> lob_column_idxs_;
// if it is a heap table, it contains hidden primary key column
// does not contain virtual generated columns

21
src/observer/table_load/ob_table_load_service.cpp
View File

@ -510,6 +510,7 @@ int ObTableLoadService::check_support_direct_load(ObSchemaGetterGuard &schema_gu
bool has_invisible_column = false;
bool has_unused_column = false;
bool has_roaringbitmap_column = false;
bool has_non_normal_local_index = false;
// check if it is a user table
const char *tmp_prefix = ObDirectLoadMode::is_insert_overwrite(load_mode) ? InsertOverwritePrefix : EmptyPrefix;
@ -607,10 +608,24 @@ int ObTableLoadService::check_support_direct_load(ObSchemaGetterGuard &schema_gu
ret = OB_NOT_SUPPORTED;
LOG_WARN("version lower than 4.3.1.0 does not support incremental direct-load", KR(ret));
FORWARD_USER_ERROR_MSG(ret, "version lower than 4.3.1.0 does not support incremental direct-load");
} else if (table_schema->get_index_tid_count() > 0) {
} else if (table_schema->get_simple_index_infos().count() > 0 &&
OB_FAIL(ObTableLoadSchema::check_has_non_local_index(
schema_guard, table_schema, has_non_normal_local_index))) {
LOG_WARN("fail to check support direct load for local index", KR(ret));
} else if (has_non_normal_local_index) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("incremental direct-load does not support table with indexes", KR(ret));
FORWARD_USER_ERROR_MSG(ret, "incremental direct-load does not support table with indexes");
LOG_WARN("incremental direct-load does not support table with non-normal local index",
KR(ret));
FORWARD_USER_ERROR_MSG(
ret, "incremental direct-load does not support table with global index or unique index");
} else if (table_schema->get_simple_index_infos().count() > 0 && !has_non_normal_local_index && compat_version < DATA_VERSION_4_3_4_0) {
ret = OB_NOT_SUPPORTED;
LOG_WARN(
"version lower than 4.3.4.0 incremental direct-load does not support table with non-normal local index",
KR(ret));
FORWARD_USER_ERROR_MSG(
ret,
"version lower than 4.3.4.0 incremental direct-load does not support table with non-normal local index");
} else if (table_schema->get_foreign_key_infos().count() > 0) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("incremental direct-load does not support table with foreign keys", KR(ret));

46
src/observer/table_load/ob_table_load_store.cpp
View File

@ -24,6 +24,8 @@
#include "observer/table_load/ob_table_load_task_scheduler.h"
#include "observer/table_load/ob_table_load_trans_store.h"
#include "observer/table_load/ob_table_load_utils.h"
#include "observer/table_load/ob_table_load_store_table_ctx.h"
#include "observer/table_load/ob_table_load_merger_manager.h"
#include "observer/table_load/ob_table_load_pre_sorter.h"
#include "storage/direct_load/ob_direct_load_insert_table_ctx.h"
#include "share/stat/ob_opt_stat_monitor_manager.h"
@ -59,6 +61,29 @@ int ObTableLoadStore::init_ctx(
return ret;
}
void ObTableLoadStore::cancel_table_ctx(ObTableLoadTableCtx *ctx)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(ctx->store_ctx_->data_store_table_ctx_) ||
OB_ISNULL(ctx->store_ctx_->data_store_table_ctx_->insert_table_ctx_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected main store table ctx or insert_table_ctx is NULL", KR(ret));
} else {
ctx->store_ctx_->data_store_table_ctx_->insert_table_ctx_->cancel();
}
FOREACH(it, ctx->store_ctx_->index_store_table_ctx_map_)
{
if (OB_ISNULL(it->second) || OB_ISNULL(it->second->insert_table_ctx_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected index store table ctx or insert_table_ctx is NULL", KR(ret),
K(it->first));
} else {
it->second->insert_table_ctx_->cancel();
}
}
}
void ObTableLoadStore::abort_ctx(ObTableLoadTableCtx *ctx, bool &is_stopped)
{
int ret = OB_SUCCESS;
@ -82,8 +107,8 @@ void ObTableLoadStore::abort_ctx(ObTableLoadTableCtx *ctx, bool &is_stopped)
if (OB_SUCCESS != (tmp_ret = abort_active_trans(ctx))) {
LOG_WARN("fail to abort active trans", KR(tmp_ret));
}
ctx->store_ctx_->insert_table_ctx_->cancel();
ctx->store_ctx_->merger_->stop();
cancel_table_ctx(ctx);
ctx->store_ctx_->merger_manager_->stop();
ctx->store_ctx_->task_scheduler_->stop();
is_stopped = ctx->store_ctx_->task_scheduler_->is_stopped() && (0 == ATOMIC_LOAD(&ctx->store_ctx_->px_writer_count_));
if (OB_NOT_NULL(ctx->store_ctx_->pre_sorter_)) {
@ -227,17 +252,14 @@ public:
OB_TABLE_LOAD_STATISTICS_TIME_COST(INFO, store_open_tablet_time_us);
int ret = OB_SUCCESS;
while (OB_SUCC(ret)) {
ObTabletID tablet_id;
ObDirectLoadInsertTabletContext *tablet_ctx = nullptr;
if (OB_FAIL(ctx_->store_ctx_->get_next_insert_tablet_ctx(tablet_id))) {
if (OB_FAIL(ctx_->store_ctx_->get_next_insert_tablet_ctx(tablet_ctx))) {
if (OB_UNLIKELY(ret != OB_ITER_END)) {
LOG_WARN("fail to get next insert tablet context", KR(ret));
} else {
ret = OB_SUCCESS;
break;
}
} else if (OB_FAIL(ctx_->store_ctx_->insert_table_ctx_->get_tablet_context(tablet_id, tablet_ctx))) {
LOG_WARN("fail to get tablet context", KR(ret), K(tablet_id));
} else {
bool is_finish = false;
while (OB_SUCC(ret)) {
@ -247,9 +269,9 @@ public:
} else if (OB_FAIL(ctx_->store_ctx_->check_status(ObTableLoadStatusType::INITED))) {
LOG_WARN("fail to check status", KR(ret));
} else if (OB_FAIL(tablet_ctx->open())) {
LOG_WARN("fail to open tablet context", KR(ret), K(tablet_id));
LOG_WARN("fail to open tablet context", KR(ret), KPC(tablet_ctx));
if (ret == OB_EAGAIN || ret == OB_MINOR_FREEZE_NOT_ALLOW) {
LOG_WARN("retry to open tablet context", K(tablet_id));
LOG_WARN("retry to open tablet context");
ret = OB_SUCCESS;
}
} else {
@ -320,7 +342,7 @@ public:
LOG_WARN("fail to close all chunk", KR(ret));
}
} else {
if (OB_FAIL(ctx_->store_ctx_->merger_->start())) {
if (OB_FAIL(ctx_->store_ctx_->merger_manager_->start())) {
LOG_WARN("fail to start merger", KR(ret));
}
}
@ -460,15 +482,15 @@ int ObTableLoadStore::commit(ObTableLoadResultInfo &result_info,
LOG_WARN("trans service is null", KR(ret));
} else if (OB_FAIL(store_ctx_->check_status(ObTableLoadStatusType::MERGED))) {
LOG_WARN("fail to check store status", KR(ret));
} else if (OB_FAIL(store_ctx_->insert_table_ctx_->commit(dml_stats, sql_statistics))) {
} else if (OB_FAIL(store_ctx_->merger_manager_->commit(dml_stats, sql_statistics))) {
LOG_WARN("fail to commit insert table", KR(ret));
} else if (ctx_->schema_.has_autoinc_column_ && OB_FAIL(store_ctx_->commit_autoinc_value())) {
} else if (store_ctx_->data_store_table_ctx_->schema_->has_autoinc_column_ && OB_FAIL(store_ctx_->commit_autoinc_value())) {
LOG_WARN("fail to commit sync auto increment value", KR(ret));
}
// 全量旁路导入的dml_stat在执行节点更新
// 增量旁路导入的dml_stat收集到协调节点在事务中更新
else if (ObDirectLoadMethod::is_full(param_.method_) &&
OB_FAIL(ObOptStatMonitorManager::update_dml_stat_info_from_direct_load(dml_stats.dml_stat_array_))) {
OB_FAIL(ObOptStatMonitorManager::update_dml_stat_info_from_direct_load(dml_stats.dml_stat_array_))) {
LOG_WARN("fail to update dml stat info", KR(ret));
} else if (ObDirectLoadMethod::is_full(param_.method_) && FALSE_IT(dml_stats.reset())) {
} else if (ObDirectLoadMethod::is_incremental(param_.method_) &&

1
src/observer/table_load/ob_table_load_store.h
View File

@ -43,6 +43,7 @@ public:
static void abort_ctx(ObTableLoadTableCtx *ctx, bool &is_stopped);
int init();
private:
static void cancel_table_ctx(ObTableLoadTableCtx *ctx);
static int abort_active_trans(ObTableLoadTableCtx *ctx);
// table load ctrl interface
public:

452
src/observer/table_load/ob_table_load_store_ctx.cpp
View File

@ -14,12 +14,16 @@
#include "observer/table_load/ob_table_load_store_ctx.h"
#include "observer/table_load/ob_table_load_error_row_handler.h"
#include "observer/table_load/ob_table_load_data_row_handler.h"
#include "observer/table_load/ob_table_load_merger.h"
#include "observer/table_load/ob_table_load_store_trans.h"
#include "observer/table_load/ob_table_load_table_ctx.h"
#include "observer/table_load/ob_table_load_task_scheduler.h"
#include "observer/table_load/ob_table_load_trans_store.h"
#include "observer/table_load/ob_table_load_utils.h"
#include "observer/table_load/ob_table_load_store_table_ctx.h"
#include "observer/table_load/ob_table_load_merger_manager.h"
#include "observer/table_load/ob_table_load_open_insert_table_ctx_manager.h"
#include "share/ob_autoincrement_service.h"
#include "share/sequence/ob_sequence_cache.h"
#include "sql/engine/cmd/ob_load_data_utils.h"
@ -47,15 +51,12 @@ ObTableLoadStoreCtx::ObTableLoadStoreCtx(ObTableLoadTableCtx *ctx)
: ctx_(ctx),
allocator_("TLD_StoreCtx"),
task_scheduler_(nullptr),
merger_(nullptr),
insert_table_ctx_(nullptr),
next_tablet_idx_(0),
opened_insert_tablet_count_(0),
is_multiple_mode_(false),
is_fast_heap_table_(false),
error_row_handler_(nullptr),
data_store_table_ctx_(nullptr),
merger_manager_(nullptr),
open_insert_tablet_ctx_manager_(nullptr),
px_writer_count_(0),
tmp_file_mgr_(nullptr),
error_row_handler_(nullptr),
sequence_schema_(&allocator_),
next_session_id_(0),
pre_sorter_(nullptr),
@ -67,8 +68,6 @@ ObTableLoadStoreCtx::ObTableLoadStoreCtx(ObTableLoadTableCtx *ctx)
is_inited_(false)
{
allocator_.set_tenant_id(MTL_ID());
ls_partition_ids_.set_tenant_id(MTL_ID());
target_ls_partition_ids_.set_tenant_id(MTL_ID());
committed_trans_store_array_.set_tenant_id(MTL_ID());
}
@ -77,6 +76,68 @@ ObTableLoadStoreCtx::~ObTableLoadStoreCtx()
destroy();
}
int ObTableLoadStoreCtx::init_store_table_ctxs(
const table::ObTableLoadArray<table::ObTableLoadLSIdAndPartitionId> &partition_id_array,
const table::ObTableLoadArray<table::ObTableLoadLSIdAndPartitionId> &target_partition_id_array)
{
int ret = OB_SUCCESS;
ObSchemaGetterGuard schema_guard;
const share::schema::ObTableSchema *data_table_schema = nullptr;
if (OB_FAIL(ObTableLoadSchema::get_schema_guard(ctx_->param_.tenant_id_, schema_guard))) {
LOG_WARN("fail to get schema guard", KR(ret));
} else if (OB_FAIL(ObTableLoadSchema::get_table_schema(
schema_guard, ctx_->param_.tenant_id_, ctx_->param_.table_id_, data_table_schema))) {
LOG_WARN("fail to get table shema of main table", KR(ret));
} else if (OB_ISNULL(data_table_schema)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("data table schema is null", KR(ret));
} else {
if (ObDirectLoadMethod::is_incremental(ctx_->param_.method_)) {
const ObIArray<ObAuxTableMetaInfo> &simple_index_infos =
data_table_schema->get_simple_index_infos();
for (int64_t i = 0; OB_SUCC(ret) && i < simple_index_infos.count(); i++) {
ObTableLoadStoreTableCtx *index_store_table_ctx = nullptr;
if (OB_ISNULL(index_store_table_ctx = OB_NEWx(ObTableLoadStoreTableCtx, (&allocator_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObTableLoadStoreTableCtx", KR(ret));
} else if (OB_FAIL(index_store_table_ctx->init(simple_index_infos.at(i).table_id_, true,
this, partition_id_array,
target_partition_id_array))) {
LOG_WARN("fail to init ObTableLoadStoreTableCtx", KR(ret));
} else if (OB_FAIL(index_store_table_ctx_map_.set_refactored(
simple_index_infos.at(i).table_id_, index_store_table_ctx))) {
LOG_WARN("fail to set index_insert_table_ctx", KR(ret));
}
if (OB_FAIL(ret)) {
if (OB_NOT_NULL(index_store_table_ctx)) {
index_store_table_ctx->~ObTableLoadStoreTableCtx();
allocator_.free(index_store_table_ctx);
index_store_table_ctx = nullptr;
}
}
}
}
if (OB_FAIL(ret)) {
} else if (OB_ISNULL(data_store_table_ctx_ =
OB_NEWx(ObTableLoadStoreTableCtx, (&allocator_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObTableLoadStoreTableCtx", KR(ret));
} else if (OB_FAIL(data_store_table_ctx_->init(ctx_->ddl_param_.dest_table_id_, false, this,
partition_id_array,
target_partition_id_array))) {
LOG_WARN("fail to init ObTableLoadStoreTableCtx", KR(ret));
}
if (OB_FAIL(ret)) {
if (OB_NOT_NULL(data_store_table_ctx_)) {
data_store_table_ctx_->~ObTableLoadStoreTableCtx();
allocator_.free(data_store_table_ctx_);
data_store_table_ctx_ = nullptr;
}
}
}
return ret;
}
int ObTableLoadStoreCtx::init(
const ObTableLoadArray<ObTableLoadLSIdAndPartitionId> &partition_id_array,
const ObTableLoadArray<ObTableLoadLSIdAndPartitionId> &target_partition_id_array)
@ -88,221 +149,114 @@ int ObTableLoadStoreCtx::init(
} else if (OB_UNLIKELY(partition_id_array.empty() || target_partition_id_array.empty())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(partition_id_array), K(target_partition_id_array));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < partition_id_array.count(); ++i) {
const ObLSID &ls_id = partition_id_array[i].ls_id_;
const ObTableLoadPartitionId &part_tablet_id = partition_id_array[i].part_tablet_id_;
if (OB_FAIL(ls_partition_ids_.push_back(ObTableLoadLSIdAndPartitionId(ls_id, part_tablet_id)))) {
LOG_WARN("fail to push back ls tablet id", KR(ret));
}
}
for (int64_t i = 0; OB_SUCC(ret) && i < target_partition_id_array.count(); ++i) {
const ObLSID &ls_id = target_partition_id_array[i].ls_id_;
const ObTableLoadPartitionId &tablet_id = target_partition_id_array[i].part_tablet_id_;
if (OB_FAIL(target_ls_partition_ids_.push_back(ObTableLoadLSIdAndPartitionId(ls_id, tablet_id)))) {
LOG_WARN("fail to push back ls tablet id", KR(ret));
}
}
if (OB_SUCC(ret)) {
table_data_desc_.rowkey_column_num_ =
(!ctx_->schema_.is_heap_table_ ? ctx_->schema_.rowkey_column_count_ : 0);
table_data_desc_.column_count_ =
(!ctx_->schema_.is_heap_table_ ? ctx_->schema_.store_column_count_
: ctx_->schema_.store_column_count_ - 1);
table_data_desc_.sstable_index_block_size_ = ObDirectLoadSSTableIndexBlock::DEFAULT_INDEX_BLOCK_SIZE;
table_data_desc_.sstable_data_block_size_ = ObDirectLoadSSTableDataBlock::DEFAULT_DATA_BLOCK_SIZE;
if (!GCTX.is_shared_storage_mode()) {
table_data_desc_.external_data_block_size_ = ObDirectLoadDataBlock::SN_DEFAULT_DATA_BLOCK_SIZE;
} else {
table_data_desc_.external_data_block_size_ = ObDirectLoadDataBlock::SS_DEFAULT_DATA_BLOCK_SIZE;
}
table_data_desc_.extra_buf_size_ = ObDirectLoadTableDataDesc::DEFAULT_EXTRA_BUF_SIZE;
table_data_desc_.compressor_type_ = ctx_->param_.compressor_type_;
table_data_desc_.is_heap_table_ = ctx_->schema_.is_heap_table_;
table_data_desc_.session_count_ = ctx_->param_.session_count_;
table_data_desc_.exe_mode_ = ctx_->param_.exe_mode_;
int64_t wa_mem_limit = 0;
if (table_data_desc_.exe_mode_ == ObTableLoadExeMode::MAX_TYPE) {
if (OB_FAIL(ObTableLoadService::get_memory_limit(wa_mem_limit))) {
LOG_WARN("failed to get work area memory limit", KR(ret), K(ctx_->param_.tenant_id_));
} else if (wa_mem_limit < ObDirectLoadMemContext::MIN_MEM_LIMIT) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("wa_mem_limit is too small", KR(ret), K(wa_mem_limit));
} else {
table_data_desc_.merge_count_per_round_ = min(wa_mem_limit / table_data_desc_.sstable_data_block_size_ / ctx_->param_.session_count_,
ObDirectLoadSSTableScanMerge::MAX_SSTABLE_COUNT);
table_data_desc_.max_mem_chunk_count_ = 128;
int64_t mem_chunk_size = wa_mem_limit / table_data_desc_.max_mem_chunk_count_;
if (mem_chunk_size <= ObDirectLoadExternalMultiPartitionRowChunk::MIN_MEMORY_LIMIT) {
mem_chunk_size = ObDirectLoadExternalMultiPartitionRowChunk::MIN_MEMORY_LIMIT;
table_data_desc_.max_mem_chunk_count_ = wa_mem_limit / mem_chunk_size;
}
table_data_desc_.mem_chunk_size_ = mem_chunk_size;
table_data_desc_.heap_table_mem_chunk_size_ = wa_mem_limit / ctx_->param_.session_count_;
}
if (OB_SUCC(ret)) {
if (table_data_desc_.is_heap_table_) {
int64_t bucket_cnt = wa_mem_limit / (ctx_->param_.session_count_ * MACRO_BLOCK_WRITER_MEM_SIZE);
if ((ls_partition_ids_.count() <= bucket_cnt) || !ctx_->param_.need_sort_) {
is_fast_heap_table_ = true;
} else {
is_multiple_mode_ = true;
}
} else {
int64_t bucket_cnt = wa_mem_limit / ctx_->param_.session_count_ /
(table_data_desc_.sstable_index_block_size_ + table_data_desc_.sstable_data_block_size_);
is_multiple_mode_ = ctx_->param_.need_sort_ || ls_partition_ids_.count() > bucket_cnt;
}
}
} else {
wa_mem_limit = ctx_->param_.avail_memory_;
if (ctx_->param_.exe_mode_ == ObTableLoadExeMode::FAST_HEAP_TABLE ||
ctx_->param_.exe_mode_ == ObTableLoadExeMode::GENERAL_TABLE_COMPACT) {
is_fast_heap_table_ = (ctx_->param_.exe_mode_ == ObTableLoadExeMode::FAST_HEAP_TABLE);
} else {
is_multiple_mode_ = true;
}
table_data_desc_.merge_count_per_round_ = min(wa_mem_limit / table_data_desc_.sstable_data_block_size_ / ctx_->param_.session_count_,
ObDirectLoadSSTableScanMerge::MAX_SSTABLE_COUNT);
table_data_desc_.max_mem_chunk_count_ = wa_mem_limit / ObDirectLoadExternalMultiPartitionRowChunk::MIN_MEMORY_LIMIT;
}
if (OB_SUCC(ret)) {
lob_id_table_data_desc_ = table_data_desc_;
lob_id_table_data_desc_.rowkey_column_num_ = 1;
lob_id_table_data_desc_.column_count_ = 1;
lob_id_table_data_desc_.is_heap_table_ = false;
}
}
if (OB_FAIL(ret)) {
}
// init trans_param_
else if (ObDirectLoadMethod::is_incremental(ctx_->param_.method_) && OB_FAIL(init_trans_param())) {
LOG_WARN("fail to init trans param", KR(ret));
}
// init trans_allocator_
else if (OB_FAIL(trans_allocator_.init("TLD_STransPool", ctx_->param_.tenant_id_))) {
LOG_WARN("fail to init trans allocator", KR(ret));
}
// init trans_map_
else if (OB_FAIL(trans_map_.create(1024, "TLD_STransMap", "TLD_STransMap",
ctx_->param_.tenant_id_))) {
LOG_WARN("fail to create trans map", KR(ret));
}
// init trans_ctx_map_
else if (OB_FAIL(trans_ctx_map_.create(1024, "TLD_TCtxMap", "TLD_TCtxMap",
ctx_->param_.tenant_id_))) {
LOG_WARN("fail to create trans ctx map", KR(ret));
}
// init segment_trans_ctx_map_
else if (OB_FAIL(segment_ctx_map_.init("TLD_SegCtxMap", ctx_->param_.tenant_id_))) {
LOG_WARN("fail to init segment ctx map", KR(ret));
}
// 初始化task_scheduler_
else if (OB_ISNULL(task_scheduler_ = OB_NEWx(ObTableLoadTaskThreadPoolScheduler, (&allocator_),
ctx_->param_.session_count_, ctx_->param_.table_id_, "Store"))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObTableLoadTaskThreadPoolScheduler", KR(ret));
} else if (OB_FAIL(task_scheduler_->init())) {
LOG_WARN("fail to init task scheduler", KR(ret));
} else if (OB_FAIL(task_scheduler_->start())) {
LOG_WARN("fail to start task scheduler", KR(ret));
}
// 初始化merger_
else if (OB_ISNULL(merger_ = OB_NEWx(ObTableLoadMerger, (&allocator_), this))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObTableLoadMerger", KR(ret));
} else if (OB_FAIL(merger_->init())) {
LOG_WARN("fail to init merger", KR(ret));
}
// init insert_table_ctx_
if (OB_SUCC(ret)) {
ObDirectLoadInsertTableParam insert_table_param;
insert_table_param.table_id_ = ctx_->ddl_param_.dest_table_id_;
insert_table_param.schema_version_ = ctx_->ddl_param_.schema_version_;
insert_table_param.snapshot_version_ = ctx_->ddl_param_.snapshot_version_;
insert_table_param.ddl_task_id_ = ctx_->ddl_param_.task_id_;
insert_table_param.data_version_ = ctx_->ddl_param_.data_version_;
insert_table_param.parallel_ = ctx_->param_.session_count_;
insert_table_param.reserved_parallel_ = is_fast_heap_table_ ? ctx_->param_.session_count_ : 0;
insert_table_param.rowkey_column_count_ = ctx_->schema_.rowkey_column_count_;
insert_table_param.column_count_ = ctx_->schema_.store_column_count_;
insert_table_param.lob_column_count_ = ctx_->schema_.lob_column_idxs_.count();
insert_table_param.is_partitioned_table_ = ctx_->schema_.is_partitioned_table_;
insert_table_param.is_heap_table_ = ctx_->schema_.is_heap_table_;
insert_table_param.is_column_store_ = ctx_->schema_.is_column_store_;
insert_table_param.online_opt_stat_gather_ = ctx_->param_.online_opt_stat_gather_;
insert_table_param.is_incremental_ = ObDirectLoadMethod::is_incremental(ctx_->param_.method_);
insert_table_param.trans_param_ = trans_param_;
insert_table_param.datum_utils_ = &(ctx_->schema_.datum_utils_);
insert_table_param.col_descs_ = &(ctx_->schema_.column_descs_);
insert_table_param.cmp_funcs_ = &(ctx_->schema_.cmp_funcs_);
insert_table_param.online_sample_percent_ = ctx_->param_.online_sample_percent_;
if (OB_ISNULL(insert_table_ctx_ =
OB_NEWx(ObDirectLoadInsertTableContext, (&allocator_)))) {
}
// init trans_param_
else if (ObDirectLoadMethod::is_incremental(ctx_->param_.method_) &&
OB_FAIL(init_trans_param())) {
LOG_WARN("fail to init trans param", KR(ret));
}
// init trans_allocator_
else if (OB_FAIL(trans_allocator_.init("TLD_STransPool", ctx_->param_.tenant_id_))) {
LOG_WARN("fail to init trans allocator", KR(ret));
}
// init trans_map_
else if (OB_FAIL(
trans_map_.create(1024, "TLD_STransMap", "TLD_STransMap", ctx_->param_.tenant_id_))) {
LOG_WARN("fail to create trans map", KR(ret));
}
// init trans_ctx_map_
else if (OB_FAIL(
trans_ctx_map_.create(1024, "TLD_TCtxMap", "TLD_TCtxMap", ctx_->param_.tenant_id_))) {
LOG_WARN("fail to create trans ctx map", KR(ret));
}
// init segment_trans_ctx_map_
else if (OB_FAIL(segment_ctx_map_.init("TLD_SegCtxMap", ctx_->param_.tenant_id_))) {
LOG_WARN("fail to init segment ctx map", KR(ret));
} else if (OB_FAIL(index_store_table_ctx_map_.create(1024, "TLD_IdxStCtxMap", "TLD_IdxStCtxMap",
ctx_->param_.tenant_id_))) {
LOG_WARN("fail to create index_store_table_ctx_map", KR(ret));
}
// 初始化task_scheduler_
else if (OB_ISNULL(task_scheduler_ =
OB_NEWx(ObTableLoadTaskThreadPoolScheduler, (&allocator_),
ctx_->param_.session_count_, ctx_->param_.table_id_, "Store"))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObTableLoadTaskThreadPoolScheduler", KR(ret));
} else if (OB_FAIL(task_scheduler_->init())) {
LOG_WARN("fail to init task scheduler", KR(ret));
} else if (OB_FAIL(task_scheduler_->start())) {
LOG_WARN("fail to start task scheduler", KR(ret));
} else if (OB_ISNULL(error_row_handler_ = OB_NEWx(ObTableLoadErrorRowHandler, (&allocator_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObTableLoadErrorRowHandler", KR(ret));
} else if (OB_FAIL(error_row_handler_->init(ctx_->param_, result_info_, ctx_->job_stat_))) {
LOG_WARN("fail to init error row handler", KR(ret));
} else if (OB_FAIL(init_store_table_ctxs(partition_id_array, target_partition_id_array))) {
LOG_WARN("fail to init store table ctxs", KR(ret));
} else if (OB_ISNULL(merger_manager_ = OB_NEWx(ObTableLoadMergerManager, (&allocator_), this))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObTableLoadMerger", KR(ret));
} else if (OB_FAIL(merger_manager_->init())) {
LOG_WARN("fail to init merger", KR(ret));
} else if (OB_ISNULL(open_insert_tablet_ctx_manager_ =
OB_NEWx(ObTableLoadOpenInsertTableCtxManager, (&allocator_), this))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObTableLoadOpenInsertTableCtxManager", KR(ret));
} else if (OB_FAIL(open_insert_tablet_ctx_manager_->init())) {
LOG_WARN("fail to init open insert tablet ctx manager", KR(ret));
} else if (OB_ISNULL(error_row_handler_ = OB_NEWx(ObTableLoadErrorRowHandler, (&allocator_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObTableLoadErrorRowHandler", KR(ret));
} else if (OB_FAIL(error_row_handler_->init(ctx_->param_, result_info_, ctx_->job_stat_))) {
LOG_WARN("fail to init error row handler", KR(ret));
}
if (OB_FAIL(ret)) {
}
// init tmp_file_mgr_
else if (OB_ISNULL(tmp_file_mgr_ = OB_NEWx(ObDirectLoadTmpFileManager, (&allocator_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObDirectLoadTmpFileManager", KR(ret));
} else if (OB_FAIL(tmp_file_mgr_->init(ctx_->param_.tenant_id_))) {
LOG_WARN("fail to init tmp file manager", KR(ret));
}
// init session_ctx_array_
else if (OB_FAIL(init_session_ctx_array())) {
LOG_WARN("fail to init session ctx array", KR(ret));
}
// init sequence_cache_ and sequence_schema_
else if (data_store_table_ctx_->schema_->has_identity_column_ && OB_FAIL(init_sequence())) {
LOG_WARN("fail to init sequence", KR(ret));
}
// init enable_pre_sort_ and pre_sorter_
if (OB_SUCC(ret)) {
enable_pre_sort_ = (data_store_table_ctx_->is_multiple_mode_
&& !data_store_table_ctx_->table_data_desc_.is_heap_table_);
if (enable_pre_sort_) {
if (OB_ISNULL(pre_sorter_ = OB_NEWx(ObTableLoadPreSorter,
(&allocator_),
this->ctx_,
this))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObDirectLoadInsertTableContext", KR(ret));
} else if (OB_FAIL(insert_table_ctx_->init(insert_table_param, ls_partition_ids_, target_ls_partition_ids_))) {
LOG_WARN("fail to init insert table ctx", KR(ret));
LOG_WARN("fail to allocate TableLoadPreSorter", KR(ret));
} else if (OB_FAIL(pre_sorter_->init())) {
LOG_WARN("fail to init pre sorter", KR(ret));
} else if (OB_FAIL(pre_sorter_->start())) {
LOG_WARN("fail to start pre_sorter", KR(ret));
}
}
if (OB_FAIL(ret)) {
}
// init tmp_file_mgr_
else if (OB_ISNULL(tmp_file_mgr_ = OB_NEWx(ObDirectLoadTmpFileManager, (&allocator_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObDirectLoadTmpFileManager", KR(ret));
} else if (OB_FAIL(tmp_file_mgr_->init(ctx_->param_.tenant_id_))) {
LOG_WARN("fail to init tmp file manager", KR(ret));
}
// init error_row_handler_
else if (OB_ISNULL(error_row_handler_ =
OB_NEWx(ObTableLoadErrorRowHandler, (&allocator_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObTableLoadErrorRowHandler", KR(ret));
} else if (OB_FAIL(error_row_handler_->init(ctx_->param_, result_info_, ctx_->job_stat_))) {
LOG_WARN("fail to init error row handler", KR(ret));
}
// init session_ctx_array_
else if (OB_FAIL(init_session_ctx_array())) {
LOG_WARN("fail to init session ctx array", KR(ret));
}
// init sequence_cache_ and sequence_schema_
else if (ctx_->schema_.has_identity_column_ && OB_FAIL(init_sequence())) {
LOG_WARN("fail to init sequence", KR(ret));
}
// init enable_pre_sort_ and pre_sorter_
if (OB_SUCC(ret)) {
enable_pre_sort_ = (is_multiple_mode_ && !table_data_desc_.is_heap_table_);
if (enable_pre_sort_) {
if (OB_ISNULL(pre_sorter_ = OB_NEWx(ObTableLoadPreSorter,
(&allocator_),
this->ctx_,
this))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to allocate TableLoadPreSorter", KR(ret));
} else if (OB_FAIL(pre_sorter_->init())) {
LOG_WARN("fail to init pre sorter", KR(ret));
} else if (OB_FAIL(pre_sorter_->start())) {
LOG_WARN("fail to start pre_sorter", KR(ret));
}
}
}
if (OB_SUCC(ret)) {
is_inited_ = true;
} else {
destroy();
}
}
if (OB_SUCC(ret)) {
is_inited_ = true;
} else {
destroy();
}
return ret;
}
void ObTableLoadStoreCtx::stop()
{
if (nullptr != merger_) {
merger_->stop();
if (nullptr != merger_manager_) {
merger_manager_->stop();
}
if (nullptr != pre_sorter_) {
pre_sorter_->stop();
@ -316,10 +270,33 @@ void ObTableLoadStoreCtx::stop()
void ObTableLoadStoreCtx::destroy()
{
if (nullptr != merger_) {
merger_->~ObTableLoadMerger();
allocator_.free(merger_);
merger_ = nullptr;
if (OB_NOT_NULL(error_row_handler_)) {
error_row_handler_->~ObTableLoadErrorRowHandler();
allocator_.free(error_row_handler_);
error_row_handler_ = nullptr;
}
FOREACH(it, index_store_table_ctx_map_) {
ObTableLoadStoreTableCtx* index_store_table_ctx = it->second;
if (OB_NOT_NULL(index_store_table_ctx)) {
index_store_table_ctx->~ObTableLoadStoreTableCtx();
allocator_.free(index_store_table_ctx);
}
}
index_store_table_ctx_map_.destroy();
if (OB_NOT_NULL(data_store_table_ctx_)) {
data_store_table_ctx_->~ObTableLoadStoreTableCtx();
allocator_.free(data_store_table_ctx_);
data_store_table_ctx_ = nullptr;
}
if (OB_NOT_NULL(merger_manager_)) {
merger_manager_->~ObTableLoadMergerManager();
allocator_.free(merger_manager_);
merger_manager_ = nullptr;
}
if (OB_NOT_NULL(open_insert_tablet_ctx_manager_)) {
open_insert_tablet_ctx_manager_->~ObTableLoadOpenInsertTableCtxManager();
allocator_.free(open_insert_tablet_ctx_manager_);
open_insert_tablet_ctx_manager_ = nullptr;
}
if (nullptr != task_scheduler_) {
task_scheduler_->stop();
@ -347,26 +324,16 @@ void ObTableLoadStoreCtx::destroy()
ctx_->free_trans_ctx(trans_ctx);
}
trans_ctx_map_.reuse();
if (nullptr != insert_table_ctx_) {
insert_table_ctx_->~ObDirectLoadInsertTableContext();
allocator_.free(insert_table_ctx_);
insert_table_ctx_ = nullptr;
}
if (nullptr != pre_sorter_) {
pre_sorter_->~ObTableLoadPreSorter();
allocator_.free(pre_sorter_);
pre_sorter_ = nullptr;
}
if (nullptr != tmp_file_mgr_) {
if (OB_NOT_NULL(tmp_file_mgr_)) {
tmp_file_mgr_->~ObDirectLoadTmpFileManager();
allocator_.free(tmp_file_mgr_);
tmp_file_mgr_ = nullptr;
}
if (nullptr != error_row_handler_) {
error_row_handler_->~ObTableLoadErrorRowHandler();
allocator_.free(error_row_handler_);
error_row_handler_ = nullptr;
}
}
int ObTableLoadStoreCtx::advance_status(ObTableLoadStatusType status)
@ -446,6 +413,7 @@ int ObTableLoadStoreCtx::check_status(ObTableLoadStatusType status) const
} else {
ret = OB_STATE_NOT_MATCH;
}
LOG_WARN("unexpected status", KR(ret), K(status), K(status_));
}
return ret;
}
@ -621,15 +589,15 @@ int ObTableLoadStoreCtx::init_session_ctx_array()
if (OB_ISNULL(buf = allocator_.alloc(sizeof(SessionContext) * ctx_->param_.write_session_count_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to allocate memory", KR(ret));
} else if (ctx_->schema_.has_autoinc_column_ && OB_FAIL(generate_autoinc_params(autoinc_param))) {
} else if (data_store_table_ctx_->schema_->has_autoinc_column_ && OB_FAIL(generate_autoinc_params(autoinc_param))) {
LOG_WARN("fail to init auto increment param", KR(ret));
} else {
session_ctx_array_ = new (buf) SessionContext[ctx_->param_.write_session_count_];
for (int64_t i = 0; OB_SUCC(ret) && i < ctx_->param_.write_session_count_; ++i) {
SessionContext *session_ctx = session_ctx_array_ + i;
session_ctx->autoinc_param_ = autoinc_param;
if (!is_fast_heap_table_) {
session_ctx->extra_buf_size_ = table_data_desc_.extra_buf_size_;
if (!data_store_table_ctx_->is_fast_heap_table_) {
session_ctx->extra_buf_size_ = data_store_table_ctx_->table_data_desc_.extra_buf_size_;
if (OB_ISNULL(session_ctx->extra_buf_ =
static_cast<char *>(allocator_.alloc(session_ctx->extra_buf_size_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
@ -1002,26 +970,14 @@ void ObTableLoadStoreCtx::clear_committed_trans_stores()
committed_trans_store_array_.reset();
}
int ObTableLoadStoreCtx::get_next_insert_tablet_ctx(ObTabletID &tablet_id)
int ObTableLoadStoreCtx::get_next_insert_tablet_ctx(ObDirectLoadInsertTabletContext *&tablet_ctx)
{
int ret = OB_SUCCESS;
ObMutexGuard guard(op_lock_);
if (next_tablet_idx_ < ls_partition_ids_.count()) {
tablet_id = ls_partition_ids_[next_tablet_idx_++].part_tablet_id_.tablet_id_;
} else {
ret = OB_ITER_END;
}
return ret;
return open_insert_tablet_ctx_manager_->get_next_insert_tablet_ctx(tablet_ctx);
}
void ObTableLoadStoreCtx::handle_open_insert_tablet_ctx_finish(bool &is_finish)
{
is_finish = false;
ObMutexGuard guard(op_lock_);
if (++opened_insert_tablet_count_ == ls_partition_ids_.count()) {
is_finish = true;
}
open_insert_tablet_ctx_manager_->handle_open_insert_tablet_ctx_finish(is_finish);
}
} // namespace observer

27
src/observer/table_load/ob_table_load_store_ctx.h
View File

@ -46,11 +46,13 @@ class ObTableLoadTransStore;
class ObITableLoadTaskScheduler;
class ObTableLoadMerger;
class ObTableLoadErrorRowHandler;
class ObTableLoadStoreTableCtx;
class ObTableLoadMergerManager;
class ObTableLoadOpenInsertTableCtxManager;
class ObTableLoadPreSorter;
class ObTableLoadStoreCtx
{
static const int64_t MACRO_BLOCK_WRITER_MEM_SIZE = 10 * 1024LL * 1024LL;
public:
ObTableLoadStoreCtx(ObTableLoadTableCtx *ctx);
~ObTableLoadStoreCtx();
@ -135,32 +137,29 @@ private:
int alloc_trans_ctx(const table::ObTableLoadTransId &trans_id, ObTableLoadTransCtx *&trans_ctx);
int alloc_trans(const table::ObTableLoadTransId &trans_id, ObTableLoadStoreTrans *&trans);
int init_session_ctx_array();
int init_store_table_ctxs(
const table::ObTableLoadArray<table::ObTableLoadLSIdAndPartitionId> &partition_id_array,
const table::ObTableLoadArray<table::ObTableLoadLSIdAndPartitionId> &target_partition_id_array);
int init_trans_param();
int generate_autoinc_params(share::AutoincParam &autoinc_param);
int init_sequence();
public:
int commit_autoinc_value();
int get_next_insert_tablet_ctx(ObTabletID &tablet_id);
int get_next_insert_tablet_ctx(ObDirectLoadInsertTabletContext *&tablet_ctx);
void handle_open_insert_tablet_ctx_finish(bool &is_finish);
public:
ObTableLoadTableCtx * const ctx_;
ObTableLoadTableCtx * ctx_;
common::ObArenaAllocator allocator_;
common::ObArray<table::ObTableLoadLSIdAndPartitionId> ls_partition_ids_;
common::ObArray<table::ObTableLoadLSIdAndPartitionId> target_ls_partition_ids_;
storage::ObDirectLoadTableDataDesc table_data_desc_;
storage::ObDirectLoadTableDataDesc lob_id_table_data_desc_;
storage::ObDirectLoadTransParam trans_param_;
table::ObTableLoadResultInfo result_info_;
ObITableLoadTaskScheduler *task_scheduler_;
ObTableLoadMerger *merger_;
storage::ObDirectLoadInsertTableContext *insert_table_ctx_;
int64_t next_tablet_idx_;
int64_t opened_insert_tablet_count_;
bool is_multiple_mode_;
bool is_fast_heap_table_;
ObTableLoadErrorRowHandler *error_row_handler_;
hash::ObHashMap<ObTableID, ObTableLoadStoreTableCtx* > index_store_table_ctx_map_;
ObTableLoadStoreTableCtx *data_store_table_ctx_;
ObTableLoadMergerManager *merger_manager_;
ObTableLoadOpenInsertTableCtxManager *open_insert_tablet_ctx_manager_;
int64_t px_writer_count_;
storage::ObDirectLoadTmpFileManager *tmp_file_mgr_;
ObTableLoadErrorRowHandler *error_row_handler_;
share::schema::ObSequenceSchema sequence_schema_;
uint64_t next_session_id_ CACHE_ALIGNED;
struct SessionContext

473
src/observer/table_load/ob_table_load_store_table_ctx.cpp Normal file
View File

@ -0,0 +1,473 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SERVER
#include "observer/table_load/ob_table_load_store_table_ctx.h"
#include "observer/table_load/ob_table_load_store_ctx.h"
#include "observer/table_load/ob_table_load_merger.h"
#include "observer/table_load/ob_table_load_data_row_handler.h"
#include "observer/table_load/ob_table_load_index_row_handler.h"
#include "storage/direct_load/ob_direct_load_external_multi_partition_table.h"
#include "observer/table_load/ob_table_load_index_table_projector.h"
#include "storage/direct_load/ob_direct_load_sstable_data_block.h"
#include "storage/direct_load/ob_direct_load_mem_context.h"
#include "storage/direct_load/ob_direct_load_i_table.h"
#include "storage/direct_load/ob_direct_load_dml_row_handler.h"
namespace oceanbase
{
namespace observer
{
ObTableLoadStoreTableCtx::ObTableLoadStoreTableCtx()
: allocator_("TLD_STCtx"),
table_id_(OB_INVALID_ID),
is_index_table_(false),
store_ctx_(NULL),
need_sort_(false),
schema_(NULL),
project_(NULL),
is_fast_heap_table_(false),
is_multiple_mode_(false),
insert_table_ctx_(NULL),
row_handler_(NULL),
merger_(NULL),
is_inited_(false)
{
}
ObTableLoadStoreTableCtx::~ObTableLoadStoreTableCtx()
{
if (OB_NOT_NULL(schema_)) {
schema_->~ObTableLoadSchema();
allocator_.free(schema_);
schema_ = NULL;
}
if (OB_NOT_NULL(project_)) {
project_->~ObTableLoadIndexTableProjector();
allocator_.free(project_);
project_ = NULL;
}
if (OB_NOT_NULL(insert_table_ctx_)) {
insert_table_ctx_->~ObDirectLoadInsertTableContext();
allocator_.free(insert_table_ctx_);
insert_table_ctx_ = NULL;
}
if (OB_NOT_NULL(row_handler_)) {
row_handler_->~ObDirectLoadDMLRowHandler();
allocator_.free(row_handler_);
row_handler_ = NULL;
}
if (OB_NOT_NULL(merger_)) {
merger_->~ObTableLoadMerger();
allocator_.free(merger_);
merger_ = NULL;
}
FOREACH(iter, index_table_builder_map_) {
ObIDirectLoadPartitionTableBuilder *table_builder = iter->second;
table_builder->~ObIDirectLoadPartitionTableBuilder();
allocator_.free(table_builder);
}
index_table_builder_map_.destroy();
}
int ObTableLoadStoreTableCtx::init_table_load_schema()
{
int ret = OB_SUCCESS;
if (OB_ISNULL(schema_ = OB_NEWx(ObTableLoadSchema, (&allocator_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObTableLoadSchema", KR(ret));
} else if (OB_FAIL(schema_->init(store_ctx_->ctx_->param_.tenant_id_, table_id_))) {
LOG_WARN("fail to init schema", KR(ret));
}
if (OB_FAIL(ret)) {
if (OB_NOT_NULL(schema_)) {
schema_->~ObTableLoadSchema();
allocator_.free(schema_);
schema_ = NULL;
}
}
return ret;
}
int ObTableLoadStoreTableCtx::init_index_projector()
{
int ret = OB_SUCCESS;
if (is_index_table_) {
ObSchemaGetterGuard schema_guard;
const share::schema::ObTableSchema *data_table_schema = nullptr;
const share::schema::ObTableSchema *index_table_schema = nullptr;
if (OB_FAIL(
ObTableLoadSchema::get_schema_guard(store_ctx_->ctx_->param_.tenant_id_, schema_guard))) {
LOG_WARN("fail to get schema guard", KR(ret));
} else if (OB_FAIL(ObTableLoadSchema::get_table_schema(
schema_guard, store_ctx_->ctx_->param_.tenant_id_,
store_ctx_->ctx_->ddl_param_.dest_table_id_, data_table_schema))) {
LOG_WARN("fail to get table shema of main table", KR(ret));
} else if (OB_FAIL(ObTableLoadSchema::get_table_schema(schema_guard,
store_ctx_->ctx_->param_.tenant_id_, table_id_, index_table_schema))) {
LOG_WARN("fail to get table shema of index table", KR(ret));
}
else if (OB_ISNULL(project_ = OB_NEWx(ObTableLoadIndexTableProjector, &allocator_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObTableLoadIndexTableProjector", KR(ret));
} else if (OB_FAIL(project_->init(data_table_schema, index_table_schema))) {
LOG_WARN("fail to init ObTableLoadIndexTableProjector", KR(ret));
}
if (OB_FAIL(ret)) {
if (OB_NOT_NULL(project_)) {
project_->~ObTableLoadIndexTableProjector();
allocator_.free(project_);
project_ = NULL;
}
}
}
return ret;
}
int ObTableLoadStoreTableCtx::init_ls_partition_ids(
const table::ObTableLoadArray<table::ObTableLoadLSIdAndPartitionId> &partition_id_array,
const table::ObTableLoadArray<table::ObTableLoadLSIdAndPartitionId> &target_partition_id_array)
{
int ret = OB_SUCCESS;
if (is_index_table_) {
for (int64_t i = 0; OB_SUCC(ret) && i < partition_id_array.count(); ++i) {
ObTabletID index_tablet_id;
ObObjectID index_part_id;
const ObTabletID &data_tablet_id = partition_id_array[i].part_tablet_id_.tablet_id_;
if (OB_FAIL(project_->get_index_tablet_id_and_part_id_by_data_tablet_id(
data_tablet_id, index_tablet_id, index_part_id))) {
LOG_WARN("fail to get index_tablet_id", KR(ret), K(data_tablet_id));
} else if (OB_FAIL(ls_partition_ids_.push_back(table::ObTableLoadLSIdAndPartitionId(
partition_id_array[i].ls_id_,
table::ObTableLoadPartitionId(index_part_id, index_tablet_id))))) {
LOG_WARN("fail to push back ls tablet id", KR(ret));
}
}
for (int64_t i = 0; OB_SUCC(ret) && i < target_partition_id_array.count(); ++i) {
ObTabletID index_tablet_id;
ObObjectID index_part_id;
const ObTabletID &data_tablet_id = target_partition_id_array[i].part_tablet_id_.tablet_id_;
if (OB_FAIL(project_->get_index_tablet_id_and_part_id_by_data_tablet_id(
data_tablet_id, index_tablet_id, index_part_id))) {
LOG_WARN("fail to get index_tablet_id", KR(ret), K(data_tablet_id));
} else if (OB_FAIL(target_ls_partition_ids_.push_back(table::ObTableLoadLSIdAndPartitionId(
target_partition_id_array[i].ls_id_,
table::ObTableLoadPartitionId(index_part_id, index_tablet_id))))) {
LOG_WARN("fail to push back ls tablet id", KR(ret));
}
}
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < partition_id_array.count(); ++i) {
const ObLSID &ls_id = partition_id_array[i].ls_id_;
const table::ObTableLoadPartitionId &part_tablet_id = partition_id_array[i].part_tablet_id_;
if (OB_FAIL(ls_partition_ids_.push_back(table::ObTableLoadLSIdAndPartitionId(ls_id, part_tablet_id)))) {
LOG_WARN("fail to push back ls tablet id", KR(ret));
}
}
for (int64_t i = 0; OB_SUCC(ret) && i < target_partition_id_array.count(); ++i) {
const ObLSID &ls_id = target_partition_id_array[i].ls_id_;
const table::ObTableLoadPartitionId &tablet_id = target_partition_id_array[i].part_tablet_id_;
if (OB_FAIL(target_ls_partition_ids_.push_back(table::ObTableLoadLSIdAndPartitionId(ls_id, tablet_id)))) {
LOG_WARN("fail to push back ls tablet id", KR(ret));
}
}
}
return ret;
}
int ObTableLoadStoreTableCtx::init_table_data_desc()
{
int ret = OB_SUCCESS;
table_data_desc_.rowkey_column_num_ =
(!schema_->is_heap_table_ ? schema_->rowkey_column_count_ : 0);
table_data_desc_.column_count_ =
(!schema_->is_heap_table_ ? schema_->store_column_count_
: schema_->store_column_count_ - 1);
table_data_desc_.sstable_index_block_size_ =
ObDirectLoadSSTableIndexBlock::DEFAULT_INDEX_BLOCK_SIZE;
table_data_desc_.sstable_data_block_size_ = ObDirectLoadSSTableDataBlock::DEFAULT_DATA_BLOCK_SIZE;
if (!GCTX.is_shared_storage_mode()) {
table_data_desc_.external_data_block_size_ = ObDirectLoadDataBlock::SN_DEFAULT_DATA_BLOCK_SIZE;
} else {
table_data_desc_.external_data_block_size_ = ObDirectLoadDataBlock::SS_DEFAULT_DATA_BLOCK_SIZE;
}
table_data_desc_.extra_buf_size_ = ObDirectLoadTableDataDesc::DEFAULT_EXTRA_BUF_SIZE;
table_data_desc_.compressor_type_ = store_ctx_->ctx_->param_.compressor_type_;
table_data_desc_.is_heap_table_ = schema_->is_heap_table_;
table_data_desc_.session_count_ = store_ctx_->ctx_->param_.session_count_;
table_data_desc_.exe_mode_ = store_ctx_->ctx_->param_.exe_mode_;
int64_t wa_mem_limit = 0;
if (table_data_desc_.exe_mode_ == ObTableLoadExeMode::MAX_TYPE) {
if (OB_FAIL(ObTableLoadService::get_memory_limit(wa_mem_limit))) {
LOG_WARN("failed to get work area memory limit", KR(ret), K(store_ctx_->ctx_->param_.tenant_id_));
} else if (wa_mem_limit < ObDirectLoadMemContext::MIN_MEM_LIMIT) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("wa_mem_limit is too small", KR(ret), K(wa_mem_limit));
} else {
table_data_desc_.merge_count_per_round_ =
min(wa_mem_limit / table_data_desc_.sstable_data_block_size_ / store_ctx_->ctx_->param_.session_count_,
ObDirectLoadSSTableScanMerge::MAX_SSTABLE_COUNT);
table_data_desc_.max_mem_chunk_count_ = 128;
int64_t mem_chunk_size = wa_mem_limit / table_data_desc_.max_mem_chunk_count_;
if (mem_chunk_size <= ObDirectLoadExternalMultiPartitionRowChunk::MIN_MEMORY_LIMIT) {
mem_chunk_size = ObDirectLoadExternalMultiPartitionRowChunk::MIN_MEMORY_LIMIT;
table_data_desc_.max_mem_chunk_count_ = wa_mem_limit / mem_chunk_size;
}
table_data_desc_.mem_chunk_size_ = mem_chunk_size;
table_data_desc_.heap_table_mem_chunk_size_ = wa_mem_limit / store_ctx_->ctx_->param_.session_count_;
}
if (OB_SUCC(ret)) {
if (table_data_desc_.is_heap_table_) {
int64_t bucket_cnt =
wa_mem_limit / (store_ctx_->ctx_->param_.session_count_ * MACRO_BLOCK_WRITER_MEM_SIZE);
if ((ls_partition_ids_.count() <= bucket_cnt) || !need_sort_) {
is_fast_heap_table_ = true;
} else {
is_multiple_mode_ = true;
}
} else {
int64_t bucket_cnt =
wa_mem_limit / store_ctx_->ctx_->param_.session_count_ /
(table_data_desc_.sstable_index_block_size_ + table_data_desc_.sstable_data_block_size_);
is_multiple_mode_ = need_sort_ || ls_partition_ids_.count() > bucket_cnt || is_index_table_;
}
}
} else {
wa_mem_limit = store_ctx_->ctx_->param_.avail_memory_;
if (!is_index_table_ && (store_ctx_->ctx_->param_.exe_mode_ == ObTableLoadExeMode::FAST_HEAP_TABLE ||
store_ctx_->ctx_->param_.exe_mode_ == ObTableLoadExeMode::GENERAL_TABLE_COMPACT)) {
is_fast_heap_table_ = (store_ctx_->ctx_->param_.exe_mode_ == ObTableLoadExeMode::FAST_HEAP_TABLE);
} else {
is_multiple_mode_ = true;
}
table_data_desc_.merge_count_per_round_ =
min(wa_mem_limit / table_data_desc_.sstable_data_block_size_ / store_ctx_->ctx_->param_.session_count_,
ObDirectLoadSSTableScanMerge::MAX_SSTABLE_COUNT);
table_data_desc_.max_mem_chunk_count_ =
wa_mem_limit / ObDirectLoadExternalMultiPartitionRowChunk::MIN_MEMORY_LIMIT;
}
if (OB_SUCC(ret)) {
lob_id_table_data_desc_ = table_data_desc_;
lob_id_table_data_desc_.rowkey_column_num_ = 1;
lob_id_table_data_desc_.column_count_ = 1;
lob_id_table_data_desc_.is_heap_table_ = false;
}
return ret;
}
int ObTableLoadStoreTableCtx::init_insert_table_ctx()
{
int ret = OB_SUCCESS;
ObDirectLoadInsertTableParam insert_table_param;
insert_table_param.table_id_ = table_id_;
insert_table_param.schema_version_ = store_ctx_->ctx_->ddl_param_.schema_version_;
insert_table_param.snapshot_version_ = store_ctx_->ctx_->ddl_param_.snapshot_version_;
insert_table_param.ddl_task_id_ = store_ctx_->ctx_->ddl_param_.task_id_;
insert_table_param.data_version_ = store_ctx_->ctx_->ddl_param_.data_version_;
insert_table_param.parallel_ = store_ctx_->ctx_->param_.session_count_;
insert_table_param.reserved_parallel_ = is_fast_heap_table_ ? store_ctx_->ctx_->param_.session_count_ : 0;
insert_table_param.rowkey_column_count_ = schema_->rowkey_column_count_;
insert_table_param.column_count_ = schema_->store_column_count_;
insert_table_param.lob_column_count_ = schema_->lob_column_idxs_.count();
insert_table_param.is_partitioned_table_ = schema_->is_partitioned_table_;
insert_table_param.is_heap_table_ = schema_->is_heap_table_;
insert_table_param.is_column_store_ = schema_->is_column_store_;
insert_table_param.online_opt_stat_gather_ = store_ctx_->ctx_->param_.online_opt_stat_gather_;
insert_table_param.is_incremental_ = ObDirectLoadMethod::is_incremental(store_ctx_->ctx_->param_.method_);
insert_table_param.trans_param_ = store_ctx_->trans_param_;
insert_table_param.datum_utils_ = &(schema_->datum_utils_);
insert_table_param.col_descs_ = &(schema_->column_descs_);
insert_table_param.cmp_funcs_ = &(schema_->cmp_funcs_);
insert_table_param.online_sample_percent_ = store_ctx_->ctx_->param_.online_sample_percent_;
if (OB_ISNULL(insert_table_ctx_ =
OB_NEWx(ObDirectLoadInsertTableContext, (&allocator_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObDirectLoadInsertTableContext", KR(ret));
} else if (OB_FAIL(insert_table_ctx_->init(insert_table_param, ls_partition_ids_, target_ls_partition_ids_))) {
LOG_WARN("fail to init insert table ctx", KR(ret));
}
if (OB_FAIL(ret)) {
if (OB_NOT_NULL(insert_table_ctx_)) {
insert_table_ctx_->~ObDirectLoadInsertTableContext();
allocator_.free(insert_table_ctx_);
insert_table_ctx_ = nullptr;
}
}
return ret;
}
int ObTableLoadStoreTableCtx::init_row_handler()
{
int ret = OB_SUCCESS;
if (!is_index_table_) {
ObTableLoadDataRowHandler *main_row_handler = nullptr;
if (OB_ISNULL(main_row_handler = OB_NEWx(ObTableLoadDataRowHandler, (&allocator_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObTableLoadDataRowHandler", KR(ret));
} else if (OB_FAIL(main_row_handler->init(store_ctx_->ctx_->param_, store_ctx_->result_info_,
store_ctx_->error_row_handler_,
&(store_ctx_->index_store_table_ctx_map_)))) {
LOG_WARN("fail to init row handler", KR(ret), KPC(store_ctx_), K(is_index_table),
K(schema_->index_table_count_), KP(project_));
} else {
row_handler_ = main_row_handler;
}
} else {
ObTableLoadIndexRowHandler *index_row_handler = nullptr;
if (OB_ISNULL(index_row_handler = OB_NEWx(ObTableLoadIndexRowHandler, (&allocator_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObTableLoadIndexRowHandler", KR(ret));
} else {
row_handler_ = index_row_handler;
}
}
if (OB_FAIL(ret)) {
if (OB_NOT_NULL(row_handler_)) {
row_handler_->~ObDirectLoadDMLRowHandler();
allocator_.free(row_handler_);
row_handler_ = nullptr;
}
}
return ret;
}
int ObTableLoadStoreTableCtx::init_merger()
{
int ret = OB_SUCCESS;
if (OB_ISNULL(merger_ = OB_NEWx(ObTableLoadMerger, (&allocator_), store_ctx_, this))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObTableLoadMerger", KR(ret));
} else if (OB_FAIL(merger_->init())) {
LOG_WARN("fail to init ObTableLoadMerger", KR(ret));
}
if (OB_FAIL(ret)) {
if (OB_NOT_NULL(merger_)) {
merger_->~ObTableLoadMerger();
allocator_.free(merger_);
merger_ = nullptr;
}
}
return ret;
}
int ObTableLoadStoreTableCtx::init(
ObTableID table_id, bool is_index_table, ObTableLoadStoreCtx *store_ctx,
const table::ObTableLoadArray<table::ObTableLoadLSIdAndPartitionId> &partition_id_array,
const table::ObTableLoadArray<table::ObTableLoadLSIdAndPartitionId> &target_partition_id_array)
{
int ret = OB_SUCCESS;
if (IS_INIT) {
ret = OB_INIT_TWICE;
LOG_WARN("ObTableLoadStoreTableCtx has been inited", KR(ret));
} else if (OB_ISNULL(store_ctx) || OB_INVALID_ID == table_id) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), KP(store_ctx), K(table_id));
} else if (OB_FAIL(index_table_builder_map_.create(1024, "TLD_IdxBd", "TLD_IdxBd", MTL_ID()))) {
LOG_WARN("fail to create index_table_builder_map_", KR(ret));
} else {
table_id_ = table_id;
is_index_table_ = is_index_table;
store_ctx_ = store_ctx;
if (is_index_table) {
need_sort_ = true;
} else {
need_sort_ = (ObTableLoadExeMode::MEM_COMPACT == store_ctx_->ctx_->param_.exe_mode_) ||
(ObTableLoadExeMode::MULTIPLE_HEAP_TABLE_COMPACT == store_ctx_->ctx_->param_.exe_mode_);
}
if (OB_FAIL(init_table_load_schema())) {
LOG_WARN("fail to init table load schema", KR(ret));
} else if (OB_FAIL(init_index_projector())) {
LOG_WARN("fail to init index projector", KR(ret));
} else if (OB_FAIL(init_ls_partition_ids(partition_id_array, target_partition_id_array))) {
LOG_WARN("fail to init ls_partition_ids", KR(ret));
} else if (OB_FAIL(init_table_data_desc())) {
LOG_WARN("fail to init table data desc", KR(ret));
} else if (OB_FAIL(init_insert_table_ctx())) {
LOG_WARN("fail to init insert table ctx", KR(ret));
} else if (OB_FAIL(init_row_handler())) {
LOG_WARN("fail to init row handler", KR(ret));
} else if (OB_FAIL(init_merger())) {
LOG_WARN("fail to init merger", KR(ret));
} else {
is_inited_ = true;
}
}
return ret;
}
int ObTableLoadStoreTableCtx::close_index_table_builder()
{
int ret = OB_SUCCESS;
FOREACH_X(item, index_table_builder_map_, OB_SUCC(ret)) {
if (item->second != nullptr) {
if (OB_FAIL(item->second->close())) {
LOG_WARN("fail to close table", KR(ret));
}
}
}
return ret;
}
int ObTableLoadStoreTableCtx::get_index_table_builder(ObIDirectLoadPartitionTableBuilder *&table_builder)
{
int ret = OB_SUCCESS;
table_builder = nullptr;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), KP(this));
} else {
const int64_t part_id = get_tid_cache();
if (OB_FAIL(index_table_builder_map_.get_refactored(get_tid_cache(), table_builder))) {
if (OB_UNLIKELY(OB_HASH_NOT_EXIST != ret)) {
LOG_WARN("fail to get table builder", KR(ret), K(get_tid_cache()));
} else {
ret = OB_SUCCESS;
ObDirectLoadExternalMultiPartitionTableBuildParam builder_param;
builder_param.table_data_desc_ = table_data_desc_;
builder_param.datum_utils_ = &(schema_->datum_utils_);
builder_param.file_mgr_ = store_ctx_->tmp_file_mgr_;
builder_param.extra_buf_ = reinterpret_cast<char *>(1); // unuse, delete in future
builder_param.extra_buf_size_ = 4096; // unuse, delete in future
{
ObMutexGuard guard(mutex_);
// use mutext to avoid thread unsafe allocator;
ObDirectLoadExternalMultiPartitionTableBuilder *new_builder = nullptr;
if (OB_ISNULL(new_builder = OB_NEWx(
ObDirectLoadExternalMultiPartitionTableBuilder, &allocator_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObDirectLoadExternalMultiPartitionTableBuilder", KR(ret));
} else if (OB_FAIL(new_builder->init(builder_param))) {
LOG_WARN("fail to init new_builder", KR(ret));
} else if (OB_FAIL(index_table_builder_map_.set_refactored(part_id, new_builder))) {
LOG_WARN("fail to set table_builder_map_", KR(ret), K(part_id));
} else {
table_builder = new_builder;
}
if (OB_FAIL(ret)) {
if (nullptr != new_builder) {
new_builder->~ObDirectLoadExternalMultiPartitionTableBuilder();
allocator_.free(new_builder);
new_builder = nullptr;
}
}
}
}
}
}
return ret;
}
}
}

87
src/observer/table_load/ob_table_load_store_table_ctx.h Normal file
View File

@ -0,0 +1,87 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include "lib/hash/ob_hashmap.h"
#include "ob_tablet_id.h"
#include "storage/direct_load/ob_direct_load_table_data_desc.h"
#include "share/table/ob_table_load_define.h"
namespace oceanbase
{
namespace storage
{
class ObDirectLoadInsertTableContext;
class ObIDirectLoadPartitionTableBuilder;
class ObDirectLoadDMLRowHandler;
}
namespace observer
{
class ObTableLoadSchema;
class ObTableLoadMerger;
class ObTableLoadIndexTableProjector;
class ObTableLoadStoreCtx;
class ObTableLoadStoreTableCtx
{
private:
typedef common::hash::ObHashMap<int64_t, ObIDirectLoadPartitionTableBuilder *> TABLE_BUILDER_MAP;
public:
static const int64_t MACRO_BLOCK_WRITER_MEM_SIZE = 10 * 1024LL * 1024LL;
ObTableLoadStoreTableCtx();
~ObTableLoadStoreTableCtx();
int init(
ObTableID table_id, bool is_index_table, ObTableLoadStoreCtx *store_ctx,
const table::ObTableLoadArray<table::ObTableLoadLSIdAndPartitionId> &partition_id_array,
const table::ObTableLoadArray<table::ObTableLoadLSIdAndPartitionId> &target_partition_id_array);
TABLE_BUILDER_MAP& get_index_table_builder_map() { return index_table_builder_map_; }
int close_index_table_builder();
int get_index_table_builder(ObIDirectLoadPartitionTableBuilder *&table_builder);
TO_STRING_KV(K_(is_inited));
bool is_valid() {
return is_inited_;
}
private:
int init_table_load_schema();
int init_index_projector();
int init_ls_partition_ids(
const table::ObTableLoadArray<table::ObTableLoadLSIdAndPartitionId> &partition_id_array,
const table::ObTableLoadArray<table::ObTableLoadLSIdAndPartitionId> &target_partition_id_array);
int init_table_data_desc();
int init_insert_table_ctx();
int init_row_handler();
int init_merger();
public:
common::ObArenaAllocator allocator_;
ObTableID table_id_;
bool is_index_table_;
ObTableLoadStoreCtx *store_ctx_;
bool need_sort_;
ObTableLoadSchema *schema_;
ObTableLoadIndexTableProjector *project_;
common::ObArray<table::ObTableLoadLSIdAndPartitionId> ls_partition_ids_;
common::ObArray<table::ObTableLoadLSIdAndPartitionId> target_ls_partition_ids_;
storage::ObDirectLoadTableDataDesc table_data_desc_;
storage::ObDirectLoadTableDataDesc lob_id_table_data_desc_;
bool is_fast_heap_table_;
bool is_multiple_mode_;
storage::ObDirectLoadInsertTableContext *insert_table_ctx_;
ObDirectLoadDMLRowHandler * row_handler_;
ObTableLoadMerger* merger_;
TABLE_BUILDER_MAP index_table_builder_map_;
private:
lib::ObMutex mutex_;
bool is_inited_;
};
}
}

7
src/observer/table_load/ob_table_load_store_trans_px_writer.cpp
View File

@ -17,6 +17,7 @@
#include "observer/table_load/ob_table_load_store_trans.h"
#include "observer/table_load/ob_table_load_table_ctx.h"
#include "observer/table_load/ob_table_load_trans_store.h"
#include "observer/table_load/ob_table_load_store_table_ctx.h"
namespace oceanbase
{
@ -130,8 +131,8 @@ int ObTableLoadStoreTransPXWriter::check_tablet(const ObTabletID &tablet_id)
LOG_WARN("unexpected store ctx is null", KR(ret), KPC(this));
} else {
bool tablet_found = false;
for (int64_t i = 0; i < store_ctx_->ls_partition_ids_.count(); ++i) {
const ObTableLoadLSIdAndPartitionId &ls_part_id = store_ctx_->ls_partition_ids_.at(i);
for (int64_t i = 0; i < store_ctx_->data_store_table_ctx_->ls_partition_ids_.count(); ++i) {
const ObTableLoadLSIdAndPartitionId &ls_part_id = store_ctx_->data_store_table_ctx_->ls_partition_ids_.at(i);
if (ls_part_id.part_tablet_id_.tablet_id_ == tablet_id) {
tablet_found = true;
break;
@ -139,7 +140,7 @@ int ObTableLoadStoreTransPXWriter::check_tablet(const ObTabletID &tablet_id)
}
if (OB_UNLIKELY(!tablet_found)) {
ret = OB_TABLET_NOT_EXIST;
LOG_WARN("tablet id not found", KR(ret), K(tablet_id), K(store_ctx_->ls_partition_ids_));
LOG_WARN("tablet id not found", KR(ret), K(tablet_id), K(store_ctx_->data_store_table_ctx_->ls_partition_ids_));
}
}
return ret;

55
src/observer/table_load/ob_table_load_table_compactor.cpp
View File

@ -23,6 +23,7 @@
#include "observer/table_load/ob_table_load_multiple_heap_table_compactor.h"
#include "observer/table_load/ob_table_load_trans_store.h"
#include "observer/table_load/ob_table_load_parallel_merge_table_compactor.h"
#include "observer/table_load/ob_table_load_store_table_ctx.h"
namespace oceanbase
{
@ -111,6 +112,10 @@ void ObTableLoadTableCompactResult::release_all_table_data()
/**
* ObTableLoadTableCompactConfig
*/
ObTableLoadTableCompactConfig::~ObTableLoadTableCompactConfig()
{
}
int ObTableLoadTableCompactConfigMainTable::handle_table_compact_success()
{
@ -176,11 +181,50 @@ int ObTableLoadTableCompactConfigMainTable::init(ObTableLoadStoreCtx *store_ctx,
return ret;
}
ObTableLoadTableCompactConfigLobIdTable::ObTableLoadTableCompactConfigLobIdTable() : merger_(nullptr)
ObTableLoadTableCompactConfigIndexTable::ObTableLoadTableCompactConfigIndexTable() : merger_(nullptr)
{
}
ObTableLoadTableCompactConfigIndexTable::~ObTableLoadTableCompactConfigIndexTable()
{
}
int ObTableLoadTableCompactConfigIndexTable::init(ObTableLoadMerger &merger)
{
int ret = OB_SUCCESS;
merger_ = &merger;
is_sort_lobid_ = false;
return ret;
}
int ObTableLoadTableCompactConfigIndexTable::handle_table_compact_success()
{
// notify merger
return merger_->handle_table_compact_success();
}
int ObTableLoadTableCompactConfigIndexTable::get_tables(common::ObIArray<storage::ObIDirectLoadPartitionTable *> &table_array,
common::ObIAllocator &allocator)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(merger_) || OB_ISNULL(merger_->store_table_ctx_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("merger is null", KR(ret));
}
FOREACH_X(item, merger_->store_table_ctx_->get_index_table_builder_map(), OB_SUCC(ret)) {
if (OB_FAIL(item->second->get_tables(table_array, allocator))) {
LOG_WARN("fail to get tables", KR(ret));
}
}
return ret;
}
ObTableLoadTableCompactConfigLobIdTable::ObTableLoadTableCompactConfigLobIdTable() : merger_(nullptr)
{
}
ObTableLoadTableCompactConfigLobIdTable::~ObTableLoadTableCompactConfigLobIdTable()
{
@ -225,10 +269,10 @@ ObTableLoadTableCompactCtx::~ObTableLoadTableCompactCtx()
{
}
int ObTableLoadTableCompactCtx::init(ObTableLoadStoreCtx *store_ctx, ObTableLoadTableCompactConfig *compact_config)
int ObTableLoadTableCompactCtx::init(ObTableLoadStoreCtx *store_ctx, ObTableLoadStoreTableCtx * store_table_ctx, ObTableLoadTableCompactConfig *compact_config)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(nullptr == store_ctx || nullptr == compact_config)) {
if (OB_UNLIKELY(nullptr == store_ctx || nullptr == store_table_ctx || nullptr == compact_config)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), KP(store_ctx), KP(compact_config));
} else {
@ -236,6 +280,7 @@ int ObTableLoadTableCompactCtx::init(ObTableLoadStoreCtx *store_ctx, ObTableLoad
LOG_WARN("fail to init result", KR(ret));
} else {
store_ctx_ = store_ctx;
store_table_ctx_ = store_table_ctx;
compact_config_ = compact_config;
}
}
@ -258,8 +303,8 @@ int ObTableLoadTableCompactCtx::new_compactor(ObTableLoadTableCompactorHandle &c
if (compact_config_->is_sort_lobid_) {
compactor = OB_NEW(ObTableLoadMemCompactor, attr);
} else {
if (store_ctx_->is_multiple_mode_) {
if (store_ctx_->table_data_desc_.is_heap_table_) {
if (store_table_ctx_->is_multiple_mode_) {
if (store_table_ctx_->table_data_desc_.is_heap_table_) {
compactor = OB_NEW(ObTableLoadMultipleHeapTableCompactor, attr);
} else {
compactor = OB_NEW(ObTableLoadMemCompactor, attr);

20
src/observer/table_load/ob_table_load_table_compactor.h
View File

@ -25,6 +25,7 @@ namespace observer
class ObTableLoadStoreCtx;
class ObTableLoadMerger;
class ObTableLoadTableCompactCtx;
class ObTableLoadStoreTableCtx;
struct ObTableLoadTableCompactTabletResult : public common::LinkHashValue<common::ObTabletID>
{
@ -56,6 +57,7 @@ class ObTableLoadTableCompactConfig
{
public:
ObTableLoadTableCompactConfig() : is_sort_lobid_(false) {}
virtual ~ObTableLoadTableCompactConfig();
virtual int handle_table_compact_success() = 0;
virtual int get_tables(common::ObIArray<storage::ObIDirectLoadPartitionTable *> &table_array,
common::ObIAllocator &allocator) = 0;
@ -66,8 +68,8 @@ public:
class ObTableLoadTableCompactConfigMainTable : public ObTableLoadTableCompactConfig
{
public:
~ObTableLoadTableCompactConfigMainTable() override {}
int init(ObTableLoadStoreCtx *store_ctx, ObTableLoadMerger &merger);
int handle_table_compact_success() override;
int get_tables(common::ObIArray<storage::ObIDirectLoadPartitionTable *> &table_array,
common::ObIAllocator &allocator) override;
@ -76,6 +78,19 @@ private:
ObTableLoadMerger *merger_;
};
class ObTableLoadTableCompactConfigIndexTable : public ObTableLoadTableCompactConfig
{
public:
ObTableLoadTableCompactConfigIndexTable();
~ObTableLoadTableCompactConfigIndexTable() override;
int init(ObTableLoadMerger &merger);
int handle_table_compact_success() override;
int get_tables(common::ObIArray<storage::ObIDirectLoadPartitionTable *> &table_array,
common::ObIAllocator &allocator) override;
private:
ObTableLoadMerger *merger_;
};
class ObTableLoadTableCompactConfigLobIdTable : public ObTableLoadTableCompactConfig
{
public:
@ -144,7 +159,7 @@ class ObTableLoadTableCompactCtx
public:
ObTableLoadTableCompactCtx();
~ObTableLoadTableCompactCtx();
int init(ObTableLoadStoreCtx *store_ctx, ObTableLoadTableCompactConfig *compact_config);
int init(ObTableLoadStoreCtx *store_ctx, ObTableLoadStoreTableCtx * store_table_ctx, ObTableLoadTableCompactConfig *compact_config);
bool is_valid() const;
int start();
void stop();
@ -157,6 +172,7 @@ private:
public:
ObTableLoadStoreCtx *store_ctx_;
ObTableLoadStoreTableCtx * store_table_ctx_;
ObTableLoadTableCompactConfig *compact_config_;
mutable obsys::ObRWLock rwlock_;
ObTableLoadTableCompactorHandle compactor_handle_;

2
src/observer/table_load/ob_table_load_table_ctx.h
View File

@ -83,7 +83,7 @@ private:
public:
ObTableLoadParam param_;
ObTableLoadDDLParam ddl_param_;
ObTableLoadSchema schema_;
ObTableLoadSchema schema_; // origin table load schema
ObTableLoadCoordinatorCtx *coordinator_ctx_; // 只在控制节点构造
ObTableLoadStoreCtx *store_ctx_; // 只在数据节点构造
sql::ObLoadDataGID gid_;

4
src/observer/table_load/ob_table_load_trans_bucket_writer.cpp
View File

@ -399,7 +399,7 @@ int ObTableLoadTransBucketWriter::write_for_partitioned(SessionContext &session_
} else if (OB_FAIL(coordinator_ctx_->partition_calc_.get_part_key(obj_rows.at(i), part_key))) {
LOG_WARN("fail to get part key", KR(ret));
} else if (OB_FAIL(coordinator_ctx_->partition_calc_.cast_part_key(part_key, allocator))) {
if (OB_FAIL(error_row_handler->handle_error_row(ret, part_key))) {
if (OB_FAIL(error_row_handler->handle_error_row(ret))) {
LOG_WARN("failed to handle error row", K(ret), K(part_key));
} else {
ret = OB_SUCCESS;
@ -422,7 +422,7 @@ int ObTableLoadTransBucketWriter::write_for_partitioned(SessionContext &session_
bool need_write = false;
if (OB_UNLIKELY(!partition_id.is_valid())) {
ret = OB_NO_PARTITION_FOR_GIVEN_VALUE;
if (OB_FAIL(error_row_handler->handle_error_row(ret, part_keys.at(i)))) {
if (OB_FAIL(error_row_handler->handle_error_row(ret))) {
LOG_WARN("failed to handle error row", K(ret), K(part_keys.at(i)));
} else {
ret = OB_SUCCESS;

28
src/observer/table_load/ob_table_load_trans_store.cpp
View File

@ -16,11 +16,14 @@
#include "observer/omt/ob_tenant_timezone_mgr.h"
#include "observer/table_load/ob_table_load_autoinc_nextval.h"
#include "observer/table_load/ob_table_load_error_row_handler.h"
#include "observer/table_load/ob_table_load_data_row_handler.h"
#include "storage/direct_load/ob_direct_load_dml_row_handler.h"
#include "observer/table_load/ob_table_load_stat.h"
#include "observer/table_load/ob_table_load_store_ctx.h"
#include "observer/table_load/ob_table_load_table_ctx.h"
#include "observer/table_load/ob_table_load_trans_ctx.h"
#include "observer/table_load/ob_table_load_utils.h"
#include "observer/table_load/ob_table_load_store_table_ctx.h"
#include "sql/engine/cmd/ob_load_data_utils.h"
#include "sql/resolver/expr/ob_raw_expr_util.h"
#include "sql/ob_sql_utils.h"
@ -157,8 +160,8 @@ int ObTableLoadTransStoreWriter::init()
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), KPC(trans_store_));
} else {
table_data_desc_ = &store_ctx_->table_data_desc_;
collation_type_ = trans_ctx_->ctx_->schema_.collation_type_;
table_data_desc_ = &store_ctx_->data_store_table_ctx_->table_data_desc_;
collation_type_ = store_ctx_->data_store_table_ctx_->schema_->collation_type_;
if (OB_FAIL(ObSQLUtils::get_default_cast_mode(store_ctx_->ctx_->session_info_, cast_mode_))) {
LOG_WARN("fail to get_default_cast_mode", KR(ret));
} else if (OB_FAIL(init_session_ctx_array())) {
@ -175,7 +178,7 @@ int ObTableLoadTransStoreWriter::init()
int ObTableLoadTransStoreWriter::init_column_schemas_and_lob_info()
{
int ret = OB_SUCCESS;
const ObIArray<ObColDesc> &column_descs = store_ctx_->ctx_->schema_.column_descs_;
const ObIArray<ObColDesc> &column_descs = store_ctx_->data_store_table_ctx_->schema_->column_descs_;
const ObTableSchema *table_schema = nullptr;
if (OB_FAIL(ObTableLoadSchema::get_table_schema(param_.tenant_id_, param_.table_id_, schema_guard_,
table_schema))) {
@ -215,12 +218,12 @@ int ObTableLoadTransStoreWriter::init_session_ctx_array()
}
ObDirectLoadTableStoreParam param;
param.table_data_desc_ = *table_data_desc_;
param.datum_utils_ = &(trans_ctx_->ctx_->schema_.datum_utils_);
param.file_mgr_ = trans_ctx_->ctx_->store_ctx_->tmp_file_mgr_;
param.is_multiple_mode_ = trans_ctx_->ctx_->store_ctx_->is_multiple_mode_;
param.is_fast_heap_table_ = trans_ctx_->ctx_->store_ctx_->is_fast_heap_table_;
param.insert_table_ctx_ = trans_ctx_->ctx_->store_ctx_->insert_table_ctx_;
param.dml_row_handler_ = trans_ctx_->ctx_->store_ctx_->error_row_handler_;
param.datum_utils_ = &(store_ctx_->data_store_table_ctx_->schema_->datum_utils_);
param.file_mgr_ = store_ctx_->tmp_file_mgr_;
param.is_multiple_mode_ = store_ctx_->data_store_table_ctx_->is_multiple_mode_;
param.is_fast_heap_table_ = store_ctx_->data_store_table_ctx_->is_fast_heap_table_;
param.insert_table_ctx_ = store_ctx_->data_store_table_ctx_->insert_table_ctx_;
param.dml_row_handler_ = store_ctx_->data_store_table_ctx_->row_handler_;
for (int64_t i = 0; OB_SUCC(ret) && i < session_count; ++i) {
SessionContext *session_ctx = session_ctx_array_ + i;
if (param_.px_mode_) {
@ -405,7 +408,7 @@ int ObTableLoadTransStoreWriter::cast_row(ObArenaAllocator &cast_allocator,
if (OB_FAIL(ret)) {
ObTableLoadErrorRowHandler *error_row_handler =
trans_ctx_->ctx_->store_ctx_->error_row_handler_;
if (OB_FAIL(error_row_handler->handle_error_row(ret, row))) {
if (OB_FAIL(error_row_handler->handle_error_row(ret))) {
LOG_WARN("failed to handle error row", K(ret), K(row));
} else {
ret = OB_EAGAIN;
@ -553,12 +556,13 @@ int ObTableLoadTransStoreWriter::write_row_to_table_store(ObDirectLoadTableStore
if (OB_FAIL(ret)) {
ObTableLoadErrorRowHandler *error_row_handler =
trans_ctx_->ctx_->store_ctx_->error_row_handler_;
ObDirectLoadDMLRowHandler *data_row_handler = trans_ctx_->ctx_->store_ctx_->data_store_table_ctx_->row_handler_;
if (OB_LIKELY(OB_ERR_PRIMARY_KEY_DUPLICATE == ret)) {
if (OB_FAIL(error_row_handler->handle_update_row(datum_row))) {
if (OB_FAIL(data_row_handler->handle_update_row(datum_row))) {
LOG_WARN("fail to handle update row", KR(ret), K(datum_row));
}
} else if (OB_LIKELY(OB_ROWKEY_ORDER_ERROR == ret)) {
if (OB_FAIL(error_row_handler->handle_error_row(ret, datum_row))) {
if (OB_FAIL(error_row_handler->handle_error_row(ret))) {
LOG_WARN("fail to handle error row", KR(ret), K(tablet_id), K(datum_row));
}
}

17
src/storage/direct_load/ob_direct_load_conflict_check.cpp
View File

@ -48,8 +48,8 @@ bool ObDirectLoadConflictCheckParam::is_valid() const
{
return tablet_id_.is_valid() && store_column_count_ > 0 && table_data_desc_.is_valid() &&
nullptr != origin_table_ && nullptr != range_ && range_->is_valid() &&
nullptr != col_descs_ && nullptr != lob_column_idxs_ && nullptr != builder_ &&
nullptr != datum_utils_ && lob_meta_datum_utils_ != nullptr &&
nullptr != col_descs_ && nullptr != lob_column_idxs_ && nullptr != builder_ &&
nullptr != datum_utils_ && nullptr != lob_meta_datum_utils_ &&
nullptr != dml_row_handler_ &&
(lob_column_idxs_->empty() || tablet_id_in_lob_id_.is_valid());
}
@ -177,7 +177,7 @@ int ObDirectLoadConflictCheck::handle_get_next_row_finish(
}
if (OB_SUCC(ret)) {
if (cmp_ret == 0) {
if (OB_FAIL(param_.dml_row_handler_->handle_update_row(*origin_row_, *load_row, datum_row))) {
if (OB_FAIL(param_.dml_row_handler_->handle_update_row(param_.tablet_id_, *origin_row_, *load_row, datum_row))) {
LOG_WARN("fail to handle update row", KR(ret), KP(origin_row_), KP(load_row));
} else {
if (datum_row == load_row) {
@ -191,12 +191,17 @@ int ObDirectLoadConflictCheck::handle_get_next_row_finish(
origin_row_ = nullptr;
} else {
datum_row = load_row;
if (OB_FAIL(param_.dml_row_handler_->handle_insert_row(*datum_row))) {
LOG_WARN("fail to handle insert row", KR(ret), KP(datum_row));
if (datum_row->row_flag_.is_delete()) {
if (OB_FAIL(param_.dml_row_handler_->handle_delete_row(param_.tablet_id_, *datum_row))) {
LOG_WARN("fail to handle insert row", KR(ret), KP(datum_row));
}
} else {
if (OB_FAIL(param_.dml_row_handler_->handle_insert_row(param_.tablet_id_, *datum_row))) {
LOG_WARN("fail to handle insert row", KR(ret), KP(datum_row));
}
}
}
}
return ret;
}

1
src/storage/direct_load/ob_direct_load_conflict_check.h
View File

@ -26,7 +26,6 @@ namespace storage
{
class ObDirectLoadSSTable;
class ObDirectLoadMultipleSSTable;
struct ObDirectLoadConflictCheckParam
{
public:

12
src/storage/direct_load/ob_direct_load_data_fuse.cpp
View File

@ -228,8 +228,14 @@ int ObDirectLoadDataFuse::inner_get_next_row(const ObDatumRow *&datum_row)
if (OB_FAIL(rows_merger_.pop())) {
LOG_WARN("fail to pop item", KR(ret));
} else if (item->iter_idx_ == LOAD_IDX) {
if (OB_FAIL(param_.dml_row_handler_->handle_insert_row(*datum_row))) {
LOG_WARN("fail to handle insert row", KR(ret), KPC(datum_row));
if (datum_row->row_flag_.is_delete()) {
if (OB_FAIL(param_.dml_row_handler_->handle_delete_row(param_.tablet_id_, *datum_row))) {
LOG_WARN("fail to handle insert row", KR(ret), KP(datum_row));
}
} else {
if (OB_FAIL(param_.dml_row_handler_->handle_insert_row(param_.tablet_id_, *datum_row))) {
LOG_WARN("fail to handle insert row", KR(ret), KP(datum_row));
}
}
}
}
@ -253,7 +259,7 @@ int ObDirectLoadDataFuse::inner_get_next_row(const ObDatumRow *&datum_row)
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(param_.dml_row_handler_->handle_update_row(*old_row, *new_row, datum_row))) {
if (OB_FAIL(param_.dml_row_handler_->handle_update_row(param_.tablet_id_, *old_row, *new_row, datum_row))) {
LOG_WARN("fail to handle update row", KR(ret), KPC(old_row), KPC(new_row));
}
}

12
src/storage/direct_load/ob_direct_load_data_insert.cpp
View File

@ -89,8 +89,16 @@ int ObDirectLoadDataInsert::get_next_row(const ObDatumRow *&datum_row)
} else if (OB_UNLIKELY(datum_row->count_ != param_.store_column_count_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected column count", KR(ret), K(datum_row->count_), K(param_.store_column_count_));
} else if (OB_FAIL(param_.dml_row_handler_->handle_insert_row(*datum_row))) {
LOG_WARN("fail to handle insert row", KR(ret), KPC(datum_row));
} else {
if (datum_row->row_flag_.is_delete()) {
if (OB_FAIL(param_.dml_row_handler_->handle_delete_row(param_.tablet_id_, *datum_row))) {
LOG_WARN("fail to handle insert row", KR(ret), KP(datum_row));
}
} else {
if (OB_FAIL(param_.dml_row_handler_->handle_insert_row(param_.tablet_id_, *datum_row))) {
LOG_WARN("fail to handle insert row", KR(ret), KP(datum_row));
}
}
}
return ret;

8
src/storage/direct_load/ob_direct_load_dml_row_handler.h
View File

@ -26,7 +26,10 @@ public:
ObDirectLoadDMLRowHandler() = default;
virtual ~ObDirectLoadDMLRowHandler() = default;
// handle rows direct insert into sstable
virtual int handle_insert_row(const blocksstable::ObDatumRow &row) = 0;
virtual int handle_insert_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) = 0;
virtual int handle_delete_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) = 0;
// only used for heap table
virtual int handle_insert_row_with_multi_version(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) = 0;
// handle rows with the same primary key in the imported data
virtual int handle_update_row(const blocksstable::ObDatumRow &row) = 0;
virtual int handle_update_row(common::ObArray<const ObDirectLoadExternalRow *> &rows,
@ -34,7 +37,8 @@ public:
virtual int handle_update_row(common::ObArray<const ObDirectLoadMultipleDatumRow *> &rows,
const ObDirectLoadMultipleDatumRow *&row) = 0;
// handle rows with the same primary key between the imported data and the original data
virtual int handle_update_row(const blocksstable::ObDatumRow &old_row,
virtual int handle_update_row(const ObTabletID tablet_id,
const blocksstable::ObDatumRow &old_row,
const blocksstable::ObDatumRow &new_row,
const blocksstable::ObDatumRow *&result_row) = 0;
DECLARE_PURE_VIRTUAL_TO_STRING;

6
src/storage/direct_load/ob_direct_load_external_multi_partition_row.cpp
View File

@ -90,7 +90,7 @@ OB_DEF_SERIALIZE_SIZE_SIMPLE(ObDirectLoadExternalMultiPartitionRow)
*/
ObDirectLoadConstExternalMultiPartitionRow::ObDirectLoadConstExternalMultiPartitionRow()
: buf_size_(0), buf_(nullptr)
: is_deleted_(false), buf_size_(0), buf_(nullptr)
{
}
@ -104,6 +104,7 @@ void ObDirectLoadConstExternalMultiPartitionRow::reset()
rowkey_datum_array_.reset();
seq_no_.reset();
buf_size_ = 0;
is_deleted_ = false;
buf_ = nullptr;
}
@ -116,6 +117,7 @@ ObDirectLoadConstExternalMultiPartitionRow &ObDirectLoadConstExternalMultiPartit
rowkey_datum_array_ = other.rowkey_datum_array_;
buf_size_ = other.buf_size_;
seq_no_ = other.seq_no_;
is_deleted_ = other.is_deleted_;
buf_ = other.buf_;
}
return *this;
@ -128,6 +130,7 @@ ObDirectLoadConstExternalMultiPartitionRow &ObDirectLoadConstExternalMultiPartit
rowkey_datum_array_ = other.external_row_.rowkey_datum_array_;
buf_size_ = other.external_row_.buf_size_;
seq_no_ = other.external_row_.seq_no_;
is_deleted_ = other.external_row_.is_deleted_;
buf_ = other.external_row_.buf_;
return *this;
}
@ -158,6 +161,7 @@ int ObDirectLoadConstExternalMultiPartitionRow::deep_copy(
} else {
buf_size_ = src.buf_size_;
seq_no_ = src.seq_no_;
is_deleted_ = src.is_deleted_;
buf_ = buf + pos;
MEMCPY(buf + pos, src.buf_, buf_size_);
pos += buf_size_;

1
src/storage/direct_load/ob_direct_load_external_multi_partition_row.h
View File

@ -63,6 +63,7 @@ public:
common::ObTabletID tablet_id_;
ObDirectLoadConstDatumArray rowkey_datum_array_;
table::ObTableLoadSequenceNo seq_no_;
bool is_deleted_;
int64_t buf_size_;
const char *buf_;
};

2
src/storage/direct_load/ob_direct_load_external_multi_partition_table.cpp
View File

@ -110,7 +110,7 @@ int ObDirectLoadExternalMultiPartitionTableBuilder::append_row(const ObTabletID
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_append_row_time_us);
row_.tablet_id_ = tablet_id;
if (OB_FAIL(row_.external_row_.from_datums(datum_row.storage_datums_, datum_row.count_,
param_.table_data_desc_.rowkey_column_num_, seq_no))) {
param_.table_data_desc_.rowkey_column_num_, seq_no, datum_row.row_flag_.is_delete()))) {
LOG_WARN("fail to from datums", KR(ret));
} else if (OB_FAIL(external_writer_.write_item(row_))) {
LOG_WARN("fail to write item", KR(ret));

13
src/storage/direct_load/ob_direct_load_external_row.cpp
View File

@ -33,6 +33,7 @@ void ObDirectLoadExternalRow::reset()
{
rowkey_datum_array_.reset();
seq_no_.reset();
is_deleted_ = false;
buf_size_ = 0;
buf_ = nullptr;
allocator_.reset();
@ -42,6 +43,7 @@ void ObDirectLoadExternalRow::reuse()
{
rowkey_datum_array_.reuse();
seq_no_.reset();
is_deleted_ = false;
buf_size_ = 0;
buf_ = nullptr;
allocator_.reuse();
@ -73,6 +75,7 @@ int ObDirectLoadExternalRow::deep_copy(const ObDirectLoadExternalRow &src, char
} else {
buf_size_ = src.buf_size_;
seq_no_ = src.seq_no_;
is_deleted_ = src.is_deleted_;
buf_ = buf + pos;
MEMCPY(buf + pos, src.buf_, buf_size_);
pos += buf_size_;
@ -82,7 +85,8 @@ int ObDirectLoadExternalRow::deep_copy(const ObDirectLoadExternalRow &src, char
}
int ObDirectLoadExternalRow::from_datums(ObStorageDatum *datums, int64_t column_count,
int64_t rowkey_column_count, const ObTableLoadSequenceNo &seq_no)
int64_t rowkey_column_count, const ObTableLoadSequenceNo &seq_no,
const bool is_deleted)
{
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, transfer_external_row_time_us);
int ret = OB_SUCCESS;
@ -110,6 +114,7 @@ int ObDirectLoadExternalRow::from_datums(ObStorageDatum *datums, int64_t column_
buf_ = buf;
buf_size_ = buf_size;
seq_no_ = seq_no;
is_deleted_ = is_deleted;
}
}
}
@ -165,7 +170,7 @@ OB_DEF_SERIALIZE_SIMPLE(ObDirectLoadExternalRow)
{
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_row_serialize_time_us);
int ret = OB_SUCCESS;
LST_DO_CODE(OB_UNIS_ENCODE, rowkey_datum_array_, seq_no_, buf_size_);
LST_DO_CODE(OB_UNIS_ENCODE, rowkey_datum_array_, seq_no_, is_deleted_, buf_size_);
if (OB_SUCC(ret) && OB_NOT_NULL(buf_)) {
MEMCPY(buf + pos, buf_, buf_size_);
pos += buf_size_;
@ -178,7 +183,7 @@ OB_DEF_DESERIALIZE_SIMPLE(ObDirectLoadExternalRow)
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_row_deserialize_time_us);
int ret = OB_SUCCESS;
reuse();
LST_DO_CODE(OB_UNIS_DECODE, rowkey_datum_array_, seq_no_, buf_size_);
LST_DO_CODE(OB_UNIS_DECODE, rowkey_datum_array_, seq_no_, is_deleted_, buf_size_);
if (OB_SUCC(ret)) {
buf_ = buf + pos;
pos += buf_size_;
@ -190,7 +195,7 @@ OB_DEF_SERIALIZE_SIZE_SIMPLE(ObDirectLoadExternalRow)
{
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_row_serialize_time_us);
int64_t len = 0;
LST_DO_CODE(OB_UNIS_ADD_LEN, rowkey_datum_array_, seq_no_, buf_size_);
LST_DO_CODE(OB_UNIS_ADD_LEN, rowkey_datum_array_, seq_no_, is_deleted_, buf_size_);
len += buf_size_;
return len;
}

8
src/storage/direct_load/ob_direct_load_external_row.h
View File

@ -32,20 +32,24 @@ public:
int deep_copy(const ObDirectLoadExternalRow &src, char *buf, const int64_t len, int64_t &pos);
// not deep copy
int from_datums(blocksstable::ObStorageDatum *datums, int64_t column_count,
int64_t rowkey_column_count, const table::ObTableLoadSequenceNo &seq_no);
int64_t rowkey_column_count, const table::ObTableLoadSequenceNo &seq_no,
const bool is_deleted);
int to_datums(blocksstable::ObStorageDatum *datums, int64_t column_count) const;
int get_rowkey(blocksstable::ObDatumRowkey &rowkey) const;
bool is_valid() const
{
return rowkey_datum_array_.is_valid() && seq_no_.is_valid() && buf_size_ > 0 && nullptr != buf_;
}
bool is_deleted() const { return is_deleted_; }
OB_INLINE int64_t get_raw_size() const { return buf_size_; }
TO_STRING_KV(K_(rowkey_datum_array), K_(seq_no), K_(buf_size), KP_(buf));
TO_STRING_KV(K_(rowkey_datum_array), K_(seq_no), K_(is_deleted), K_(buf_size), KP_(buf));
public:
common::ObArenaAllocator allocator_;
ObDirectLoadDatumArray rowkey_datum_array_;
table::ObTableLoadSequenceNo seq_no_;
bool is_deleted_;//default is false
int64_t buf_size_;
const char *buf_;
};

2
src/storage/direct_load/ob_direct_load_external_table_builder.cpp
View File

@ -104,7 +104,7 @@ int ObDirectLoadExternalTableBuilder::append_row(const ObTabletID &tablet_id,
} else {
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_append_row_time_us);
if (OB_FAIL(external_row_.from_datums(datum_row.storage_datums_, datum_row.count_,
build_param_.table_data_desc_.rowkey_column_num_, seq_no))) {
build_param_.table_data_desc_.rowkey_column_num_, seq_no, datum_row.row_flag_.is_delete()))) {
LOG_WARN("fail to from datums", KR(ret));
} else if (OB_FAIL(external_writer_.write_item(external_row_))) {
LOG_WARN("fail to write item", KR(ret));

2
src/storage/direct_load/ob_direct_load_fast_heap_table_builder.cpp
View File

@ -220,7 +220,7 @@ int ObDirectLoadFastHeapTableBuilder::append_row(const ObTabletID &tablet_id,
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(param_.dml_row_handler_->handle_insert_row(datum_row_))) {
if (OB_FAIL(param_.dml_row_handler_->handle_insert_row_with_multi_version(param_.tablet_id_, datum_row_))) {
LOG_WARN("fail to handle insert row", KR(ret), K_(datum_row));
}
}

4
src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp
View File

@ -776,7 +776,7 @@ int ObDirectLoadInsertTableContext::init(
MTL(ObTenantDirectLoadMgr *)->alloc_execution_context_id(ddl_ctrl_.context_id_))) {
LOG_WARN("alloc execution context id failed", K(ret));
} else if (OB_FAIL(create_all_tablet_contexts(ls_partition_ids, target_ls_partition_ids))) {
LOG_WARN("fail to create all tablet contexts", KR(ret));
LOG_WARN("fail to create all tablet contexts", KR(ret), K(ls_partition_ids), K(target_ls_partition_ids));
} else if (param_.online_opt_stat_gather_ &&
sql_stat_map_.create(1024, "TLD_SqlStatMap", "TLD_SqlStatMap", MTL_ID())) {
LOG_WARN("fail to create sql stat map", KR(ret));
@ -800,7 +800,7 @@ int ObDirectLoadInsertTableContext::create_all_tablet_contexts(
if (OB_ISNULL(tablet_ctx = OB_NEWx(ObDirectLoadInsertTabletContext, (&allocator_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObDirectLoadInsertTabletContext", KR(ret));
} else if (OB_FAIL(tablet_ctx->init(this, ls_id, origin_tablet_id, tablet_id))) {
} else if (OB_FAIL(tablet_ctx->init(this, ls_id, origin_tablet_id, tablet_id))){
LOG_WARN("fail to init fast heap table tablet ctx", KR(ret));
} else if (OB_FAIL(tablet_ctx_map_.set_refactored(origin_tablet_id, tablet_ctx))) {
LOG_WARN("fail to set tablet ctx map", KR(ret));

4
src/storage/direct_load/ob_direct_load_insert_table_ctx.h
View File

@ -140,6 +140,8 @@ public:
const ObLobId &get_min_insert_lob_id() const { return min_insert_lob_id_; }
OB_INLINE bool is_incremental() const {return nullptr != param_ ? param_->is_incremental_ : false;};
public:
int init(ObDirectLoadInsertTableContext *table_ctx,
const share::ObLSID &ls_id,
@ -252,7 +254,7 @@ public:
OB_INLINE bool need_rescan() const { return (!param_.is_incremental_ && param_.is_column_store_); }
OB_INLINE bool need_del_lob() const { return (param_.is_incremental_ && param_.lob_column_count_ > 0); }
OB_INLINE TABLET_CTX_MAP &get_tablet_ctx_map() { return tablet_ctx_map_; }
int64_t get_sql_stat_column_count() const;
int get_sql_statistics(table::ObTableLoadSqlStatistics *&sql_statistics);
int update_sql_statistics(table::ObTableLoadSqlStatistics &sql_statistics,

15
src/storage/direct_load/ob_direct_load_lob_meta_row_iter.h
View File

@ -28,7 +28,18 @@ class ObDirectLoadLobIdConflictHandler : public ObDirectLoadDMLRowHandler
public:
ObDirectLoadLobIdConflictHandler() {}
virtual ~ObDirectLoadLobIdConflictHandler() {}
int handle_insert_row(const blocksstable::ObDatumRow &row) override { return OB_SUCCESS; }
int handle_insert_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row)
{
return OB_ERR_UNEXPECTED;
}
int handle_delete_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row)
{
return OB_ERR_UNEXPECTED;
}
int handle_insert_row_with_multi_version(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row)
{
return OB_ERR_UNEXPECTED;
}
int handle_update_row(const blocksstable::ObDatumRow &row) override { return OB_ERR_UNEXPECTED; }
int handle_update_row(common::ObArray<const ObDirectLoadExternalRow *> &rows,
const ObDirectLoadExternalRow *&row) override
@ -40,7 +51,7 @@ public:
{
return OB_ERR_UNEXPECTED;
}
int handle_update_row(const blocksstable::ObDatumRow &old_row,
int handle_update_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &old_row,
const blocksstable::ObDatumRow &new_row,
const blocksstable::ObDatumRow *&result_row) override
{

1
src/storage/direct_load/ob_direct_load_mem_dump.cpp
View File

@ -278,6 +278,7 @@ int ObDirectLoadMemDump::dump_tables()
}
}
if (OB_SUCC(ret)) {
datum_row.row_flag_.set_flag(external_row->is_deleted_?ObDmlFlag::DF_DELETE : ObDmlFlag::DF_INSERT);
if (OB_FAIL(external_row->to_datums(datum_row.storage_datums_, datum_row.count_))) {
LOG_WARN("fail to transfer dataum row", KR(ret));
} else if (OB_FAIL(table_builder->append_row(external_row->tablet_id_, external_row->seq_no_, datum_row))) {

1
src/storage/direct_load/ob_direct_load_merge_ctx.cpp
View File

@ -13,6 +13,7 @@
#define USING_LOG_PREFIX STORAGE
#include "storage/direct_load/ob_direct_load_merge_ctx.h"
#include "observer/table_load/ob_table_load_store_ctx.h"
#include "share/ob_tablet_autoincrement_service.h"
#include "storage/direct_load/ob_direct_load_external_table.h"
#include "storage/direct_load/ob_direct_load_multiple_heap_table.h"

7
src/storage/direct_load/ob_direct_load_merge_ctx.h
View File

@ -71,7 +71,8 @@ public:
KP_(insert_table_ctx),
KP_(dml_row_handler),
KP_(file_mgr),
K_(trans_param));
K_(trans_param),
K_(index_table_count));
public:
uint64_t table_id_;
uint64_t lob_meta_table_id_;
@ -95,6 +96,7 @@ public:
ObDirectLoadDMLRowHandler *dml_row_handler_;
ObDirectLoadTmpFileManager *file_mgr_;
ObDirectLoadTransParam trans_param_;
int index_table_count_;
};
class ObDirectLoadMergeCtx
@ -162,8 +164,9 @@ public:
{
return param_.is_incremental_ &&
(param_.insert_mode_ == ObDirectLoadInsertMode::NORMAL ||
(param_.insert_mode_ == ObDirectLoadInsertMode::INC_REPLACE && param_.lob_column_idxs_->count() > 0));
(param_.insert_mode_ == ObDirectLoadInsertMode::INC_REPLACE && (param_.lob_column_idxs_->count() > 0 || param_.index_table_count_ > 0)));
}
const ObDirectLoadMergeParam &get_param() const { return param_; }
const common::ObTabletID &get_tablet_id() const { return tablet_id_; }
const common::ObIArray<ObDirectLoadPartitionMergeTask *> &get_tasks() const

14
src/storage/direct_load/ob_direct_load_multiple_datum_row.cpp
View File

@ -24,7 +24,7 @@ using namespace blocksstable;
using namespace table;
ObDirectLoadMultipleDatumRow::ObDirectLoadMultipleDatumRow()
: allocator_("TLD_MultiRow"), buf_size_(0), buf_(nullptr)
: allocator_("TLD_MultiRow"), is_deleted_(false), buf_size_(0), buf_(nullptr)
{
allocator_.set_tenant_id(MTL_ID());
}
@ -37,6 +37,7 @@ void ObDirectLoadMultipleDatumRow::reset()
{
rowkey_.reset();
seq_no_.reset();
is_deleted_ = false;
buf_size_ = 0;
buf_ = nullptr;
allocator_.reset();
@ -46,6 +47,7 @@ void ObDirectLoadMultipleDatumRow::reuse()
{
rowkey_.reuse();
seq_no_.reset();
is_deleted_ = false;
buf_size_ = 0;
buf_ = nullptr;
allocator_.reuse();
@ -77,6 +79,7 @@ int ObDirectLoadMultipleDatumRow::deep_copy(const ObDirectLoadMultipleDatumRow &
} else {
buf_size_ = src.buf_size_;
seq_no_ = src.seq_no_;
is_deleted_ = src.is_deleted_;
buf_ = buf + pos;
MEMCPY(buf + pos, src.buf_, buf_size_);
pos += buf_size_;
@ -86,7 +89,7 @@ int ObDirectLoadMultipleDatumRow::deep_copy(const ObDirectLoadMultipleDatumRow &
}
int ObDirectLoadMultipleDatumRow::from_datums(const ObTabletID &tablet_id, ObStorageDatum *datums,
int64_t column_count, int64_t rowkey_column_count, const ObTableLoadSequenceNo &seq_no)
int64_t column_count, int64_t rowkey_column_count, const ObTableLoadSequenceNo &seq_no, const bool is_deleted)
{
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, transfer_external_row_time_us);
int ret = OB_SUCCESS;
@ -98,6 +101,7 @@ int ObDirectLoadMultipleDatumRow::from_datums(const ObTabletID &tablet_id, ObSto
} else {
reuse();
seq_no_ = seq_no;
is_deleted_ = is_deleted;
ObDirectLoadDatumArray serialize_datum_array;
if (OB_FAIL(rowkey_.assign(tablet_id, datums, rowkey_column_count))) {
LOG_WARN("fail to assign rowkey", KR(ret));
@ -171,7 +175,7 @@ OB_DEF_SERIALIZE_SIMPLE(ObDirectLoadMultipleDatumRow)
{
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_row_serialize_time_us);
int ret = OB_SUCCESS;
LST_DO_CODE(OB_UNIS_ENCODE, rowkey_, seq_no_, buf_size_);
LST_DO_CODE(OB_UNIS_ENCODE, rowkey_, seq_no_, is_deleted_, buf_size_);
if (OB_SUCC(ret) && OB_NOT_NULL(buf_)) {
MEMCPY(buf + pos, buf_, buf_size_);
pos += buf_size_;
@ -184,7 +188,7 @@ OB_DEF_DESERIALIZE_SIMPLE(ObDirectLoadMultipleDatumRow)
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_row_deserialize_time_us);
int ret = OB_SUCCESS;
reuse();
LST_DO_CODE(OB_UNIS_DECODE, rowkey_, seq_no_, buf_size_);
LST_DO_CODE(OB_UNIS_DECODE, rowkey_, seq_no_, is_deleted_, buf_size_);
if (OB_SUCC(ret)) {
buf_ = buf + pos;
pos += buf_size_;
@ -196,7 +200,7 @@ OB_DEF_SERIALIZE_SIZE_SIMPLE(ObDirectLoadMultipleDatumRow)
{
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_row_serialize_time_us);
int64_t len = 0;
LST_DO_CODE(OB_UNIS_ADD_LEN, rowkey_, seq_no_, buf_size_);
LST_DO_CODE(OB_UNIS_ADD_LEN, rowkey_, seq_no_, is_deleted_, buf_size_);
len += buf_size_;
return len;
}

3
src/storage/direct_load/ob_direct_load_multiple_datum_row.h
View File

@ -33,7 +33,7 @@ public:
// not deep copy
int from_datums(const common::ObTabletID &tablet_id, blocksstable::ObStorageDatum *datums,
int64_t column_count, int64_t rowkey_column_count,
const table::ObTableLoadSequenceNo &seq_no);
const table::ObTableLoadSequenceNo &seq_no, const bool is_deleted);
int to_datums(blocksstable::ObStorageDatum *datums, int64_t column_count) const;
OB_INLINE bool is_valid() const
{
@ -46,6 +46,7 @@ public:
common::ObArenaAllocator allocator_;
ObDirectLoadMultipleDatumRowkey rowkey_;
table::ObTableLoadSequenceNo seq_no_;
bool is_deleted_;
int64_t buf_size_;
const char *buf_;
};

2
src/storage/direct_load/ob_direct_load_multiple_sstable_builder.cpp
View File

@ -165,7 +165,7 @@ int ObDirectLoadMultipleSSTableBuilder::append_row(const ObTabletID &tablet_id,
LOG_WARN("invalid args", KR(ret), K(param_), K(datum_row));
} else {
if (OB_FAIL(row_.from_datums(tablet_id, datum_row.storage_datums_, datum_row.count_,
param_.table_data_desc_.rowkey_column_num_, seq_no))) {
param_.table_data_desc_.rowkey_column_num_, seq_no, datum_row.row_flag_.is_delete()))) {
LOG_WARN("fail to from datum row", KR(ret));
} else if (OB_FAIL(append_row(row_))) {
LOG_WARN("fail to append row", KR(ret), K(row_));

1
src/storage/direct_load/ob_direct_load_multiple_sstable_scan_merge.cpp
View File

@ -286,6 +286,7 @@ int ObDirectLoadMultipleSSTableScanMerge::get_next_row(const ObDatumRow *&datum_
multiple_datum_row->to_datums(datum_row_.storage_datums_, datum_row_.count_))) {
LOG_WARN("fail to transfer datum row", KR(ret));
} else {
datum_row_.row_flag_.set_flag(multiple_datum_row->is_deleted_? ObDmlFlag::DF_DELETE : ObDmlFlag::DF_INSERT);
datum_row = &datum_row_;
}
}

14
src/storage/direct_load/ob_direct_load_partition_merge_task.cpp
View File

@ -301,6 +301,8 @@ int ObDirectLoadPartitionRangeMergeTask::RowIterator::inner_get_next_row(ObDatum
i < datum_row->count_; ++i, ++j) {
datum_row_.storage_datums_[j] = datum_row->storage_datums_[i];
}
datum_row_.row_flag_.set_flag(datum_row->row_flag_.is_delete() ? blocksstable::DF_DELETE : blocksstable::DF_INSERT,
datum_row->row_flag_.is_delete() || !insert_tablet_ctx_->is_incremental() ? DF_TYPE_NORMAL : DF_TYPE_INSERT_DELETE);
result_row = &datum_row_;
}
}
@ -550,6 +552,8 @@ int ObDirectLoadPartitionRangeMultipleMergeTask::RowIterator::inner_get_next_row
i < datum_row->count_; ++i, ++j) {
datum_row_.storage_datums_[j] = datum_row->storage_datums_[i];
}
datum_row_.row_flag_.set_flag(datum_row->row_flag_.is_delete() ? blocksstable::DF_DELETE : blocksstable::DF_INSERT,
datum_row->row_flag_.is_delete() || !insert_tablet_ctx_->is_incremental() ? DF_TYPE_NORMAL : DF_TYPE_INSERT_DELETE);
result_row = &datum_row_;
}
}
@ -716,6 +720,7 @@ int ObDirectLoadPartitionHeapTableMergeTask::RowIterator::init(
deserialize_datum_cnt_ = merge_param.store_column_count_ - merge_param.rowkey_column_num_;
pk_interval_ = pk_interval;
dml_row_handler_ = merge_param.dml_row_handler_;
tablet_id_ = tablet_id;
is_inited_ = true;
}
}
@ -747,7 +752,7 @@ int ObDirectLoadPartitionHeapTableMergeTask::RowIterator::inner_get_next_row(
result_row = &datum_row_;
}
if (OB_SUCC(ret)) {
if (OB_FAIL(dml_row_handler_->handle_insert_row(*result_row))) {
if (OB_FAIL(dml_row_handler_->handle_insert_row_with_multi_version(tablet_id_, *result_row))) {
LOG_WARN("fail to handle insert row", KR(ret), KPC(result_row));
}
}
@ -873,6 +878,7 @@ int ObDirectLoadPartitionHeapTableMultipleMergeTask::RowIterator::init(
deserialize_datum_cnt_ = merge_param.store_column_count_ - merge_param.rowkey_column_num_;
pk_interval_ = pk_interval;
dml_row_handler_ = merge_param.dml_row_handler_;
tablet_id_ = tablet_id;
is_inited_ = true;
}
}
@ -905,7 +911,7 @@ int ObDirectLoadPartitionHeapTableMultipleMergeTask::RowIterator::inner_get_next
result_row = &datum_row_;
}
if (OB_SUCC(ret)) {
if (OB_FAIL(dml_row_handler_->handle_insert_row(*result_row))) {
if (OB_FAIL(dml_row_handler_->handle_insert_row_with_multi_version(tablet_id_, *result_row))) {
LOG_WARN("fail to handle insert row", KR(ret), KPC(result_row));
}
}
@ -1107,6 +1113,8 @@ int ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::RowIterator::inner
i < datum_row->count_; ++i, ++j) {
datum_row_.storage_datums_[j] = datum_row->storage_datums_[i];
}
datum_row_.row_flag_.set_flag(datum_row->row_flag_.is_delete() ? blocksstable::DF_DELETE : blocksstable::DF_INSERT,
datum_row->row_flag_.is_delete() || !insert_tablet_ctx_->is_incremental() ? DF_TYPE_NORMAL : DF_TYPE_INSERT_DELETE);
result_row = &datum_row_;
}
}
@ -1137,7 +1145,7 @@ int ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::RowIterator::inner
result_row = &datum_row_;
}
if (OB_SUCC(ret) && nullptr != result_row) {
if (OB_FAIL(dml_row_handler_->handle_insert_row(*result_row))) {
if (OB_FAIL(dml_row_handler_->handle_insert_row_with_multi_version(tablet_id_, *result_row))) {
LOG_WARN("fail to handle insert row", KR(ret), KPC(result_row));
}
}

2
src/storage/direct_load/ob_direct_load_partition_merge_task.h
View File

@ -189,6 +189,7 @@ private:
int64_t deserialize_datum_cnt_;
share::ObTabletCacheInterval pk_interval_;
ObDirectLoadDMLRowHandler *dml_row_handler_;
ObTabletID tablet_id_;
};
private:
ObDirectLoadExternalTable *external_table_;
@ -229,6 +230,7 @@ private:
int64_t deserialize_datum_cnt_;
share::ObTabletCacheInterval pk_interval_;
ObDirectLoadDMLRowHandler *dml_row_handler_;
ObTabletID tablet_id_;
};
private:
ObDirectLoadMultipleHeapTable *heap_table_;

2
src/storage/direct_load/ob_direct_load_sstable_builder.cpp
View File

@ -102,7 +102,7 @@ int ObDirectLoadSSTableBuilder::append_row(const ObTabletID &tablet_id, const ta
if (OB_FAIL(check_rowkey_order(key))) {
LOG_WARN("fail to check rowkey order", KR(ret), K(datum_row));
} else if (OB_FAIL(external_row.from_datums(datum_row.storage_datums_, datum_row.count_,
param_.table_data_desc_.rowkey_column_num_, seq_no))) {
param_.table_data_desc_.rowkey_column_num_, seq_no, datum_row.row_flag_.is_delete()))) {
LOG_WARN("fail to from datum row", KR(ret));
} else if (OB_FAIL(data_block_writer_.append_row(external_row))) {
LOG_WARN("fail to append row to data block writer", KR(ret), K(external_row));

1
src/storage/direct_load/ob_direct_load_sstable_scan_merge.cpp
View File

@ -289,6 +289,7 @@ int ObDirectLoadSSTableScanMerge::get_next_row(const ObDatumRow *&datum_row)
} else if (OB_FAIL(external_row->to_datums(datum_row_.storage_datums_, datum_row_.count_))) {
LOG_WARN("fail to transfer datum row", KR(ret));
} else {
datum_row_.row_flag_.set_flag(external_row->is_deleted() ? DF_DELETE : DF_INSERT);
datum_row = &datum_row_;
}
}

1
src/storage/direct_load/ob_direct_load_sstable_scanner.cpp
View File

@ -558,6 +558,7 @@ int ObDirectLoadSSTableScanner::get_next_row(const ObDatumRow *&datum_row)
} else if (OB_FAIL(external_row->to_datums(datum_row_.storage_datums_, datum_row_.count_))) {
LOG_WARN("fail to transfer datum row", KR(ret));
} else {
datum_row_.row_flag_.set_flag(external_row->is_deleted() ? DF_DELETE: DF_INSERT);
datum_row = &datum_row_;
}
}