hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

display of merge progress

Browse Source
This commit is contained in:
coolfishchen
2024-04-07 07:24:50 +00:00
committed by ob-robot
parent 784d2231f6
commit cac0a7d1e6
26 changed files with 226 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/storage/direct_load/ob_direct_load_mem_dump.cpp
View File

@ -26,6 +26,7 @@ using namespace common;
using namespace blocksstable;
using namespace table;
using namespace sql;
using namespace observer;
/**
* Context
@ -76,10 +77,12 @@ int ObDirectLoadMemDump::Context::add_table(const ObTabletID &tablet_id, int64_t
* ObDirectLoadMemDump
*/
ObDirectLoadMemDump::ObDirectLoadMemDump(ObDirectLoadMemContext *mem_ctx,
ObDirectLoadMemDump::ObDirectLoadMemDump(ObTableLoadTableCtx *ctx,
ObDirectLoadMemContext *mem_ctx,
const RangeType &range,
ObTableLoadHandle<Context> context_ptr, int64_t range_idx)
: allocator_("TLD_MemDump"),
ctx_(ctx),
mem_ctx_(mem_ctx),
range_(range),
context_ptr_(context_ptr),
@ -285,6 +288,8 @@ int ObDirectLoadMemDump::dump_tables()
} else {
LOG_WARN("fail to append row", KR(ret), K(datum_row));
}
} else {
ATOMIC_AAF(&ctx_->job_stat_->store_.compact_stage_dump_rows_, 1);
}
}
}
@ -382,6 +387,9 @@ int ObDirectLoadMemDump::compact_tables()
LOG_WARN("fail to compact tablet tables", KR(ret));
}
}
if (OB_SUCC(ret)) {
ATOMIC_AAF(&ctx_->job_stat_->store_.compact_stage_product_tmp_files_, keys.count());
}
return ret;
}

5
src/storage/direct_load/ob_direct_load_mem_dump.h
View File

@ -17,6 +17,7 @@
#include "storage/direct_load/ob_direct_load_mem_define.h"
#include "storage/direct_load/ob_direct_load_multi_map.h"
#include "storage/direct_load/ob_direct_load_sstable_builder.h"
#include "observer/table_load/ob_table_load_table_ctx.h"
namespace oceanbase
{
@ -61,7 +62,8 @@ public:
};
public:
ObDirectLoadMemDump(ObDirectLoadMemContext *mem_ctx,
ObDirectLoadMemDump(observer::ObTableLoadTableCtx *ctx,
ObDirectLoadMemContext *mem_ctx,
const RangeType &range,
table::ObTableLoadHandle<Context> context_ptr, int64_t range_idx);
~ObDirectLoadMemDump();
@ -91,6 +93,7 @@ private:
private:
// data members
ObArenaAllocator allocator_;
observer::ObTableLoadTableCtx *ctx_;
ObDirectLoadMemContext *mem_ctx_;
RangeType range_;
table::ObTableLoadHandle<Context> context_ptr_;

5
src/storage/direct_load/ob_direct_load_mem_loader.cpp
View File

@ -27,8 +27,8 @@ using namespace blocksstable;
* ObDirectLoadMemLoader
*/
ObDirectLoadMemLoader::ObDirectLoadMemLoader(ObDirectLoadMemContext *mem_ctx)
: mem_ctx_(mem_ctx)
ObDirectLoadMemLoader::ObDirectLoadMemLoader(observer::ObTableLoadTableCtx *ctx, ObDirectLoadMemContext *mem_ctx)
: ctx_(ctx), mem_ctx_(mem_ctx)
{
}
@ -121,6 +121,7 @@ int ObDirectLoadMemLoader::work()
LOG_WARN("fail to add item", KR(ret));
} else {
external_row = nullptr;
ATOMIC_AAF(&ctx_->job_stat_->store_.compact_stage_load_rows_, 1);
}
}
}

5
src/storage/direct_load/ob_direct_load_mem_loader.h
View File

@ -15,6 +15,8 @@
#include "storage/direct_load/ob_direct_load_i_table.h"
#include "storage/direct_load/ob_direct_load_mem_context.h"
#include "storage/direct_load/ob_direct_load_mem_worker.h"
#include "observer/table_load/ob_table_load_service.h"
#include "observer/table_load/ob_table_load_table_ctx.h"
namespace oceanbase
{
@ -27,7 +29,7 @@ class ObDirectLoadMemLoader : public ObDirectLoadMemWorker
typedef ObDirectLoadExternalMultiPartitionRowChunk ChunkType;
typedef ObDirectLoadExternalMultiPartitionRowCompare CompareType;
public:
ObDirectLoadMemLoader(ObDirectLoadMemContext *mem_ctx);
ObDirectLoadMemLoader(observer::ObTableLoadTableCtx *ctx, ObDirectLoadMemContext *mem_ctx);
virtual ~ObDirectLoadMemLoader();
int add_table(ObIDirectLoadPartitionTable *table) override;
int work() override;
@ -35,6 +37,7 @@ public:
private:
int close_chunk(ChunkType *&chunk);
private:
observer::ObTableLoadTableCtx *ctx_;
ObDirectLoadMemContext *mem_ctx_;
ObDirectLoadExternalFragmentArray fragments_;
};

6
src/storage/direct_load/ob_direct_load_mem_sample.cpp
View File

@ -26,8 +26,8 @@ using namespace blocksstable;
using namespace observer;
using namespace table;
ObDirectLoadMemSample::ObDirectLoadMemSample(ObDirectLoadMemContext *mem_ctx)
: mem_ctx_(mem_ctx), range_count_(mem_ctx_->mem_dump_task_count_) {}
ObDirectLoadMemSample::ObDirectLoadMemSample(observer::ObTableLoadTableCtx *ctx, ObDirectLoadMemContext *mem_ctx)
: ctx_(ctx), mem_ctx_(mem_ctx), range_count_(mem_ctx_->mem_dump_task_count_) {}
int ObDirectLoadMemSample::gen_ranges(ObIArray<ChunkType *> &chunks, ObIArray<RangeType> &ranges)
@ -125,7 +125,7 @@ int ObDirectLoadMemSample::add_dump(int64_t idx,
{
int ret = OB_SUCCESS;
storage::ObDirectLoadMemDump *mem_dump = OB_NEW(
ObDirectLoadMemDump, ObMemAttr(MTL_ID(), "TLD_mem_dump"), mem_ctx_, range, context_ptr, idx);
ObDirectLoadMemDump, ObMemAttr(MTL_ID(), "TLD_mem_dump"), ctx_, mem_ctx_, range, context_ptr, idx);
if (mem_dump == nullptr) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to allocate mem dump", KR(ret));

4
src/storage/direct_load/ob_direct_load_mem_sample.h
View File

@ -17,6 +17,7 @@
#include <memory>
#include "storage/direct_load/ob_direct_load_mem_dump.h"
#include "storage/direct_load/ob_direct_load_mem_context.h"
#include "observer/table_load/ob_table_load_table_ctx.h"
namespace oceanbase
{
@ -31,7 +32,7 @@ class ObDirectLoadMemSample
typedef ObDirectLoadExternalMultiPartitionRowRange RangeType;
typedef ObDirectLoadExternalMultiPartitionRowCompare CompareType;
public:
ObDirectLoadMemSample(ObDirectLoadMemContext *mem_ctx);
ObDirectLoadMemSample(observer::ObTableLoadTableCtx *ctx, ObDirectLoadMemContext *mem_ctx);
virtual ~ObDirectLoadMemSample() {}
int do_sample();
@ -47,6 +48,7 @@ private:
private:
// data members
observer::ObTableLoadTableCtx *ctx_;
ObDirectLoadMemContext *mem_ctx_;
int64_t range_count_;
};

36
src/storage/direct_load/ob_direct_load_merge_ctx.cpp
View File

@ -81,7 +81,7 @@ bool ObDirectLoadMergeParam::is_valid() const
*/
ObDirectLoadMergeCtx::ObDirectLoadMergeCtx()
: allocator_("TLD_MergeCtx"), is_inited_(false)
: allocator_("TLD_MergeCtx"), ctx_(nullptr), is_inited_(false)
{
allocator_.set_tenant_id(MTL_ID());
tablet_merge_ctx_array_.set_tenant_id(MTL_ID());
@ -97,7 +97,8 @@ ObDirectLoadMergeCtx::~ObDirectLoadMergeCtx()
tablet_merge_ctx_array_.reset();
}
int ObDirectLoadMergeCtx::init(const ObDirectLoadMergeParam &param,
int ObDirectLoadMergeCtx::init(ObTableLoadTableCtx *ctx,
const ObDirectLoadMergeParam &param,
const ObIArray<ObTableLoadLSIdAndPartitionId> &ls_partition_ids,
const ObIArray<ObTableLoadLSIdAndPartitionId> &target_ls_partition_ids)
{
@ -105,13 +106,15 @@ int ObDirectLoadMergeCtx::init(const ObDirectLoadMergeParam &param,
if (IS_INIT) {
ret = OB_INIT_TWICE;
LOG_WARN("ObDirectLoadMerger init twice", KR(ret), KP(this));
} else if (OB_UNLIKELY(!param.is_valid()
} else if (OB_UNLIKELY(nullptr == ctx
|| !param.is_valid()
|| ls_partition_ids.empty()
|| target_ls_partition_ids.empty()
|| (ls_partition_ids.count() != target_ls_partition_ids.count()))) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(param), K(ls_partition_ids), K(target_ls_partition_ids));
} else {
ctx_ = ctx;
param_ = param;
if (OB_FAIL(create_all_tablet_ctxs(ls_partition_ids, target_ls_partition_ids))) {
LOG_WARN("fail to create all tablet ctxs", KR(ret));
@ -138,7 +141,7 @@ int ObDirectLoadMergeCtx::create_all_tablet_ctxs(
if (OB_ISNULL(partition_ctx = OB_NEWx(ObDirectLoadTabletMergeCtx, (&allocator_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObDirectLoadTabletMergeCtx", KR(ret));
} else if (OB_FAIL(partition_ctx->init(param_, ls_partition_id, target_ls_partition_id))) {
} else if (OB_FAIL(partition_ctx->init(ctx_, param_, ls_partition_id, target_ls_partition_id))) {
LOG_WARN("fail to init tablet ctx", KR(ret), K(param_), K(ls_partition_id), K(target_ls_partition_id));
} else if (OB_FAIL(tablet_merge_ctx_array_.push_back(partition_ctx))) {
LOG_WARN("fail to push back", KR(ret));
@ -186,7 +189,8 @@ ObDirectLoadTabletMergeCtx::~ObDirectLoadTabletMergeCtx()
rescan_task_array_.reset();
}
int ObDirectLoadTabletMergeCtx::init(const ObDirectLoadMergeParam &param,
int ObDirectLoadTabletMergeCtx::init(ObTableLoadTableCtx *ctx,
const ObDirectLoadMergeParam &param,
const ObTableLoadLSIdAndPartitionId &ls_partition_id,
const ObTableLoadLSIdAndPartitionId &target_ls_partition_id)
@ -195,7 +199,8 @@ int ObDirectLoadTabletMergeCtx::init(const ObDirectLoadMergeParam &param,
if (IS_INIT) {
ret = OB_INIT_TWICE;
LOG_WARN("ObDirectLoadTabletMergeCtx init twice", KR(ret), KP(this));
} else if (OB_UNLIKELY(!param.is_valid()
} else if (OB_UNLIKELY(nullptr == ctx
|| !param.is_valid()
|| !ls_partition_id.is_valid()
|| !target_ls_partition_id.is_valid())) {
ret = OB_INVALID_ARGUMENT;
@ -208,6 +213,7 @@ int ObDirectLoadTabletMergeCtx::init(const ObDirectLoadMergeParam &param,
if (OB_FAIL(origin_table_.init(origin_table_param))) {
LOG_WARN("fail to init origin sstable", KR(ret));
} else {
ctx_ = ctx;
param_ = param;
target_partition_id_ = target_ls_partition_id.part_tablet_id_.partition_id_;
tablet_id_ = ls_partition_id.part_tablet_id_.tablet_id_;
@ -449,7 +455,7 @@ int ObDirectLoadTabletMergeCtx::build_empty_data_merge_task(const ObIArray<ObCol
if (OB_ISNULL(merge_task = OB_NEWx(ObDirectLoadPartitionRangeMergeTask, (&allocator_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObDirectLoadPartitionRangeMergeTask", KR(ret));
} else if (OB_FAIL(merge_task->init(param_, this, &origin_table_, sstable_array_, range, i))) {
} else if (OB_FAIL(merge_task->init(ctx_, param_, this, &origin_table_, sstable_array_, range, i))) {
LOG_WARN("fail to init merge task", KR(ret));
} else if (OB_FAIL(task_array_.push_back(merge_task))) {
LOG_WARN("fail to push back merge task", KR(ret));
@ -494,7 +500,7 @@ int ObDirectLoadTabletMergeCtx::build_pk_table_merge_task(
if (OB_ISNULL(merge_task = OB_NEWx(ObDirectLoadPartitionRangeMergeTask, (&allocator_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObDirectLoadPartitionRangeMergeTask", KR(ret));
} else if (OB_FAIL(merge_task->init(param_, this, &origin_table_, sstable_array_, range, i))) {
} else if (OB_FAIL(merge_task->init(ctx_, param_, this, &origin_table_, sstable_array_, range, i))) {
LOG_WARN("fail to init merge task", KR(ret));
} else if (OB_FAIL(task_array_.push_back(merge_task))) {
LOG_WARN("fail to push back merge task", KR(ret));
@ -540,7 +546,7 @@ int ObDirectLoadTabletMergeCtx::build_pk_table_multiple_merge_task(
OB_NEWx(ObDirectLoadPartitionRangeMultipleMergeTask, (&allocator_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObDirectLoadPartitionRangeMultipleMergeTask", KR(ret));
} else if (OB_FAIL(merge_task->init(param_, this, &origin_table_, multiple_sstable_array_,
} else if (OB_FAIL(merge_task->init(ctx_, param_, this, &origin_table_, multiple_sstable_array_,
range, i))) {
LOG_WARN("fail to init merge task", KR(ret));
} else if (OB_FAIL(task_array_.push_back(merge_task))) {
@ -587,7 +593,7 @@ int ObDirectLoadTabletMergeCtx::build_merge_task_for_multiple_pk_table(
OB_NEWx(ObDirectLoadPartitionRangeMultipleMergeTask, (&allocator_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObDirectLoadPartitionRangeMultipleMergeTask", KR(ret));
} else if (OB_FAIL(merge_task->init(param_, this, &origin_table_, multiple_sstable_array_,
} else if (OB_FAIL(merge_task->init(ctx_, param_, this, &origin_table_, multiple_sstable_array_,
range, i))) {
LOG_WARN("fail to init merge task", KR(ret));
} else if (OB_FAIL(task_array_.push_back(merge_task))) {
@ -631,7 +637,7 @@ int ObDirectLoadTabletMergeCtx::build_heap_table_merge_task(
if (OB_ISNULL(merge_task = OB_NEWx(ObDirectLoadPartitionRangeMergeTask, (&allocator_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObDirectLoadPartitionRangeMergeTask", KR(ret));
} else if (OB_FAIL(merge_task->init(param_, this, &origin_table_, sstable_array_, range,
} else if (OB_FAIL(merge_task->init(ctx_, param_, this, &origin_table_, sstable_array_, range,
parallel_idx++))) {
LOG_WARN("fail to init merge task", KR(ret));
} else if (OB_FAIL(task_array_.push_back(merge_task))) {
@ -659,7 +665,7 @@ int ObDirectLoadTabletMergeCtx::build_heap_table_merge_task(
OB_NEWx(ObDirectLoadPartitionHeapTableMergeTask, (&allocator_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObDirectLoadPartitionHeapTableMergeTask", KR(ret));
} else if (OB_FAIL(merge_task->init(param_, this, external_table, pk_interval,
} else if (OB_FAIL(merge_task->init(ctx_, param_, this, external_table, pk_interval,
parallel_idx++))) {
LOG_WARN("fail to init merge task", KR(ret));
} else if (OB_FAIL(task_array_.push_back(merge_task))) {
@ -705,7 +711,7 @@ int ObDirectLoadTabletMergeCtx::build_heap_table_multiple_merge_task(
if (OB_ISNULL(merge_task = OB_NEWx(ObDirectLoadPartitionRangeMergeTask, (&allocator_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObDirectLoadPartitionRangeMergeTask", KR(ret));
} else if (OB_FAIL(merge_task->init(param_, this, &origin_table_, sstable_array_, range,
} else if (OB_FAIL(merge_task->init(ctx_, param_, this, &origin_table_, sstable_array_, range,
parallel_idx++))) {
LOG_WARN("fail to init merge task", KR(ret));
} else if (OB_FAIL(task_array_.push_back(merge_task))) {
@ -735,7 +741,7 @@ int ObDirectLoadTabletMergeCtx::build_heap_table_multiple_merge_task(
(&allocator_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObDirectLoadPartitionHeapTableMultipleMergeTask", KR(ret));
} else if (OB_FAIL(merge_task->init(param_, this, heap_table, pk_interval, parallel_idx++))) {
} else if (OB_FAIL(merge_task->init(ctx_, param_, this, heap_table, pk_interval, parallel_idx++))) {
LOG_WARN("fail to init merge task", KR(ret));
} else if (OB_FAIL(task_array_.push_back(merge_task))) {
LOG_WARN("fail to push back merge task", KR(ret));
@ -778,7 +784,7 @@ int ObDirectLoadTabletMergeCtx::build_aggregate_merge_task_for_multiple_heap_tab
(&allocator_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask", KR(ret));
} else if (OB_FAIL(merge_task->init(param_, this, &origin_table_, multiple_heap_table_array_,
} else if (OB_FAIL(merge_task->init(ctx_, param_, this, &origin_table_, multiple_heap_table_array_,
pk_interval))) {
LOG_WARN("fail to init merge task", KR(ret));
} else if (OB_FAIL(task_array_.push_back(merge_task))) {

12
src/storage/direct_load/ob_direct_load_merge_ctx.h
View File

@ -24,6 +24,7 @@
#include "storage/direct_load/ob_direct_load_origin_table.h"
#include "storage/direct_load/ob_direct_load_table_data_desc.h"
#include "storage/direct_load/ob_direct_load_fast_heap_table.h"
#include "observer/table_load/ob_table_load_table_ctx.h"
namespace oceanbase
{
@ -81,7 +82,8 @@ class ObDirectLoadMergeCtx
public:
ObDirectLoadMergeCtx();
~ObDirectLoadMergeCtx();
int init(const ObDirectLoadMergeParam &param,
int init(observer::ObTableLoadTableCtx *ctx,
const ObDirectLoadMergeParam &param,
const common::ObIArray<table::ObTableLoadLSIdAndPartitionId> &ls_partition_ids,
const common::ObIArray<table::ObTableLoadLSIdAndPartitionId> &target_ls_partition_ids);
const common::ObIArray<ObDirectLoadTabletMergeCtx *> &get_tablet_merge_ctxs() const
@ -93,6 +95,7 @@ private:
const common::ObIArray<table::ObTableLoadLSIdAndPartitionId> &target_ls_partition_ids);
private:
common::ObArenaAllocator allocator_;
observer::ObTableLoadTableCtx *ctx_;
ObDirectLoadMergeParam param_;
common::ObArray<ObDirectLoadTabletMergeCtx *> tablet_merge_ctx_array_;
bool is_inited_;
@ -103,8 +106,10 @@ class ObDirectLoadTabletMergeCtx
public:
ObDirectLoadTabletMergeCtx();
~ObDirectLoadTabletMergeCtx();
int init(const ObDirectLoadMergeParam &param, const table::ObTableLoadLSIdAndPartitionId &ls_partition_id,
const table::ObTableLoadLSIdAndPartitionId &target_ls_partition_id);
int init(observer::ObTableLoadTableCtx *ctx,
const ObDirectLoadMergeParam &param,
const table::ObTableLoadLSIdAndPartitionId &ls_partition_id,
const table::ObTableLoadLSIdAndPartitionId &target_ls_partition_id);
int build_rescan_task(int64_t thread_count);
int build_merge_task(const common::ObIArray<ObIDirectLoadPartitionTable *> &table_array,
const common::ObIArray<share::schema::ObColDesc> &col_descs,
@ -159,6 +164,7 @@ private:
int get_autoincrement_value(uint64_t count, share::ObTabletCacheInterval &interval);
private:
common::ObArenaAllocator allocator_;
observer::ObTableLoadTableCtx *ctx_;
ObDirectLoadMergeParam param_;
uint64_t target_partition_id_;
common::ObTabletID tablet_id_;

6
src/storage/direct_load/ob_direct_load_multiple_heap_table_sorter.cpp
View File

@ -27,7 +27,8 @@ using namespace blocksstable;
ObDirectLoadMultipleHeapTableSorter::ObDirectLoadMultipleHeapTableSorter(
ObDirectLoadMemContext *mem_ctx)
: mem_ctx_(mem_ctx),
: ctx_(nullptr),
mem_ctx_(mem_ctx),
allocator_("TLD_Sorter"),
extra_buf_(nullptr),
index_dir_id_(-1),
@ -149,6 +150,8 @@ int ObDirectLoadMultipleHeapTableSorter::get_tables(
LOG_WARN("unexpected table", KR(ret), KPC(table));
} else if (OB_FAIL(heap_table_array_->push_back(heap_table))) {
LOG_WARN("fail to push back heap table", KR(ret));
} else {
ATOMIC_AAF(&ctx_->job_stat_->store_.compact_stage_product_tmp_files_, 1);
}
}
return ret;
@ -218,6 +221,7 @@ int ObDirectLoadMultipleHeapTableSorter::work()
LOG_WARN("fail to add item", KR(ret));
} else {
row = nullptr;
ATOMIC_AAF(&ctx_->job_stat_->store_.compact_stage_load_rows_, 1);
}
}
}

6
src/storage/direct_load/ob_direct_load_multiple_heap_table_sorter.h
View File

@ -16,6 +16,7 @@
#include "storage/direct_load/ob_direct_load_i_table.h"
#include "storage/direct_load/ob_direct_load_mem_context.h"
#include "storage/direct_load/ob_direct_load_mem_worker.h"
#include "observer/table_load/ob_table_load_table_ctx.h"
namespace oceanbase
{
@ -32,10 +33,12 @@ public:
int init();
int add_table(ObIDirectLoadPartitionTable *table) override;
void set_work_param(int64_t index_dir_id, int64_t data_dir_id,
void set_work_param(observer::ObTableLoadTableCtx *ctx,
int64_t index_dir_id, int64_t data_dir_id,
common::ObIArray<ObDirectLoadMultipleHeapTable *> &heap_table_array,
common::ObIAllocator &heap_table_allocator)
{
ctx_ = ctx;
index_dir_id_ = index_dir_id;
data_dir_id_ = data_dir_id;
heap_table_array_ = &heap_table_array;
@ -52,6 +55,7 @@ private:
private:
// data members
observer::ObTableLoadTableCtx *ctx_;
ObDirectLoadMemContext *mem_ctx_;
ObDirectLoadExternalFragmentArray fragments_;
ObArenaAllocator allocator_;

56
src/storage/direct_load/ob_direct_load_partition_merge_task.cpp
View File

@ -28,13 +28,15 @@ namespace storage
using namespace common;
using namespace blocksstable;
using namespace share;
using namespace observer;
/**
* ObDirectLoadPartitionMergeTask
*/
ObDirectLoadPartitionMergeTask::ObDirectLoadPartitionMergeTask()
: merge_param_(nullptr),
: ctx_(nullptr),
merge_param_(nullptr),
merge_ctx_(nullptr),
parallel_idx_(-1),
affected_rows_(0),
@ -57,6 +59,27 @@ ObDirectLoadPartitionMergeTask::~ObDirectLoadPartitionMergeTask()
}
}
class ObStoreRowIteratorWrapper : public ObIStoreRowIterator
{
public:
ObStoreRowIteratorWrapper(observer::ObTableLoadTableCtx *ctx, ObIStoreRowIterator *inner_iter) :
ctx_(ctx), inner_iter_(inner_iter) {
}
int get_next_row(const blocksstable::ObDatumRow *&row)
{
int ret = inner_iter_->get_next_row(row);
if (ret == OB_SUCCESS) {
ATOMIC_AAF(&ctx_->job_stat_->store_.merge_stage_write_rows_, 1);
}
return ret;
}
private:
observer::ObTableLoadTableCtx *ctx_;
ObIStoreRowIterator *inner_iter_;
};
int ObDirectLoadPartitionMergeTask::process()
{
int ret = OB_SUCCESS;
@ -87,10 +110,11 @@ int ObDirectLoadPartitionMergeTask::process()
K(block_start_seq));
} else {
LOG_INFO("add sstable slice begin", KP(tablet_ctx), K(slice_id));
ObStoreRowIteratorWrapper row_iter_wrapper(ctx_, row_iter);
if (OB_UNLIKELY(is_stop_)) {
ret = OB_CANCELED;
LOG_WARN("merge task canceled", KR(ret));
} else if (OB_FAIL(tablet_ctx->fill_sstable_slice(slice_id, *row_iter, affected_rows_))) {
} else if (OB_FAIL(tablet_ctx->fill_sstable_slice(slice_id, row_iter_wrapper, affected_rows_))) {
LOG_WARN("fail to fill sstable slice", KR(ret));
} else if (OB_FAIL(tablet_ctx->close_sstable_slice(slice_id))) {
LOG_WARN("fail to close writer", KR(ret));
@ -290,7 +314,8 @@ ObDirectLoadPartitionRangeMergeTask::~ObDirectLoadPartitionRangeMergeTask()
{
}
int ObDirectLoadPartitionRangeMergeTask::init(const ObDirectLoadMergeParam &merge_param,
int ObDirectLoadPartitionRangeMergeTask::init(ObTableLoadTableCtx *ctx,
const ObDirectLoadMergeParam &merge_param,
ObDirectLoadTabletMergeCtx *merge_ctx,
ObDirectLoadOriginTable *origin_table,
const ObIArray<ObDirectLoadSSTable *> &sstable_array,
@ -301,12 +326,13 @@ int ObDirectLoadPartitionRangeMergeTask::init(const ObDirectLoadMergeParam &merg
if (IS_INIT) {
ret = OB_INIT_TWICE;
LOG_WARN("ObDirectLoadPartitionRangeMergeTask init twice", KR(ret), KP(this));
} else if (OB_UNLIKELY(!merge_param.is_valid() || nullptr == merge_ctx ||
} else if (OB_UNLIKELY(nullptr == ctx || !merge_param.is_valid() || nullptr == merge_ctx ||
nullptr == origin_table || !range.is_valid() || parallel_idx < 0)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(merge_param), KP(merge_ctx), K(sstable_array), K(range),
K(parallel_idx));
} else {
ctx_ = ctx;
merge_param_ = &merge_param;
merge_ctx_ = merge_ctx;
parallel_idx_ = parallel_idx;
@ -464,6 +490,7 @@ ObDirectLoadPartitionRangeMultipleMergeTask::~ObDirectLoadPartitionRangeMultiple
}
int ObDirectLoadPartitionRangeMultipleMergeTask::init(
ObTableLoadTableCtx *ctx,
const ObDirectLoadMergeParam &merge_param,
ObDirectLoadTabletMergeCtx *merge_ctx,
ObDirectLoadOriginTable *origin_table,
@ -475,12 +502,13 @@ int ObDirectLoadPartitionRangeMultipleMergeTask::init(
if (IS_INIT) {
ret = OB_INIT_TWICE;
LOG_WARN("ObDirectLoadPartitionRangeMultipleMergeTask init twice", KR(ret), KP(this));
} else if (OB_UNLIKELY(!merge_param.is_valid() || nullptr == merge_ctx ||
} else if (OB_UNLIKELY(nullptr == ctx || !merge_param.is_valid() || nullptr == merge_ctx ||
nullptr == origin_table || !range.is_valid() || parallel_idx < 0)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(merge_param), KP(merge_ctx), K(sstable_array), K(range),
K(parallel_idx));
} else {
ctx_ = ctx;
merge_param_ = &merge_param;
merge_ctx_ = merge_ctx;
parallel_idx_ = parallel_idx;
@ -643,7 +671,8 @@ ObDirectLoadPartitionHeapTableMergeTask::~ObDirectLoadPartitionHeapTableMergeTas
{
}
int ObDirectLoadPartitionHeapTableMergeTask::init(const ObDirectLoadMergeParam &merge_param,
int ObDirectLoadPartitionHeapTableMergeTask::init(ObTableLoadTableCtx *ctx,
const ObDirectLoadMergeParam &merge_param,
ObDirectLoadTabletMergeCtx *merge_ctx,
ObDirectLoadExternalTable *external_table,
const ObTabletCacheInterval &pk_interval,
@ -653,13 +682,14 @@ int ObDirectLoadPartitionHeapTableMergeTask::init(const ObDirectLoadMergeParam &
if (IS_INIT) {
ret = OB_INIT_TWICE;
LOG_WARN("ObDirectLoadPartitionHeapTableMergeTask init twice", KR(ret), KP(this));
} else if (OB_UNLIKELY(!merge_param.is_valid() || nullptr == merge_ctx ||
} else if (OB_UNLIKELY(nullptr == ctx || !merge_param.is_valid() || nullptr == merge_ctx ||
nullptr == external_table || parallel_idx < 0 ||
0 == pk_interval.count())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(merge_param), KP(merge_ctx), KP(external_table),
K(parallel_idx), K(pk_interval));
} else {
ctx_ = ctx;
merge_param_ = &merge_param;
merge_ctx_ = merge_ctx;
parallel_idx_ = parallel_idx;
@ -820,6 +850,7 @@ ObDirectLoadPartitionHeapTableMultipleMergeTask::~ObDirectLoadPartitionHeapTable
}
int ObDirectLoadPartitionHeapTableMultipleMergeTask::init(
ObTableLoadTableCtx *ctx,
const ObDirectLoadMergeParam &merge_param,
ObDirectLoadTabletMergeCtx *merge_ctx,
ObDirectLoadMultipleHeapTable *heap_table,
@ -830,12 +861,14 @@ int ObDirectLoadPartitionHeapTableMultipleMergeTask::init(
if (IS_INIT) {
ret = OB_INIT_TWICE;
LOG_WARN("ObDirectLoadPartitionHeapTableMultipleMergeTask init twice", KR(ret), KP(this));
} else if (OB_UNLIKELY(!merge_param.is_valid() || nullptr == merge_ctx || nullptr == heap_table ||
} else if (OB_UNLIKELY(nullptr == ctx || !merge_param.is_valid() ||
nullptr == merge_ctx || nullptr == heap_table ||
parallel_idx < 0 || 0 == pk_interval.count())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(merge_param), KP(merge_ctx), KPC(heap_table),
K(parallel_idx), K(pk_interval));
} else {
ctx_ = ctx;
merge_param_ = &merge_param;
merge_ctx_ = merge_ctx;
parallel_idx_ = parallel_idx;
@ -1076,7 +1109,9 @@ ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::
}
int ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::init(
const ObDirectLoadMergeParam &merge_param, ObDirectLoadTabletMergeCtx *merge_ctx,
ObTableLoadTableCtx *ctx,
const ObDirectLoadMergeParam &merge_param,
ObDirectLoadTabletMergeCtx *merge_ctx,
ObDirectLoadOriginTable *origin_table,
const ObIArray<ObDirectLoadMultipleHeapTable *> &heap_table_array,
const ObTabletCacheInterval &pk_interval)
@ -1085,12 +1120,13 @@ int ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::init(
if (IS_INIT) {
ret = OB_INIT_TWICE;
LOG_WARN("ObDirectLoadPartitionHeapTableMultipleMergeTask init twice", KR(ret), KP(this));
} else if (OB_UNLIKELY(!merge_param.is_valid() || nullptr == merge_ctx ||
} else if (OB_UNLIKELY(nullptr == ctx || !merge_param.is_valid() || nullptr == merge_ctx ||
nullptr == origin_table || heap_table_array.empty())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(merge_param), KP(merge_ctx), KP(origin_table),
K(heap_table_array));
} else {
ctx_ = ctx;
merge_param_ = &merge_param;
merge_ctx_ = merge_ctx;
parallel_idx_ = 0;

17
src/storage/direct_load/ob_direct_load_partition_merge_task.h
View File

@ -19,6 +19,7 @@
#include "storage/direct_load/ob_direct_load_multiple_heap_table_scanner.h"
#include "storage/direct_load/ob_direct_load_insert_table_row_iterator.h"
#include "sql/engine/expr/ob_expr_sys_op_opnsize.h"
#include "observer/table_load/ob_table_load_table_ctx.h"
namespace oceanbase
{
@ -57,6 +58,7 @@ private:
int init_sql_statistics();
int init_lob_builder();
protected:
observer::ObTableLoadTableCtx *ctx_;
const ObDirectLoadMergeParam *merge_param_;
ObDirectLoadTabletMergeCtx *merge_ctx_;
int64_t parallel_idx_;
@ -73,7 +75,8 @@ class ObDirectLoadPartitionRangeMergeTask : public ObDirectLoadPartitionMergeTas
public:
ObDirectLoadPartitionRangeMergeTask();
virtual ~ObDirectLoadPartitionRangeMergeTask();
int init(const ObDirectLoadMergeParam &merge_param,
int init(observer::ObTableLoadTableCtx *ctx,
const ObDirectLoadMergeParam &merge_param,
ObDirectLoadTabletMergeCtx *merge_ctx,
ObDirectLoadOriginTable *origin_table,
const common::ObIArray<ObDirectLoadSSTable *> &sstable_array,
@ -112,7 +115,8 @@ class ObDirectLoadPartitionRangeMultipleMergeTask : public ObDirectLoadPartition
public:
ObDirectLoadPartitionRangeMultipleMergeTask();
virtual ~ObDirectLoadPartitionRangeMultipleMergeTask();
int init(const ObDirectLoadMergeParam &merge_param,
int init(observer::ObTableLoadTableCtx *ctx,
const ObDirectLoadMergeParam &merge_param,
ObDirectLoadTabletMergeCtx *merge_ctx,
ObDirectLoadOriginTable *origin_table,
const common::ObIArray<ObDirectLoadMultipleSSTable *> &sstable_array,
@ -151,7 +155,8 @@ class ObDirectLoadPartitionHeapTableMergeTask : public ObDirectLoadPartitionMerg
public:
ObDirectLoadPartitionHeapTableMergeTask();
virtual ~ObDirectLoadPartitionHeapTableMergeTask();
int init(const ObDirectLoadMergeParam &merge_param,
int init(observer::ObTableLoadTableCtx *ctx,
const ObDirectLoadMergeParam &merge_param,
ObDirectLoadTabletMergeCtx *merge_ctx,
ObDirectLoadExternalTable *external_table,
const share::ObTabletCacheInterval &pk_interval,
@ -190,7 +195,8 @@ class ObDirectLoadPartitionHeapTableMultipleMergeTask : public ObDirectLoadParti
public:
ObDirectLoadPartitionHeapTableMultipleMergeTask();
virtual ~ObDirectLoadPartitionHeapTableMultipleMergeTask();
int init(const ObDirectLoadMergeParam &merge_param,
int init(observer::ObTableLoadTableCtx *ctx,
const ObDirectLoadMergeParam &merge_param,
ObDirectLoadTabletMergeCtx *merge_ctx,
ObDirectLoadMultipleHeapTable *heap_table,
const share::ObTabletCacheInterval &pk_interval,
@ -230,7 +236,8 @@ class ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask
public:
ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask();
virtual ~ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask();
int init(const ObDirectLoadMergeParam &merge_param, ObDirectLoadTabletMergeCtx *merge_ctx,
int init(observer::ObTableLoadTableCtx *ctx,
const ObDirectLoadMergeParam &merge_param, ObDirectLoadTabletMergeCtx *merge_ctx,
ObDirectLoadOriginTable *origin_table,
const common::ObIArray<ObDirectLoadMultipleHeapTable *> &heap_table_array,
const share::ObTabletCacheInterval &pk_interval);