hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FTS.BUGFIX] optimize fulltext index insert performance

Browse Source
This commit is contained in:
Tyshawn
2024-04-30 09:45:48 +00:00
committed by ob-robot
parent cc09840d51
commit 7e3d97f4bd
5 changed files with 89 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
src/sql/das/ob_das_dml_ctx_define.cpp
View File

@ -119,6 +119,20 @@ int ObDASDMLIterator::get_next_domain_index_row(ObNewRow *&row)
return ret;
}
int ObDASDMLIterator::get_next_domain_index_rows(ObNewRow *&rows, int64_t &row_count)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(domain_iter_) && OB_FAIL(ObDomainDMLIterator::create_domain_dml_iterator(
allocator_, row_projector_, write_iter_, das_ctdef_, main_ctdef_, domain_iter_))) {
LOG_WARN("fail to create domain index dml iterator", K(ret));
} else if (OB_FAIL(domain_iter_->get_next_domain_rows(rows, row_count))) {
if (OB_ITER_END != ret) {
LOG_WARN("fail to get next domain rows", K(ret));
}
}
return ret;
}
int ObDASDMLIterator::get_next_row(ObNewRow *&row)
{
int ret = OB_SUCCESS;
@ -168,7 +182,7 @@ int ObDASDMLIterator::get_next_rows(ObNewRow *&rows, int64_t &row_count)
int ret = OB_SUCCESS;
const bool is_domain_index = das_ctdef_->table_param_.get_data_table().is_domain_index();
row_count = 0;
if (is_domain_index || 1 == batch_size_) {
if (1 == batch_size_) {
if (OB_FAIL(get_next_row(rows))) {
if (OB_ITER_END != ret) {
LOG_WARN("Failed to get next row", K(ret), K_(batch_size), K(is_domain_index));
@ -184,29 +198,35 @@ int ObDASDMLIterator::get_next_rows(ObNewRow *&rows, int64_t &row_count)
LOG_WARN("Failed to begin write iterator", K(ret));
}
}
while (OB_SUCC(ret) && row_count < batch_size_) {
const ObChunkDatumStore::StoredRow *sr = nullptr;
if (OB_FAIL(write_iter_.get_next_row(sr))) {
if (OB_ITER_END != ret) {
LOG_WARN("Failed to get next row from result iterator", K(ret));
}
} else if (OB_FAIL(ObDASUtils::project_storage_row(*das_ctdef_,
*sr,
*row_projector_,
allocator_,
cur_rows_[row_count]))) {
LOG_WARN("Failed to project storage row", K(ret));
} else {
++row_count;
LOG_TRACE("Get next rows from dml das iterator", KPC(sr), K(cur_rows_[row_count - 1]), K_(das_ctdef));
if (OB_SUCC(ret) && is_domain_index) {
if (OB_FAIL(get_next_domain_index_rows(rows, row_count))) {
LOG_WARN("fail to get next domain index rows", K(ret));
}
}
if (OB_SUCC(ret) || OB_LIKELY(OB_ITER_END == ret)) {
if (0 == row_count) {
ret = OB_ITER_END;
} else {
rows = cur_rows_;
ret = OB_SUCCESS;
} else {
while (OB_SUCC(ret) && row_count < batch_size_) {
const ObChunkDatumStore::StoredRow *sr = nullptr;
if (OB_FAIL(write_iter_.get_next_row(sr))) {
if (OB_ITER_END != ret) {
LOG_WARN("Failed to get next row from result iterator", K(ret));
}
} else if (OB_FAIL(ObDASUtils::project_storage_row(*das_ctdef_,
*sr,
*row_projector_,
allocator_,
cur_rows_[row_count]))) {
LOG_WARN("Failed to project storage row", K(ret));
} else {
++row_count;
LOG_TRACE("Get next rows from dml das iterator", KPC(sr), K(cur_rows_[row_count - 1]), K_(das_ctdef));
}
}
if (OB_SUCC(ret) || OB_LIKELY(OB_ITER_END == ret)) {
if (0 == row_count) {
ret = OB_ITER_END;
} else {
rows = cur_rows_;
ret = OB_SUCCESS;
}
}
}
}

1
src/sql/das/ob_das_dml_ctx_define.h
View File

@ -485,6 +485,7 @@ public:
private:
void set_ctdef(const ObDASDMLBaseCtDef *das_ctdef);
int get_next_domain_index_row(ObNewRow *&row);
int get_next_domain_index_rows(ObNewRow *&rows, int64_t &row_count);
private:
ObDASWriteBuffer &write_buffer_;
const ObDASDMLBaseCtDef *das_ctdef_;

37
src/sql/das/ob_das_domain_utils.cpp
View File

@ -544,6 +544,43 @@ int ObDomainDMLIterator::get_next_domain_row(ObNewRow *&row)
return ret;
}
int ObDomainDMLIterator::get_next_domain_rows(ObNewRow *&row, int64_t &row_count)
{
int ret = OB_SUCCESS;
const ObChunkDatumStore::StoredRow *sr = nullptr;
bool got_row = false;
if (OB_FAIL(THIS_WORKER.check_status())) {
LOG_WARN("worker interrupt", K(ret));
}
row_count = 0;
while (OB_SUCC(ret) && !got_row) {
if (row_idx_ >= rows_.count()) {
rows_.reuse();
row_idx_ = 0;
if (OB_UNLIKELY(!das_ctdef_->table_param_.get_data_table().is_domain_index())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected error, not domain index", K(ret), K(das_ctdef_->table_param_.get_data_table()));
} else if (FAILEDx(write_iter_.get_next_row(sr))) {
if (OB_ITER_END != ret) {
LOG_WARN("get next row from result iterator failed", K(ret));
}
} else if (OB_FAIL(generate_domain_rows(sr))) {
if (ret != OB_ITER_END) {
LOG_WARN("fail to generate domain index row", K(ret), KPC(sr));
}
}
}
if (OB_SUCC(ret) && row_idx_ < rows_.count()) {
row = &(rows_[row_idx_]);
row_count = rows_.count() - row_idx_;
row_idx_ = rows_.count();
got_row = true;
}
}
LOG_DEBUG("get next domain rows", K(ret), K(got_row), K(row_idx_), K(row_count), K(rows_), KPC(row), KPC(sr));
return ret;
}
int ObSpatialDMLIterator::generate_domain_rows(const ObChunkDatumStore::StoredRow *store_row)
{
int ret = OB_SUCCESS;

1
src/sql/das/ob_das_domain_utils.h
View File

@ -106,6 +106,7 @@ public:
void set_ctdef(const ObDASDMLBaseCtDef *das_ctdef, const IntFixedArray *row_projector);
void set_row_projector(const IntFixedArray *row_projector) { row_projector_ = row_projector; }
int get_next_domain_row(ObNewRow *&row);
int get_next_domain_rows(ObNewRow *&row, int64_t &row_count);
bool is_same_domain_type(const ObDASDMLBaseCtDef *das_ctdef) const;
TO_STRING_KV(K_(row_idx), K_(rows), KPC_(row_projector), KPC_(das_ctdef), K_(main_ctdef));

10
src/storage/ls/ob_ls_tablet_service.cpp
View File

@ -2691,7 +2691,7 @@ int ObLSTabletService::insert_rows(
void *ptr = nullptr;
ObStoreRow *tbl_rows = nullptr;
int64_t row_count = 0;
bool first_bulk = true;
int64_t row_buf_cnt = 0;
ObNewRow *rows = nullptr;
if (OB_FAIL(prepare_dml_running_ctx(&column_ids, nullptr, tablet_handle, run_ctx))) {
LOG_WARN("failed to prepare dml running ctx", K(ret));
@ -2717,12 +2717,16 @@ int ObLSTabletService::insert_rows(
} else if (1 == row_count) {
tbl_rows = &reserved_row;
tbl_rows[0].flag_.set_flag(ObDmlFlag::DF_INSERT);
} else if (first_bulk) {
first_bulk = false;
} else if (row_buf_cnt < row_count) {
if (nullptr != ptr) {
work_allocator.free(ptr);
ptr = nullptr;
}
if (OB_ISNULL(ptr = work_allocator.alloc(row_count * sizeof(ObStoreRow)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("fail to allocate memory", K(ret), K(row_count));
} else {
row_buf_cnt = row_count;
tbl_rows = new (ptr) ObStoreRow[row_count];
for (int64_t i = 0; i < row_count; i++) {
tbl_rows[i].flag_.set_flag(ObDmlFlag::DF_INSERT);