hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[FEAT MERGE] Optimize dml performance in multi-local index scenarios

Browse Source 
Co-authored-by: Handora <qcdsr970209@gmail.com>
Co-authored-by: Naynahs <cfzy002@126.com>
Co-authored-by: ZenoWang <wzybuaasoft@163.com>
This commit is contained in:
windye
2024-04-16 15:27:19 +00:00
committed by ob-robot
parent 5c28cef93c
commit 7aca4ef065
59 changed files with 696875 additions and 299635 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/sql/das/ob_das_dml_ctx_define.h
View File

@ -455,7 +455,7 @@ private:
class ObDASDMLIterator : public common::ObNewRowIterator
{
public:
static const int64_t DEFAULT_BATCH_SIZE = 1;
static const int64_t DEFAULT_BATCH_SIZE = 256;
public:
ObDASDMLIterator(const ObDASDMLBaseCtDef *das_ctdef,
ObDASWriteBuffer &write_buffer,

13
src/sql/das/ob_das_insert_op.cpp
View File

@ -157,10 +157,20 @@ int ObDASInsertOp::insert_row_with_fetch()
ObAccessService *as = MTL(ObAccessService *);
ObDMLBaseParam dml_param;
ObDASDMLIterator dml_iter(ins_ctdef_, insert_buffer_, op_alloc_);
storage::ObStoreCtxGuard store_ctx_guard;
if (ins_ctdef_->table_rowkey_types_.empty()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("table_rowkey_types is invalid", K(ret));
} else if (OB_FAIL(ObDMLService::init_dml_param(*ins_ctdef_, *ins_rtdef_, *snapshot_, write_branch_id_, op_alloc_, dml_param))) {
} else if (OB_FAIL(as->get_write_store_ctx_guard(ls_id_,
ins_rtdef_->timeout_ts_,
*trans_desc_,
*snapshot_,
write_branch_id_,
store_ctx_guard))) {
LOG_WARN("fail to get_write_store_ctx_guard", K(ret), K(ls_id_));
} else if (OB_FAIL(ObDMLService::init_dml_param(*ins_ctdef_, *ins_rtdef_,
*snapshot_, write_branch_id_, op_alloc_, store_ctx_guard, dml_param))) {
LOG_WARN("init dml param failed", K(ret), KPC_(ins_ctdef), KPC_(ins_rtdef));
} else if (OB_ISNULL(buf = op_alloc_.alloc(sizeof(ObDASConflictIterator)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
@ -226,6 +236,7 @@ int ObDASInsertOp::insert_row_with_fetch()
*snapshot_,
write_branch_id_,
op_alloc_,
store_ctx_guard,
dml_param))) {
LOG_WARN("init index dml param failed", K(ret), KPC(index_ins_ctdef), KPC(index_ins_rtdef));
}

12
src/sql/das/ob_das_lock_op.cpp
View File

@ -55,7 +55,17 @@ int ObDASLockOp::open_op()
ObDASDMLIterator dml_iter(lock_ctdef_, lock_buffer_, op_alloc_);
ObAccessService *as = MTL(ObAccessService *);
if (OB_FAIL(ObDMLService::init_dml_param(*lock_ctdef_, *lock_rtdef_, *snapshot_, write_branch_id_, op_alloc_, dml_param))) {
storage::ObStoreCtxGuard store_ctx_guard;
if (OB_FAIL(as->get_write_store_ctx_guard(ls_id_,
lock_rtdef_->timeout_ts_,
*trans_desc_,
*snapshot_,
write_branch_id_,
store_ctx_guard))) {
LOG_WARN("fail to get_write_access_tx_ctx_guard", K(ret), K(ls_id_));
} else if (OB_FAIL(ObDMLService::init_dml_param(
*lock_ctdef_, *lock_rtdef_, *snapshot_, write_branch_id_, op_alloc_, store_ctx_guard, dml_param))) {
LOG_WARN("init dml param failed", K(ret));
} else if (OB_FAIL(as->lock_rows(ls_id_,
tablet_id_,

2
src/sql/engine/dml/ob_dml_service.cpp
View File

@ -1184,6 +1184,7 @@ int ObDMLService::init_dml_param(const ObDASDMLBaseCtDef &base_ctdef,
transaction::ObTxReadSnapshot &snapshot,
const int16_t write_branch_id,
ObIAllocator &das_alloc,
storage::ObStoreCtxGuard &store_ctx_gurad,
storage::ObDMLBaseParam &dml_param)
{
int ret = OB_SUCCESS;
@ -1200,6 +1201,7 @@ int ObDMLService::init_dml_param(const ObDASDMLBaseCtDef &base_ctdef,
dml_param.dml_allocator_ = &das_alloc;
dml_param.snapshot_ = snapshot;
dml_param.branch_id_ = write_branch_id;
dml_param.store_ctx_guard_ = &store_ctx_gurad;
if (base_ctdef.is_batch_stmt_) {
dml_param.write_flag_.set_is_dml_batch_opt();
}

99
src/sql/engine/dml/ob_dml_service.h
View File

@ -15,6 +15,8 @@
#include "sql/engine/dml/ob_dml_ctx_define.h"
#include "sql/das/ob_das_context.h"
#include "ob_table_modify_op.h"
#include "storage/tx_storage/ob_access_service.h"
namespace oceanbase
{
namespace sql
@ -144,6 +146,7 @@ public:
transaction::ObTxReadSnapshot &snapshot,
const int16_t write_branch_id,
common::ObIAllocator &das_alloc,
storage::ObStoreCtxGuard &store_ctx_gurad,
storage::ObDMLBaseParam &dml_param);
static int init_das_dml_rtdef(ObDMLRtCtx &dml_rtctx,
const ObDASDMLBaseCtDef &das_ctdef,
@ -331,40 +334,54 @@ int ObDASIndexDMLAdaptor<N, DMLIterator>::write_tablet(DMLIterator &iter, int64_
if (OB_FAIL(write_tablet_with_ignore(iter, affected_rows))) {
LOG_WARN("write tablet with ignore failed", K(ret));
}
} else if (OB_FAIL(ObDMLService::init_dml_param(*ctdef_, *rtdef_, *snapshot_, write_branch_id_, *das_allocator_, dml_param_))) {
SQL_DAS_LOG(WARN, "init dml param failed", K(ret), K(ctdef_->table_id_), K(ctdef_->index_tid_));
} else if (OB_FAIL(write_rows(ls_id_, tablet_id_, *ctdef_, *rtdef_, iter, affected_rows))) {
SQL_DAS_LOG(WARN, "write rows failed", K(ret),
K(ls_id_), K(tablet_id_), K(ctdef_->table_id_), K(ctdef_->index_tid_));
} else if (related_ctdefs_ != nullptr && !related_ctdefs_->empty()) {
//write local index
for (int64_t i = 0; OB_SUCC(ret) && i < related_ctdefs_->count(); ++i) {
const CtDefType *related_ctdef = static_cast<const CtDefType*>(related_ctdefs_->at(i));
RtDefType *related_rtdef = static_cast<RtDefType*>(related_rtdefs_->at(i));
ObTabletID related_tablet_id = related_tablet_ids_->at(i);
int64_t index_affected_rows = 0;
SQL_DAS_LOG(DEBUG, "rewind iterator and write local index tablet",
K(ls_id_), K(related_tablet_id), K(related_ctdef->table_id_), K(related_ctdef->index_tid_));
if (OB_FAIL(iter.rewind(related_ctdef))) {
SQL_DAS_LOG(WARN, "rewind iterator failed", K(ret));
} else if (OB_FAIL(ObDMLService::init_dml_param(*related_ctdef, *related_rtdef, *snapshot_, write_branch_id_, *das_allocator_, dml_param_))) {
SQL_DAS_LOG(WARN, "init index dml param failed", K(ret),
K(related_ctdef->table_id_), K(related_ctdef->index_tid_));
} else if (OB_FAIL(write_rows(ls_id_,
related_tablet_id,
*related_ctdef,
*related_rtdef,
iter,
index_affected_rows))) {
SQL_DAS_LOG(WARN, "write local index rows failed", K(ret),
K(related_tablet_id), K(related_ctdef->table_id_), K(related_ctdef->index_tid_));
} else if (OB_FAIL(ObDMLService::check_local_index_affected_rows(affected_rows,
index_affected_rows,
*ctdef_,
*rtdef_,
*related_ctdef,
*related_rtdef))) {
SQL_DAS_LOG(WARN, "check local index affected rows failed", K(ret));
} else {
ObAccessService *as = MTL(ObAccessService *);
storage::ObStoreCtxGuard store_ctx_guard;
if (OB_FAIL(as->get_write_store_ctx_guard(ls_id_,
rtdef_->timeout_ts_,
*tx_desc_,
*snapshot_,
write_branch_id_,
store_ctx_guard))) {
LOG_WARN("fail to get_write_store_ctx_guard", K(ret), K(ls_id_));
} else if (OB_FAIL(ObDMLService::init_dml_param(
*ctdef_, *rtdef_, *snapshot_, write_branch_id_, *das_allocator_, store_ctx_guard, dml_param_))) {
SQL_DAS_LOG(WARN, "init dml param failed", K(ret), K(ctdef_->table_id_), K(ctdef_->index_tid_));
} else if (OB_FAIL(write_rows(ls_id_, tablet_id_, *ctdef_, *rtdef_, iter, affected_rows))) {
SQL_DAS_LOG(WARN, "write rows failed", K(ret),
K(ls_id_), K(tablet_id_), K(ctdef_->table_id_), K(ctdef_->index_tid_));
} else if (related_ctdefs_ != nullptr && !related_ctdefs_->empty()) {
//write local index
for (int64_t i = 0; OB_SUCC(ret) && i < related_ctdefs_->count(); ++i) {
const CtDefType *related_ctdef = static_cast<const CtDefType*>(related_ctdefs_->at(i));
RtDefType *related_rtdef = static_cast<RtDefType*>(related_rtdefs_->at(i));
ObTabletID related_tablet_id = related_tablet_ids_->at(i);
int64_t index_affected_rows = 0;
SQL_DAS_LOG(DEBUG, "rewind iterator and write local index tablet",
K(ls_id_), K(related_tablet_id), K(related_ctdef->table_id_), K(related_ctdef->index_tid_));
if (OB_FAIL(iter.rewind(related_ctdef))) {
SQL_DAS_LOG(WARN, "rewind iterator failed", K(ret));
} else if (OB_FAIL(ObDMLService::init_dml_param(*related_ctdef, *related_rtdef,
*snapshot_, write_branch_id_, *das_allocator_, store_ctx_guard, dml_param_))) {
SQL_DAS_LOG(WARN, "init index dml param failed", K(ret),
K(related_ctdef->table_id_), K(related_ctdef->index_tid_));
} else if (OB_FAIL(write_rows(ls_id_,
related_tablet_id,
*related_ctdef,
*related_rtdef,
iter,
index_affected_rows))) {
SQL_DAS_LOG(WARN, "write local index rows failed", K(ret),
K(related_tablet_id), K(related_ctdef->table_id_), K(related_ctdef->index_tid_));
} else if (OB_FAIL(ObDMLService::check_local_index_affected_rows(affected_rows,
index_affected_rows,
*ctdef_,
*rtdef_,
*related_ctdef,
*related_rtdef))) {
SQL_DAS_LOG(WARN, "check local index affected rows failed", K(ret));
}
}
}
}
@ -379,6 +396,7 @@ int ObDASIndexDMLAdaptor<N, DMLIterator>::write_tablet_with_ignore(DMLIterator &
affected_rows = 0;
const ObDASWriteBuffer::DmlRow *dml_row = nullptr;
ObDASWriteBuffer::Iterator write_iter;
ObAccessService *as = MTL(ObAccessService *);
const bool with_local_index = related_ctdefs_ != nullptr && !related_ctdefs_->empty();
if (OB_FAIL(iter.get_write_buffer().begin(write_iter))) {
LOG_WARN("begin write iterator failed", K(ret));
@ -405,7 +423,17 @@ int ObDASIndexDMLAdaptor<N, DMLIterator>::write_tablet_with_ignore(DMLIterator &
SQL_DAS_LOG(TRACE, "write table dml row with ignore", KPC(dml_row), K(ls_id_), K(tablet_id_),
K(ctdef_->table_id_), K(ctdef_->index_tid_));
DMLIterator single_row_iter(ctdef_, single_row_buffer, *das_allocator_);
if (OB_FAIL(ObDMLService::init_dml_param(*ctdef_, *rtdef_, *snapshot_, write_branch_id_, *das_allocator_, dml_param_))) {
storage::ObStoreCtxGuard store_ctx_guard;
if (OB_FAIL(as->get_write_store_ctx_guard(ls_id_,
rtdef_->timeout_ts_,
*tx_desc_,
*snapshot_,
write_branch_id_,
store_ctx_guard))) {
LOG_WARN("fail to get_write_store_ctx_guard", K(ret), K(ls_id_));
} else if (OB_FAIL(ObDMLService::init_dml_param(*ctdef_, *rtdef_, *snapshot_, write_branch_id_,
*das_allocator_, store_ctx_guard, dml_param_))) {
SQL_DAS_LOG(WARN, "init dml param failed", K(ret), KPC_(ctdef), KPC_(rtdef));
} else if (with_local_index && FALSE_IT(dml_param_.write_flag_.set_skip_flush_redo())) {
} else if (OB_FAIL(write_rows(ls_id_,
@ -432,6 +460,7 @@ int ObDASIndexDMLAdaptor<N, DMLIterator>::write_tablet_with_ignore(DMLIterator &
*snapshot_,
write_branch_id_,
*das_allocator_,
store_ctx_guard,
dml_param_))) {
SQL_DAS_LOG(WARN, "init index dml param failed", K(ret),
KPC(related_ctdef), KPC(related_rtdef));

20
src/sql/ob_sql_trans_control.cpp
View File

@ -730,31 +730,41 @@ int ObSqlTransControl::stmt_sanity_check_(ObSQLSessionInfo *session,
ObPhysicalPlanCtx *plan_ctx)
{
int ret = OB_SUCCESS;
auto current_consist_level = plan_ctx->get_consistency_level();
ObConsistencyLevel current_consist_level = plan_ctx->get_consistency_level();
CK (current_consist_level != ObConsistencyLevel::INVALID_CONSISTENCY);
bool is_plain_select = plan->is_plain_select();
// adjust stmt's consistency level
if (OB_SUCC(ret)) {
// Weak read statement with inner table should be converted to strong read.
// For example, schema refresh statement;
if (plan->is_contain_inner_table() ||
(!is_plain_select && current_consist_level != ObConsistencyLevel::STRONG)) {
(!plan->is_plain_select() && current_consist_level != ObConsistencyLevel::STRONG)) {
plan_ctx->set_consistency_level(ObConsistencyLevel::STRONG);
}
}
// check isolation with consistency type
if (OB_SUCC(ret) && session->is_in_transaction()) {
// check consistency type volatile
ObConsistencyLevel current_consist_level = plan_ctx->get_consistency_level();
if (current_consist_level == ObConsistencyLevel::WEAK) {
// read write transaction
if (!session->get_tx_desc()->is_clean()) {
plan_ctx->set_consistency_level(ObConsistencyLevel::STRONG);
}
}
// check isolation with consistency type
auto iso = session->get_tx_desc()->get_isolation_level();
auto cl = plan_ctx->get_consistency_level();
if (ObConsistencyLevel::WEAK == cl && (iso == ObTxIsolationLevel::SERIAL || iso == ObTxIsolationLevel::RR)) {
if (ObConsistencyLevel::WEAK == cl &&
(iso == ObTxIsolationLevel::SERIAL || iso == ObTxIsolationLevel::RR)) {
ret = OB_NOT_SUPPORTED;
TRANS_LOG(ERROR, "statement of weak consistency is not allowed under SERIALIZABLE isolation",
KR(ret), "trans_id", session->get_tx_id(), "consistency_level", cl);
LOG_USER_ERROR(OB_NOT_SUPPORTED, "weak consistency under SERIALIZABLE and REPEATABLE-READ isolation level");
}
}
return ret;
}