hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FEAT MERGE]

Browse Source 
Co-authored-by: Handora <qcdsr970209@gmail.com>
This commit is contained in:
yishenglanlingzui 2024-11-23 00:45:17 +00:00 committed by ob-robot
parent 6d3a8a8b76
commit 7887fcce92
128 changed files with 5244 additions and 2432 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
deps/oblib/src/lib/ob_name_id_def.h vendored
View File

@ -510,6 +510,8 @@ DEF_NAME(id, "id")
//dml event
DEF_NAME(sync_auto_value, "sync_auto_value")
DEF_NAME(start_execute_das_task, "start_execute_das_task")
DEF_NAME(end_execute_das_task, "end_execute_das_task")
DEF_NAME(value_start_next_row, "value_start_next_row")
DEF_NAME(value_end_next_row, "value_end_next_row")
@ -524,55 +526,32 @@ DEF_NAME(id, "id")
DEF_NAME(insert_open, "insert_open")
DEF_NAME(insert_close, "inser_close")
DEF_NAME(insert_start, "insert_start")
DEF_NAME(insert_end, "insert_end")
DEF_NAME(insert_start_next_row, "insert_start_next_row")
DEF_NAME(insert_start_calc_row, "insert_start_calc_row")
DEF_NAME(insert_end_next_row, "insert_end_next_row")
DEF_NAME(insert_iter_end, "insert_iter_end")
DEF_NAME(insertup_open, "insertup_open")
DEF_NAME(insert_up_load_all_row, "insert_up_load_all_row")
DEF_NAME(insert_up_try_insert, "insert_up_try_insert")
DEF_NAME(insert_up_start_lookup, "insert_up_start_lookup")
DEF_NAME(insert_up_start_shuff, "insert_up_start_shuff")
DEF_NAME(insert_up_final_shuff, "insert_up_final_shuff")
DEF_NAME(insert_up_inner_close, "insert_up_inner_close")
DEF_NAME(insert_up_start_load_row, "insert_up_start_load_row")
DEF_NAME(insertup_start_do, "insertup_start_do")
DEF_NAME(insertup_start_calc_insert_row, "insertup_start_calc_insert_row")
DEF_NAME(insertup_start_insert_row, "insertup_start_insert_row")
DEF_NAME(insertup_end_insert_row, "insertup_end_insert_row")
DEF_NAME(insertup_end, "insertup_end")
DEF_NAME(insertup_before_scan, "insertup_before_scan")
DEF_NAME(insertup_start_update_row, "insertup_start_update_row")
DEF_NAME(insertup_end_update_row, "insertup_end_update_row")
DEF_NAME(insertup_start_calc_update_row, "insertup_start_calc_update_row")
DEF_NAME(insertup_calc_new_row, "insertup_calc_new_row")
DEF_NAME(insertup_auto_increment, "insertup_auto_increment")
DEF_NAME(insertup_end_auto_increment, "insertup_end_auto_increment")
DEF_NAME(insert_up_try_insert, "insert_up_try_insert")
DEF_NAME(insertup_build_fetch_rowkey, "insertup_build_fetch_rowkey")
DEF_NAME(insertup_start_rollback, "insertup_start_rollback")
DEF_NAME(insertup_start_shuff, "insertup_start_shuff")
DEF_NAME(insertup_calc_auto_inc, "insertup_calc_auto_inc")
DEF_NAME(insertup_final_write, "insertup_final_write")
DEF_NAME(insertup_inner_close, "insertup_inner_close")
DEF_NAME(replace_open, "replace_open")
DEF_NAME(replace_load_all_row, "replace_load_all_row")
DEF_NAME(replace_start_load_row, "replace_start_load_row")
DEF_NAME(replace_try_insert, "replace_try_insert")
DEF_NAME(replace_start_lookup, "replace_start_lookup")
DEF_NAME(replace_build_fetch_rowkey, "replace_build_fetch_rowkey")
DEF_NAME(replace_start_rollback, "replace_start_rollback")
DEF_NAME(start_fetch_conflict_row, "start_fetch_conflict_row")
DEF_NAME(replace_start_shuff, "replace_start_shuff")
DEF_NAME(replace_final_shuff, "replace_final_shuff")
DEF_NAME(replace_final_write, "replace_final_write")
DEF_NAME(replace_inner_close, "replace_inner_close")
DEF_NAME(revert_insert_iter, "revert_insert_iter")
DEF_NAME(revert_insert_iter_end, "revert_insert_iter_end")
DEF_NAME(revert_scan_iter_end, "revert_scan_iter_end")
DEF_NAME(replace_end, "replace_end")
DEF_NAME(repalce_start_insert, "repalce_start_insert")
DEF_NAME(replace_end_insert, "replace_end_insert")
DEF_NAME(replace_start_delete, "replace_start_delete")
DEF_NAME(replace_end_delete, "replace_end_delete")
DEF_NAME(replace_start_insert, "replace_start_insert")
DEF_NAME(replace_start_scan_row, "replace_start_scan_row")
DEF_NAME(replace_start_table_scan, "replace_start_table_scan")
DEF_NAME(replace_end_table_scan, "replace_end_table_scan")
DEF_NAME(update_open, "update_open")
DEF_NAME(update_iter_end, "update_iter_end")
DEF_NAME(update_end, "update_end")

2
deps/oblib/src/rpc/ob_request.h vendored
View File

@ -48,7 +48,7 @@ class ObRequest: public common::ObLink
{
public:
friend class ObSqlRequestOperator;
enum Type { OB_RPC, OB_MYSQL, OB_TASK, OB_TS_TASK, OB_SQL_TASK, OB_SQL_SOCK_TASK };
enum Type { OB_RPC, OB_MYSQL, OB_TASK, OB_TS_TASK, OB_SQL_TASK, OB_SQL_SOCK_TASK, OB_DAS_PARALLEL_TASK };
enum TransportProto { TRANSPORT_PROTO_EASY = 0, TRANSPORT_PROTO_POC = 1 };
enum Stat {
OB_EASY_REQUEST_EZ_RECV = 0,

6
src/observer/mysql/obmp_query.cpp
View File

@ -835,8 +835,10 @@ OB_INLINE int ObMPQuery::do_process(ObSQLSessionInfo &session,
tmp_ret = OB_E(EventTable::EN_PRINT_QUERY_SQL) OB_SUCCESS;
if (OB_SUCCESS != tmp_ret) {
LOG_INFO("query info:", K(sql_),
"sess_id", result.get_session().get_sessid(),
"trans_id", result.get_session().get_tx_id());
"sess_id", result.get_session().get_sessid(),
"database_id", result.get_session().get_database_id(),
"database_name", result.get_session().get_database_name(),
"trans_id", audit_record.trans_id_);
}
//监控项统计结束

7
src/observer/ob_srv_xlator.cpp
View File

@ -68,6 +68,7 @@
#include "observer/table/ob_table_query_processor.h"
#include "observer/table/ob_table_query_and_mutate_processor.h"
#include "logservice/palf/log_rpc_processor.h"
#include "sql/das/ob_das_parallel_handler.h"
using namespace oceanbase::observer;
using namespace oceanbase::lib;
@ -390,7 +391,8 @@ ObReqProcessor *ObSrvXlator::get_processor(ObRequest &req)
}
} else if (ObRequest::OB_TASK == req.get_type() ||
ObRequest::OB_TS_TASK == req.get_type() ||
ObRequest::OB_SQL_TASK == req.get_type()) {
ObRequest::OB_SQL_TASK == req.get_type() ||
ObRequest::OB_DAS_PARALLEL_TASK == req.get_type()) {
processor = &static_cast<ObSrvTask&>(req).get_processor();
} else {
LOG_WARN("can't translate packet", "type", req.get_type());
@ -458,6 +460,9 @@ int ObSrvXlator::release(ObReqProcessor *processor)
} else if (ObRequest::OB_SQL_TASK == req_type) {
ObSqlTaskFactory::get_instance().free(static_cast<ObSqlTask *>(req));
req = NULL;
} else if (ObRequest::OB_DAS_PARALLEL_TASK == req_type) {
ObDASParallelTaskFactory::free(static_cast<ObDASParallelTask *>(req));
req = NULL;
} else {
worker_allocator_delete(processor);
processor = NULL;

3
src/observer/table/ob_table_insert_up_executor.cpp
View File

@ -105,7 +105,8 @@ int ObTableApiInsertUpExecutor::open()
ret = OB_ERR_UNEXPECTED;
LOG_WARN("table location is invalid", K(ret));
} else if (OB_FAIL(conflict_checker_.init_conflict_checker(insert_up_spec_.get_expr_frame_info(),
table_loc))) {
table_loc,
false))) {
LOG_WARN("fail to init conflict_checker", K(ret));
} else if (OB_FAIL(calc_local_tablet_loc(tablet_loc))) {
LOG_WARN("fail to calc tablet loc", K(ret));

3
src/observer/table/ob_table_replace_executor.cpp
View File

@ -96,7 +96,8 @@ int ObTableApiReplaceExecutor::open()
ret = OB_ERR_UNEXPECTED;
LOG_WARN("table location is invalid", K(ret));
} else if (OB_FAIL(conflict_checker_.init_conflict_checker(replace_spec_.get_expr_frame_info(),
table_loc))) {
table_loc,
false))) {
LOG_WARN("fail to init conflict_checker", K(ret));
} else if (OB_FAIL(calc_local_tablet_loc(tablet_loc))) {
LOG_WARN("fail to calc tablet location", K(ret));

2
src/observer/table/ttl/ob_table_ttl_executor.cpp
View File

@ -97,7 +97,7 @@ int ObTableApiTTLExecutor::open()
if (OB_ISNULL(table_loc)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("table location is invalid", K(ret));
} else if (OB_FAIL(conflict_checker_.init_conflict_checker(ttl_spec_.get_expr_frame_info(), table_loc))) {
} else if (OB_FAIL(conflict_checker_.init_conflict_checker(ttl_spec_.get_expr_frame_info(), table_loc, false))) {
LOG_WARN("fail to init conflict_checker", K(ret));
} else if (OB_FAIL(calc_local_tablet_loc(tablet_loc))) {
LOG_WARN("fail to calc tablet location", K(ret));

6
src/share/diagnosis/ob_sql_monitor_statname.h
View File

@ -37,7 +37,11 @@ SQL_MONITOR_STATNAME_DEF(JOIN_FILTER_BIT_SET, sql_monitor_statname::INT, "filter
// PDML
SQL_MONITOR_STATNAME_DEF(PDML_PARTITION_FLUSH_TIME, sql_monitor_statname::INT, "clock time cost write storage", "total time cost writing data to storage by pdml op")
SQL_MONITOR_STATNAME_DEF(PDML_PARTITION_FLUSH_COUNT, sql_monitor_statname::INT, "times write to storage", "total times writing data to storage by pdml op")
SQL_MONITOR_STATNAME_DEF(PDML_GET_ROW_COUNT_FROM_CHILD_OP, sql_monitor_statname::INT, "clock time cost write storage", "total time cost writing data to storage by pdml op")
SQL_MONITOR_STATNAME_DEF(PDML_WRITE_DAS_BUFF_ROW_COUNT, sql_monitor_statname::INT, "row_count write to das buff", "total row count writing data to das buff by pdml op")
SQL_MONITOR_STATNAME_DEF(PDML_SKIP_ROW_COUNT, sql_monitor_statname::INT, "the count of skip write", "total row count which is not needed to write to storage by pdml op")
SQL_MONITOR_STATNAME_DEF(PDML_STORAGE_RETURN_ROW_COUNT, sql_monitor_statname::INT, "row_count storage return", "total write row count storage return to pdml op")
// reshuffle
SQL_MONITOR_STATNAME_DEF(EXCHANGE_DROP_ROW_COUNT, sql_monitor_statname::INT, "drop row count", "total row dropped by exchange out op for unmatched partition")
// MONITORING DUMP

7
src/share/parameter/ob_parameter_seed.ipp
View File

@ -1683,7 +1683,12 @@ DEF_BOOL(_enable_new_sql_nio, OB_CLUSTER_PARAMETER, "true",
"specifies whether SQL serial network is turned on. Turned on to support mysql_send_long_data"
"The default value is FALSE. Value: TRUE: turned on FALSE: turned off",
ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::STATIC_EFFECTIVE));
DEF_BOOL(_enable_parallel_das_dml, OB_TENANT_PARAMETER, "False",
"By default, the das service is allowed to use multiple threads to submit das tasks",
ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
DEF_BOOL(_enable_insertup_replace_gts_opt, OB_TENANT_PARAMETER, "True",
"By default, insert/replace ... values statement can use gts optimization",
ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
// Add a config to enable use das if the sql statement has variable assignment
DEF_BOOL(_enable_var_assign_use_das, OB_TENANT_PARAMETER, "False",
"enable use das if the sql statement has variable assignment",

3
src/share/system_variable/ob_sys_var_class_type.h
View File

@ -498,6 +498,9 @@ enum ObSysVarClassType
SYS_VAR_SKIP_EXTERNAL_LOCKING = 10390,
SYS_VAR_SUPER_READ_ONLY = 10391,
SYS_VAR_PLSQL_OPTIMIZE_LEVEL = 10393,
SYS_VAR_LOW_PRIORITY_UPDATES = 10394,
SYS_VAR_MAX_ERROR_COUNT = 10395,
SYS_VAR_MAX_INSERT_DELAYED_THREADS = 10396,
SYS_VAR_FT_STOPWORD_FILE = 10397,
SYS_VAR_INNODB_FT_CACHE_SIZE = 10398,
SYS_VAR_INNODB_FT_SORT_PLL_DEGREE = 10399,

3
src/share/system_variable/ob_system_variable_alias.h
View File

@ -493,6 +493,9 @@ namespace share
static const char* const OB_SV_SKIP_EXTERNAL_LOCKING = "skip_external_locking";
static const char* const OB_SV_SUPER_READ_ONLY = "super_read_only";
static const char* const OB_SV_PLSQL_OPTIMIZE_LEVEL = "plsql_optimize_level";
static const char* const OB_SV_LOW_PRIORITY_UPDATES = "low_priority_updates";
static const char* const OB_SV_MAX_ERROR_COUNT = "max_error_count";
static const char* const OB_SV_MAX_INSERT_DELAYED_THREADS = "max_insert_delayed_threads";
static const char* const OB_SV_FT_STOPWORD_FILE = "ft_stopword_file";
static const char* const OB_SV_INNODB_FT_CACHE_SIZE = "innodb_ft_cache_size";
static const char* const OB_SV_INNODB_FT_SORT_PLL_DEGREE = "innodb_ft_sort_pll_degree";

72
src/share/system_variable/ob_system_variable_factory.cpp
View File

@ -821,6 +821,7 @@ const char *ObSysVarFactory::SYS_VAR_NAMES_SORTED_BY_NAME[] = {
"log_builtin_as_identified_by_password",
"log_row_value_options",
"long_query_time",
"low_priority_updates",
"lower_case_file_system",
"lower_case_table_names",
"master_info_repository",
@ -832,7 +833,9 @@ const char *ObSysVarFactory::SYS_VAR_NAMES_SORTED_BY_NAME[] = {
"max_connect_errors",
"max_connections",
"max_digest_length",
"max_error_count",
"max_execution_time",
"max_insert_delayed_threads",
"max_join_size",
"max_length_for_sort_data",
"max_prepared_stmt_count",
@ -1434,6 +1437,7 @@ const ObSysVarClassType ObSysVarFactory::SYS_VAR_IDS_SORTED_BY_NAME[] = {
SYS_VAR_LOG_BUILTIN_AS_IDENTIFIED_BY_PASSWORD,
SYS_VAR_LOG_ROW_VALUE_OPTIONS,
SYS_VAR_LONG_QUERY_TIME,
SYS_VAR_LOW_PRIORITY_UPDATES,
SYS_VAR_LOWER_CASE_FILE_SYSTEM,
SYS_VAR_LOWER_CASE_TABLE_NAMES,
SYS_VAR_MASTER_INFO_REPOSITORY,
@ -1445,7 +1449,9 @@ const ObSysVarClassType ObSysVarFactory::SYS_VAR_IDS_SORTED_BY_NAME[] = {
SYS_VAR_MAX_CONNECT_ERRORS,
SYS_VAR_MAX_CONNECTIONS,
SYS_VAR_MAX_DIGEST_LENGTH,
SYS_VAR_MAX_ERROR_COUNT,
SYS_VAR_MAX_EXECUTION_TIME,
SYS_VAR_MAX_INSERT_DELAYED_THREADS,
SYS_VAR_MAX_JOIN_SIZE,
SYS_VAR_MAX_LENGTH_FOR_SORT_DATA,
SYS_VAR_MAX_PREPARED_STMT_COUNT,
@ -2219,6 +2225,9 @@ const char *ObSysVarFactory::SYS_VAR_NAMES_SORTED_BY_ID[] = {
"skip_external_locking",
"super_read_only",
"plsql_optimize_level",
"low_priority_updates",
"max_error_count",
"max_insert_delayed_threads",
"ft_stopword_file",
"innodb_ft_cache_size",
"innodb_ft_sort_pll_degree",
@ -3033,6 +3042,9 @@ int ObSysVarFactory::create_all_sys_vars()
+ sizeof(ObSysVarSkipExternalLocking)
+ sizeof(ObSysVarSuperReadOnly)
+ sizeof(ObSysVarPlsqlOptimizeLevel)
+ sizeof(ObSysVarLowPriorityUpdates)
+ sizeof(ObSysVarMaxErrorCount)
+ sizeof(ObSysVarMaxInsertDelayedThreads)
+ sizeof(ObSysVarFtStopwordFile)
+ sizeof(ObSysVarInnodbFtCacheSize)
+ sizeof(ObSysVarInnodbFtSortPllDegree)
@ -7467,6 +7479,33 @@ int ObSysVarFactory::create_all_sys_vars()
ptr = (void *)((char *)ptr + sizeof(ObSysVarPlsqlOptimizeLevel));
}
}
if (OB_SUCC(ret)) {
if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarLowPriorityUpdates())) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("fail to new ObSysVarLowPriorityUpdates", K(ret));
} else {
store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast<int64_t>(SYS_VAR_LOW_PRIORITY_UPDATES))] = sys_var_ptr;
ptr = (void *)((char *)ptr + sizeof(ObSysVarLowPriorityUpdates));
}
}
if (OB_SUCC(ret)) {
if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxErrorCount())) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("fail to new ObSysVarMaxErrorCount", K(ret));
} else {
store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast<int64_t>(SYS_VAR_MAX_ERROR_COUNT))] = sys_var_ptr;
ptr = (void *)((char *)ptr + sizeof(ObSysVarMaxErrorCount));
}
}
if (OB_SUCC(ret)) {
if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxInsertDelayedThreads())) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("fail to new ObSysVarMaxInsertDelayedThreads", K(ret));
} else {
store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast<int64_t>(SYS_VAR_MAX_INSERT_DELAYED_THREADS))] = sys_var_ptr;
ptr = (void *)((char *)ptr + sizeof(ObSysVarMaxInsertDelayedThreads));
}
}
if (OB_SUCC(ret)) {
if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarFtStopwordFile())) {
ret = OB_ALLOCATE_MEMORY_FAILED;
@ -13921,6 +13960,39 @@ int ObSysVarFactory::create_sys_var(ObIAllocator &allocator_, ObSysVarClassType
}
break;
}
case SYS_VAR_LOW_PRIORITY_UPDATES: {
void *ptr = NULL;
if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarLowPriorityUpdates)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarLowPriorityUpdates)));
} else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarLowPriorityUpdates())) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("fail to new ObSysVarLowPriorityUpdates", K(ret));
}
break;
}
case SYS_VAR_MAX_ERROR_COUNT: {
void *ptr = NULL;
if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarMaxErrorCount)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarMaxErrorCount)));
} else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxErrorCount())) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("fail to new ObSysVarMaxErrorCount", K(ret));
}
break;
}
case SYS_VAR_MAX_INSERT_DELAYED_THREADS: {
void *ptr = NULL;
if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarMaxInsertDelayedThreads)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarMaxInsertDelayedThreads)));
} else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarMaxInsertDelayedThreads())) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("fail to new ObSysVarMaxInsertDelayedThreads", K(ret));
}
break;
}
case SYS_VAR_FT_STOPWORD_FILE: {
void *ptr = NULL;
if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarFtStopwordFile)))) {

289
src/share/system_variable/ob_system_variable_factory.h
View File

File diff suppressed because it is too large Load Diff

2549
src/share/system_variable/ob_system_variable_init.cpp
View File

File diff suppressed because it is too large Load Diff

9
src/share/system_variable/ob_system_variable_init.json
View File

@ -6924,8 +6924,7 @@
"publish_version": "424",
"info_cn": "",
"background_cn": "",
"ref_url": "",
"placeholder": true
"ref_url": ""
},
"max_error_count": {
"id": 10395,
@ -6940,8 +6939,7 @@
"publish_version": "424",
"info_cn": "",
"background_cn": "",
"ref_url": "",
"placeholder": true
"ref_url": ""
},
"max_insert_delayed_threads": {
"id": 10396,
@ -6956,8 +6954,7 @@
"publish_version": "424",
"info_cn": "",
"background_cn": "",
"ref_url": "",
"placeholder": true
"ref_url": ""
},
"ft_stopword_file": {
"id": 10397,

1
src/sql/CMakeLists.txt
View File

@ -67,6 +67,7 @@ ob_set_subtarget(ob_sql das
das/ob_das_attach_define.cpp
das/ob_group_scan_iter.cpp
das/ob_das_dml_vec_iter.cpp
das/ob_das_parallel_handler.cpp
das/iter/ob_das_iter.cpp
das/iter/ob_das_merge_iter.cpp
das/iter/ob_das_lookup_iter.cpp

190
src/sql/code_generator/ob_static_engine_cg.cpp
View File

@ -2952,6 +2952,9 @@ int ObStaticEngineCG::generate_merge_with_das(ObLogMerge &op,
spec.use_dist_das_ = op.is_multi_part_dml();
spec.gi_above_ = op.is_gi_above() && !spec.use_dist_das_;
spec.table_location_uncertain_ = op.is_table_location_uncertain();
spec.is_pdml_ = op.is_pdml();
spec.plan_->set_das_dop(op.get_das_dop());
spec.das_dop_ = op.get_das_dop();
spec.merge_ctdefs_.set_capacity(modified_index_ids.count());
// TODO those 2 line must fixed after remove old engine
@ -3070,6 +3073,7 @@ int ObStaticEngineCG::check_is_insert_overwrite_stmt(const ObLogPlan *plan, bool
int ObStaticEngineCG::generate_insert_with_das(ObLogInsert &op, ObTableInsertSpec &spec)
{
int ret = OB_SUCCESS;
bool is_plain_insert = false;
spec.check_fk_batch_ = true;
const ObLogPlan *log_plan = op.get_plan();
const ObIArray<IndexDMLInfo *> &index_dml_infos = op.get_index_dml_infos();
@ -3090,12 +3094,18 @@ int ObStaticEngineCG::generate_insert_with_das(ObLogInsert &op, ObTableInsertSpe
} else if (OB_FAIL(spec.ins_ctdefs_.at(0).allocate_array(phy_plan_->get_allocator(),
index_dml_infos.count()))) {
LOG_WARN("allocate insert ctdef array failed", K(ret), K(index_dml_infos.count()));
} else if (OB_FAIL(op.is_plain_insert(is_plain_insert))) {
LOG_WARN("fail to check is plain insert", K(ret));
} else {
spec.plan_->set_ignore(op.is_ignore());
spec.plan_->need_drive_dml_query_ = true;
spec.use_dist_das_ = op.is_multi_part_dml();
spec.gi_above_ = op.is_gi_above() && !spec.use_dist_das_;
spec.is_returning_ = op.is_returning();
spec.is_pdml_ = op.is_pdml();
spec.das_dop_ = op.get_das_dop();
spec.plan_->set_das_dop(op.get_das_dop());
spec.plan_->set_is_plain_insert(is_plain_insert);
}
}
for (int64_t i = 0; OB_SUCC(ret) && i < index_dml_infos.count(); ++i) {
@ -3177,6 +3187,9 @@ int ObStaticEngineCG::generate_delete_with_das(ObLogDelete &op, ObTableDeleteSpe
spec.use_dist_das_ = op.is_multi_part_dml();
spec.gi_above_ = op.is_gi_above() && !spec.use_dist_das_;
spec.is_returning_ = op.is_returning();
spec.is_pdml_ = op.is_pdml();
spec.plan_->set_das_dop(op.get_das_dop());
spec.das_dop_ = op.get_das_dop();
if (OB_FAIL(spec.del_ctdefs_.allocate_array(phy_plan_->get_allocator(),
delete_table_list.count()))) {
LOG_WARN("allocate delete ctdef array failed", K(ret));
@ -3246,16 +3259,65 @@ int ObStaticEngineCG::generate_spec(ObLogInsert &op, ObTableReplaceSpec &spec, c
{
int ret = OB_SUCCESS;
UNUSED(in_root_job);
bool can_do_gts_opt = false;
bool has_unique_index = false;
bool has_partition_index = false;
const ObIArray<IndexDMLInfo *> &insert_dml_infos = op.get_index_dml_infos();;
const ObIArray<IndexDMLInfo *> &del_dml_infos = op.get_replace_index_dml_infos();
const IndexDMLInfo *primary_dml_info = insert_dml_infos.at(0);
CK (NULL != primary_dml_info);
if (NULL == primary_dml_info) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null", K(ret));
} else if (OB_FAIL(op.is_insertup_or_replace_values(can_do_gts_opt))) {
LOG_WARN("fail to check is plain insert", K(ret));
} else if (!can_do_gts_opt) {
// do nothing
LOG_TRACE("can't do insert_up gts opt", K(op.get_insert_up_index_dml_infos()));
} else if (OB_FAIL(check_has_global_partiton_index(op.get_plan(),
primary_dml_info->ref_table_id_,
has_partition_index))) {
LOG_WARN("check has global partition index failed", K(ret), K(primary_dml_info->ref_table_id_));
} else if (has_partition_index) {
LOG_TRACE("has partition index, can't support gts opt");
} else {
spec.plan_->set_insertup_can_do_gts_opt(can_do_gts_opt);
if (OB_FAIL(check_has_global_unique_index(op.get_plan(), primary_dml_info->ref_table_id_, has_unique_index))) {
LOG_WARN("check has global unique index", K(ret), K(primary_dml_info->ref_table_id_));
} else {
spec.has_global_unique_index_ = has_unique_index;
}
}
// for replace_into multi_query batch_dml_optimization
if (OB_SUCC(ret) && op.get_stmt_id_expr() != nullptr) {
if (OB_FAIL(generate_rt_expr(*op.get_stmt_id_expr(), spec.ab_stmt_id_))) {
LOG_WARN("generate ab stmt id expr failed", K(ret));
}
}
if (OB_SUCC(ret)) {
ObSEArray<ObRawExpr *, 32> all_need_save_exprs;
ObRawExpr *stmt_id_expr = const_cast<ObRawExpr *>(op.get_stmt_id_expr());
if (OB_FAIL(append(all_need_save_exprs, primary_dml_info->column_convert_exprs_))) {
LOG_WARN("fail to append expr to array", K(ret));
} else if (stmt_id_expr != nullptr && OB_FAIL(all_need_save_exprs.push_back(stmt_id_expr))) {
LOG_WARN("fail to append stmt_id_expr to array", K(ret));
} else if (OB_FAIL(generate_rt_exprs(all_need_save_exprs, spec.all_saved_exprs_))) {
LOG_WARN("fail to generate all_saved_expr", K(ret), K(all_need_save_exprs));
} else {
LOG_TRACE("print all_need_save_exprs", K(all_need_save_exprs));
}
}
if (OB_SUCC(ret)) {
spec.is_ignore_ = op.is_ignore();
spec.plan_->need_drive_dml_query_ = true;
spec.use_dist_das_ = op.is_multi_part_dml();
spec.gi_above_ = op.is_gi_above() && !spec.use_dist_das_;
phy_plan_->set_ignore(op.is_ignore());
spec.is_pdml_ = op.is_pdml();
spec.plan_->set_das_dop(op.get_das_dop());
spec.das_dop_ = op.get_das_dop();
// todo @wenber.wb delete it after support trigger
ObLogPlan *log_plan = op.get_plan();
@ -3354,6 +3416,9 @@ int ObStaticEngineCG::generate_update_with_das(ObLogUpdate &op, ObTableUpdateSpe
spec.use_dist_das_ = op.is_multi_part_dml();
spec.gi_above_ = op.is_gi_above() && !spec.use_dist_das_;
spec.is_returning_ = op.is_returning();
spec.is_pdml_ = op.is_pdml();
spec.plan_->set_das_dop(op.get_das_dop());
spec.das_dop_ = op.get_das_dop();
if (OB_FAIL(spec.upd_ctdefs_.allocate_array(phy_plan_->get_allocator(),
table_list.count()))) {
LOG_WARN("allocate update ctdef array failed", K(ret), K(table_list));
@ -3461,6 +3526,13 @@ int ObStaticEngineCG::generate_spec(ObLogInsert &op, ObTableInsertUpSpec &spec,
ret = OB_ERR_UNEXPECTED;
LOG_WARN("update dml info is empty", K(ret));
}
// for insertup multi_query batch_dml_optimization
if (OB_SUCC(ret) && op.get_stmt_id_expr() != nullptr) {
if (OB_FAIL(generate_rt_expr(*op.get_stmt_id_expr(), spec.ab_stmt_id_))) {
LOG_WARN("generate ab stmt id expr failed", K(ret));
}
}
if (OB_SUCC(ret)) {
const ObIArray<IndexDMLInfo *> &insert_dml_infos = op.get_index_dml_infos();
const ObIArray<IndexDMLInfo *> &upd_dml_infos = op.get_insert_up_index_dml_infos();
@ -3470,6 +3542,9 @@ int ObStaticEngineCG::generate_spec(ObLogInsert &op, ObTableInsertUpSpec &spec,
spec.use_dist_das_ = op.is_multi_part_dml();
spec.gi_above_ = op.is_gi_above() && !spec.use_dist_das_;
phy_plan_->set_ignore(op.is_ignore());
spec.is_pdml_ = op.is_pdml();
spec.plan_->set_das_dop(op.get_das_dop());
spec.das_dop_ = op.get_das_dop();
ObLogPlan *log_plan = op.get_plan();
ObSchemaGetterGuard *schema_guard = NULL;
@ -3528,10 +3603,13 @@ int ObStaticEngineCG::generate_spec(ObLogInsert &op, ObTableInsertUpSpec &spec,
}
}
if (OB_SUCC(ret)) {
ObRawExpr *stmt_id_expr = const_cast<ObRawExpr *>(op.get_stmt_id_expr());
if (OB_FAIL(append(all_need_save_exprs, ins_pri_dml_info->column_convert_exprs_))) {
LOG_WARN("fail to append expr to array", K(ret));
} else if (OB_FAIL(append(all_need_save_exprs, contain_exprs))) {
LOG_WARN("fail to append expr to array", K(ret));
} else if (stmt_id_expr != nullptr && OB_FAIL(all_need_save_exprs.push_back(stmt_id_expr))) {
LOG_WARN("fail to append stmt_id_expr to array", K(ret));
} else if (OB_FAIL(generate_rt_exprs(all_need_save_exprs, spec.all_saved_exprs_))) {
LOG_WARN("fail to generate all_saved_expr", K(ret), K(all_need_save_exprs));
} else {
@ -3541,6 +3619,38 @@ int ObStaticEngineCG::generate_spec(ObLogInsert &op, ObTableInsertUpSpec &spec,
}
}
if (OB_SUCC(ret)) {
bool can_do_gts_opt = false;
bool has_unique_index = false;
bool update_part_key = false;
bool has_partition_index = false;
const IndexDMLInfo *ins_pri_dml_info = op.get_index_dml_infos().at(0);
if (OB_FAIL(op.is_insertup_or_replace_values(can_do_gts_opt))) {
LOG_WARN("fail to check is plain insert", K(ret));
} else if (!can_do_gts_opt) {
// do nothing
LOG_TRACE("can't do insert_up gts opt", K(op.get_insert_up_index_dml_infos()));
} else if (OB_FAIL(check_has_update_part_key(op.get_insert_up_index_dml_infos(), update_part_key))) {
LOG_WARN("fail to check has update part key", K(ret), K(op.get_insert_up_index_dml_infos()));
} else if (update_part_key) {
// global index orprimary table update part key
LOG_TRACE("global index or primary table update part_key", K(op.get_insert_up_index_dml_infos()));
} else if (OB_FAIL(check_has_global_partiton_index(op.get_plan(),
ins_pri_dml_info->ref_table_id_,
has_partition_index))) {
LOG_WARN("check has global partition index failed", K(ins_pri_dml_info->ref_table_id_));
} else if (has_partition_index) {
LOG_TRACE("has partition index, can't support gts opt");
} else {
spec.plan_->set_insertup_can_do_gts_opt(can_do_gts_opt);
if (OB_FAIL(check_has_global_unique_index(op.get_plan(), ins_pri_dml_info->ref_table_id_, has_unique_index))) {
LOG_WARN("check has global unique index", K(ret), K(ins_pri_dml_info->ref_table_id_));
} else {
spec.has_global_unique_index_ = has_unique_index;
}
}
}
if (OB_SUCC(ret)) {
const ObInsertUpCtDef *insert_up_ctdef = spec.insert_up_ctdefs_.at(0);
if (OB_ISNULL(insert_up_ctdef)) {
@ -3567,6 +3677,19 @@ int ObStaticEngineCG::generate_spec(ObLogInsert &op, ObTableInsertUpSpec &spec,
return ret;
}
int ObStaticEngineCG::check_has_update_part_key(const ObIArray<IndexDMLInfo *> &index_dml_infos, bool &update_part_key)
{
int ret = OB_SUCCESS;
update_part_key = false;
for (int64_t i = 0; OB_SUCC(ret) && !update_part_key && i < index_dml_infos.count(); i++) {
const IndexDMLInfo *upd_pri_dml_info = index_dml_infos.at(i);
if (upd_pri_dml_info->is_update_part_key_) {
update_part_key = true;
}
}
return ret;
}
int ObStaticEngineCG::generate_spec(ObLogTopk &op,
ObTopKSpec &spec,
const bool in_root_job)
@ -10157,6 +10280,71 @@ int ObStaticEngineCG::check_only_one_unique_key(const ObLogPlan& log_plan,
return ret;
}
int ObStaticEngineCG::check_has_global_partiton_index(ObLogPlan *log_plan,
const uint64_t table_id,
bool &has_global_partition_index)
{
int ret = OB_SUCCESS;
has_global_partition_index = false;
uint64_t index_tid[OB_MAX_INDEX_PER_TABLE];
int64_t index_cnt = OB_MAX_INDEX_PER_TABLE;
ObSchemaGetterGuard *schema_guard = nullptr;
if (OB_ISNULL(log_plan)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null ptr", K(ret));
} else if (OB_ISNULL(schema_guard = log_plan->get_optimizer_context().get_schema_guard())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null ptr", K(ret));
} else if (OB_FAIL(schema_guard->get_can_write_index_array(MTL_ID(), table_id, index_tid, index_cnt, true))) {
LOG_WARN("failed to get can read index array", K(ret));
}
for (int64_t i = 0; OB_SUCC(ret) && !has_global_partition_index && i < index_cnt; ++i) {
const ObTableSchema* index_schema = NULL;
if (OB_FAIL(schema_guard->get_table_schema(MTL_ID(), index_tid[i], index_schema))) {
LOG_WARN("failed to get table schema", K(ret));
} else if (OB_ISNULL(index_schema)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to get table schema", K(index_tid[i]), K(ret));
} else if (index_schema->is_partitioned_table()) {
has_global_partition_index = true;
index_schema->is_partitioned_table();
LOG_TRACE("is partition global index", K(index_schema->get_table_name_str()),
K(index_schema->get_table_id()));
}
}
return ret;
}
int ObStaticEngineCG::check_has_global_unique_index(ObLogPlan *log_plan, const uint64_t table_id, bool &has_unique_index)
{
int ret = OB_SUCCESS;
has_unique_index = false;
uint64_t index_tid[OB_MAX_INDEX_PER_TABLE];
int64_t index_cnt = OB_MAX_INDEX_PER_TABLE;
ObSchemaGetterGuard *schema_guard = nullptr;
if (OB_ISNULL(log_plan)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null ptr", K(ret));
} else if (OB_ISNULL(schema_guard = log_plan->get_optimizer_context().get_schema_guard())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null ptr", K(ret));
} else if (OB_FAIL(schema_guard->get_can_write_index_array(MTL_ID(), table_id, index_tid, index_cnt, true))) {
LOG_WARN("failed to get can read index array", K(ret));
}
for (int64_t i = 0; OB_SUCC(ret) && !has_unique_index && i < index_cnt; ++i) {
const ObTableSchema* index_schema = NULL;
if (OB_FAIL(schema_guard->get_table_schema(MTL_ID(), index_tid[i], index_schema))) {
LOG_WARN("failed to get table schema", K(ret));
} else if (OB_ISNULL(index_schema)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to get table schema", K(index_tid[i]), K(ret));
} else if (index_schema->is_global_unique_index_table()) {
has_unique_index = true;
}
}
return ret;
}
bool ObStaticEngineCG::has_cycle_reference(DASTableIdList &parent_tables, const uint64_t table_id)
{
bool ret = false;

4
src/sql/code_generator/ob_static_engine_cg.h
View File

@ -480,6 +480,7 @@ private:
// direct load
int generate_spec(ObLogInsert &op, ObTableDirectInsertSpec &spec, const bool in_root_job);
private:
int check_has_update_part_key(const ObIArray<IndexDMLInfo *> &update_index_dml_infos, bool &update_part_key);
int disable_use_rich_format(const ObLogicalOperator &op, ObOpSpec &spec);
int add_update_set(ObSubPlanFilterSpec &spec);
int generate_basic_transmit_spec(
@ -582,7 +583,8 @@ private:
int add_output_datum_check_flag(ObOpSpec &spec);
int generate_calc_part_id_expr(const ObRawExpr &src, const ObDASTableLocMeta *loc_meta, ObExpr *&dst);
int check_only_one_unique_key(const ObLogPlan &log_plan, const ObTableSchema* table_schema, bool& only_one_unique_key);
int check_has_global_unique_index(ObLogPlan *log_plan, const uint64_t table_id, bool &has_unique_index);
int check_has_global_partiton_index(ObLogPlan *log_plan, const uint64_t table_id, bool &has_global_partition_index);
bool is_simple_aggr_expr(const ObItemType &expr_type,
const bool enable_rich_format) { return T_FUN_COUNT == expr_type
|| T_FUN_SUM == expr_type

11
src/sql/das/ob_das_context.h
View File

@ -75,6 +75,8 @@ public:
skip_scan_group_id_(-1),
group_rescan_cnt_(-1),
same_tablet_addr_(),
real_das_dop_(0),
use_gts_opt_(false),
flags_(0)
{
is_fk_cascading_ = 0;
@ -159,13 +161,18 @@ public:
int find_group_param_by_param_idx(int64_t param_idx,
bool &exist, uint64_t &array_idx);
int64_t get_real_das_dop() { return real_das_dop_; }
void set_real_das_dop(int64_t v) { real_das_dop_ = v; }
void set_use_gts_opt(bool v) { use_gts_opt_ = v; }
bool get_use_gts_opt() { return use_gts_opt_; }
TO_STRING_KV(K_(table_locs),
K_(external_table_locs),
K_(is_fk_cascading),
K_(snapshot),
K_(savepoint),
K_(write_branch_id));
K_(write_branch_id),
K_(real_das_dop));
private:
int check_same_server(const ObDASTabletLoc *tablet_loc);
private:
@ -193,6 +200,8 @@ private:
int64_t skip_scan_group_id_; //only allowed to be modified by GroupParamBackupGuard
int64_t group_rescan_cnt_; //only allowed to be modified by GroupParamBackupGuard
ObAddr same_tablet_addr_;
int64_t real_das_dop_;
bool use_gts_opt_; // without get gts
public:
union {
uint64_t flags_;

2
src/sql/das/ob_das_define.h
View File

@ -58,6 +58,8 @@ const int64_t OB_DAS_MAX_PACKET_SIZE = 2 * 1024 * 1024l - 8 * 1024;
*/
const int64_t OB_DAS_MAX_TOTAL_PACKET_SIZE = 1 * OB_DAS_MAX_PACKET_SIZE;
const int64_t OB_DAS_MAX_META_TENANT_PACKET_SIZE = 1 * 1024 * 1024l - 8 * 1024;
// offset of das parallel thread_pool group_id
static const int32_t OB_DAS_PARALLEL_POOL_MARK = 1 << 30;
} // namespace das
enum class ObDasTaskStatus: uint8_t

39
src/sql/das/ob_das_delete_op.cpp
View File

@ -122,12 +122,31 @@ int ObDASDeleteOp::open_op()
LOG_WARN("delete row to partition storage failed", K(ret));
}
} else {
del_rtdef_->affected_rows_ += affected_rows;
affected_rows_ = affected_rows;
}
return ret;
}
int ObDASDeleteOp::record_task_result_to_rtdef()
{
int ret = OB_SUCCESS;
del_rtdef_->affected_rows_ += affected_rows_;
return ret;
}
int ObDASDeleteOp::assign_task_result(ObIDASTaskOp *other)
{
int ret = OB_SUCCESS;
if (other->get_type() != get_type()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected task type", K(ret), KPC(other));
} else {
ObDASDeleteOp *del_op = static_cast<ObDASDeleteOp *>(other);
affected_rows_ = del_op->get_affected_rows();
}
return ret;
}
int ObDASDeleteOp::release_op()
{
int ret = OB_SUCCESS;
@ -143,7 +162,7 @@ int ObDASDeleteOp::decode_task_result(ObIDASTaskResult *task_result)
#endif
if (OB_SUCC(ret)) {
ObDASDeleteResult *del_result = static_cast<ObDASDeleteResult*>(task_result);
del_rtdef_->affected_rows_ += del_result->get_affected_rows();
affected_rows_ = del_result->get_affected_rows();
}
return ret;
}
@ -176,29 +195,25 @@ int ObDASDeleteOp::init_task_info(uint32_t row_extend_size)
int ObDASDeleteOp::swizzling_remote_task(ObDASRemoteInfo *remote_info)
{
int ret = OB_SUCCESS;
if (remote_info != nullptr) {
if (OB_FAIL(ObIDASTaskOp::swizzling_remote_task(remote_info))) {
LOG_WARN("fail to swizzling remote task", K(ret));
} else if (remote_info != nullptr) {
//DAS delete is executed remotely
trans_desc_ = remote_info->trans_desc_;
snapshot_ = &remote_info->snapshot_;
}
return ret;
}
int ObDASDeleteOp::write_row(const ExprFixedArray &row,
ObEvalCtx &eval_ctx,
ObChunkDatumStore::StoredRow *&stored_row,
bool &buffer_full)
ObChunkDatumStore::StoredRow *&stored_row)
{
int ret = OB_SUCCESS;
bool added = false;
buffer_full = false;
if (!write_buffer_.is_inited()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("buffer not inited", K(ret));
} else if (OB_FAIL(write_buffer_.try_add_row(row, &eval_ctx, das::OB_DAS_MAX_PACKET_SIZE, stored_row, added, true))) {
LOG_WARN("try add row to datum store failed", K(ret), K(row), K(write_buffer_));
} else if (!added) {
buffer_full = true;
} else if (OB_FAIL(write_buffer_.add_row(row, &eval_ctx, stored_row, true))) {
LOG_WARN("add row to datum store failed", K(ret), K(row), K(write_buffer_));
}
return ret;
}

6
src/sql/das/ob_das_delete_op.h
View File

@ -29,6 +29,8 @@ public:
virtual int open_op() override;
virtual int release_op() override;
virtual int record_task_result_to_rtdef() override;
virtual int assign_task_result(ObIDASTaskOp *other) override;
virtual int decode_task_result(ObIDASTaskResult *task_result) override;
virtual int fill_task_result(ObIDASTaskResult &task_result, bool &has_more, int64_t &memory_limit) override;
virtual int init_task_info(uint32_t row_extend_size) override;
@ -37,8 +39,7 @@ public:
virtual ObDASBaseRtDef *get_rtdef() override { return del_rtdef_; }
int write_row(const ExprFixedArray &row,
ObEvalCtx &eval_ctx,
ObChunkDatumStore::StoredRow *&stored_row,
bool &buffer_full);
ObChunkDatumStore::StoredRow *&stored_row);
int64_t get_row_cnt() const { return write_buffer_.get_row_cnt(); }
void set_das_ctdef(const ObDASDelCtDef *del_ctdef) { del_ctdef_ = del_ctdef; }
void set_das_rtdef(ObDASDelRtDef *del_rtdef) { del_rtdef_ = del_rtdef; }
@ -46,6 +47,7 @@ public:
{
return write_buffer_.dump_data(*del_ctdef_);
}
int64_t get_affected_rows() { return affected_rows_; }
INHERIT_TO_STRING_KV("parent", ObIDASTaskOp,
KPC_(del_ctdef),

77
src/sql/das/ob_das_dml_ctx_define.cpp
View File

@ -446,6 +446,71 @@ int ObDASWriteBuffer::init_dml_shadow_row(int64_t column_cnt, bool strip_lob_loc
return ret;
}
int ObDASWriteBuffer::add_row(const common::ObIArray<ObExpr*> &exprs,
ObEvalCtx *ctx,
DmlRow *&stored_row,
bool strip_lob_locator)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(dml_shadow_row_)) {
if (OB_FAIL(init_dml_shadow_row(exprs.count(), strip_lob_locator))) {
LOG_WARN("init dml shadow row failed", K(ret));
}
} else {
dml_shadow_row_->reuse();
}
if (OB_SUCC(ret)) {
if (OB_FAIL(dml_shadow_row_->shadow_copy(exprs, *ctx))) {
LOG_WARN("shadow copy dml row failed", K(ret));
} else if (OB_FAIL(add_row(*dml_shadow_row_, &stored_row))) {
LOG_WARN("try add row with shadow row failed", KK(ret));
} else if (OB_ISNULL(stored_row)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("stored row is null", K(ret));
} else {
LOG_DEBUG("succ add dml_row", KPC(stored_row));
}
}
return ret;
}
// 以后das的add_row接口只许成功，不许失败
int ObDASWriteBuffer::add_row(const DmlShadowRow &sr, DmlRow **stored_row)
{
int ret = OB_SUCCESS;
bool row_added = false;
const DmlRow *lsr = sr.get_store_row();
int64_t simulate_len = - EVENT_CALL(EventTable::EN_DAS_WRITE_ROW_LIST_LEN);
int64_t final_row_list_len = simulate_len > 0 ? simulate_len : DAS_WRITE_ROW_LIST_LEN;
if (OB_LIKELY(buffer_list_.size_ < final_row_list_len)) {
//link write row buffer to dlist
//avoid to create ObChunkDatumStore,
//because it is too heavy for small dml queries
ret = add_row_to_dlist(sr, row_added, stored_row);
} else {
ret = add_row_to_store(sr, stored_row);
}
return ret;
}
int ObDASWriteBuffer::add_row_to_store(const ObChunkDatumStore::ShadowStoredRow &sr,
ObChunkDatumStore::StoredRow **stored_sr)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(datum_store_)) {
if (OB_FAIL(create_datum_store())) {
LOG_WARN("create datum store failed", K(ret));
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(datum_store_->add_row(sr, stored_sr))) {
LOG_WARN("try add row to store failed", K(ret), K_(buffer_list_.mem_used));
}
}
return ret;
}
int ObDASWriteBuffer::try_add_row(const ObIArray<ObExpr*> &exprs,
ObEvalCtx *ctx,
const int64_t memory_limit,
@ -472,7 +537,7 @@ int ObDASWriteBuffer::try_add_row(const ObIArray<ObExpr*> &exprs,
ret = OB_ERR_UNEXPECTED;
LOG_WARN("stored row is null", K(ret));
} else {
LOG_DEBUG("add dml_row pay_load here", KPC(stored_row));
LOG_DEBUG("succ add dml_row", KPC(stored_row));
}
}
@ -489,7 +554,13 @@ int ObDASWriteBuffer::try_add_row(const DmlShadowRow &sr,
int64_t row_size = lsr->row_size_;
int64_t simulate_len = - EVENT_CALL(EventTable::EN_DAS_WRITE_ROW_LIST_LEN);
int64_t final_row_list_len = simulate_len > 0 ? simulate_len : DAS_WRITE_ROW_LIST_LEN;
if (OB_UNLIKELY(row_size + get_mem_used() > memory_limit && get_mem_used() > 0)) {
int64_t final_mem_limit = memory_limit;
int64_t simulate_mem_limit = - EVENT_CALL(EventTable::EN_DAS_SIMULATE_DAS_TASK_SIZE);
if (simulate_mem_limit != 0 && final_mem_limit > simulate_mem_limit) {
LOG_TRACE("simulate_mem_limit", K(simulate_mem_limit));
final_mem_limit = simulate_mem_limit;
}
if (OB_UNLIKELY(row_size + get_mem_used() > final_mem_limit && get_mem_used() > 0)) {
//if the size of the first row exceeds memory_limit,
//writing is also allowed,
//ensuring that there is at least one row of data
@ -500,7 +571,7 @@ int ObDASWriteBuffer::try_add_row(const DmlShadowRow &sr,
//because it is too heavy for small dml queries
ret = add_row_to_dlist(sr, row_added, stored_row);
} else {
ret = add_row_to_store(sr, memory_limit, row_added, stored_row);
ret = add_row_to_store(sr, final_mem_limit, row_added, stored_row);
}
return ret;
}

8
src/sql/das/ob_das_dml_ctx_define.h
View File

@ -407,6 +407,12 @@ public:
return buffer_list_.header_.next_ != nullptr ? buffer_list_.header_.next_->cnt_ : 0;
}
inline uint64_t get_tenant_id() const { return mem_attr_.tenant_id_; }
int add_row(const common::ObIArray<ObExpr*> &exprs,
ObEvalCtx *ctx,
DmlRow *&stored_row,
bool strip_lob_locator);
int add_row(const DmlShadowRow &sr, DmlRow **stored_row = nullptr);
int try_add_row(const common::ObIArray<ObExpr*> &exprs,
ObEvalCtx *ctx,
const int64_t memory_limit,
@ -440,6 +446,8 @@ private:
const int64_t memory_limit,
bool &row_added,
ObChunkDatumStore::StoredRow **stored_sr);
int add_row_to_store(const ObChunkDatumStore::ShadowStoredRow &sr,
ObChunkDatumStore::StoredRow **stored_sr);
static DmlRow *get_next_dml_row(DmlRow *cur_row);
int serialize_buffer_list(char *buf, const int64_t buf_len, int64_t &pos) const;
int64_t get_buffer_list_serialize_size() const;

5
src/sql/das/ob_das_factory.cpp
View File

@ -290,17 +290,18 @@ int ObDASTaskFactory::create_das_async_cb(
int ret = OB_SUCCESS;
void *buffer = nullptr;
ObDasAsyncRpcCallBackContext *context = nullptr;
DASRefCountContext &ref_count_ctx = das_ref.get_das_ref_count_ctx();
if (OB_ISNULL(buffer = allocator_.alloc(sizeof(ObDasAsyncRpcCallBackContext)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to allocate das async cb context memory", K(ret), K(sizeof(ObDasAsyncRpcCallBackContext)));
} else if (FALSE_IT(context = new (buffer) ObDasAsyncRpcCallBackContext(das_ref, task_ops, timeout_ts))) {
} else if (FALSE_IT(context = new (buffer) ObDasAsyncRpcCallBackContext(ref_count_ctx, task_ops, timeout_ts))) {
} else if (OB_FAIL(context->init(attr))) {
LOG_WARN("fail to init das async cb context", K(ret));
} else if (OB_ISNULL(buffer = allocator_.alloc(sizeof(ObRpcDasAsyncAccessCallBack)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to allocate das async cb memory", K(ret), K(sizeof(ObRpcDasAsyncAccessCallBack)));
} else {
async_cb = new (buffer) ObRpcDasAsyncAccessCallBack(context);
async_cb = new (buffer) ObRpcDasAsyncAccessCallBack(context, &das_ref.get_das_factory());
STORE_DAS_OBJ(das_async_cb_store_, async_cb, ObRpcDasAsyncAccessCallBack);
}
return ret;

98
src/sql/das/ob_das_insert_op.cpp
View File

@ -109,7 +109,7 @@ int ObDASInsertOp::open_op()
{
int ret = OB_SUCCESS;
if (ins_rtdef_->need_fetch_conflict_ && OB_FAIL(insert_row_with_fetch())) {
LOG_WARN("fail to do insert with conflict fetch", K(ret));
LOG_WARN("fail to do insert with conflict fetch", K(ret), K(das_gts_opt_info_));
} else if (!ins_rtdef_->need_fetch_conflict_ && OB_FAIL(insert_rows())) {
LOG_WARN("fail to do insert", K(ret));
}
@ -139,7 +139,6 @@ int ObDASInsertOp::insert_rows()
LOG_WARN("insert rows to access service failed", K(ret));
}
} else {
ins_rtdef_->affected_rows_ += affected_rows;
affected_rows_ = affected_rows;
}
return ret;
@ -152,14 +151,15 @@ int ObDASInsertOp::insert_index_with_fetch(ObDMLBaseParam &dml_param,
ObDASInsRtDef *ins_rtdef,
storage::ObStoreCtxGuard &store_ctx_guard,
const UIntFixedArray *duplicated_column_ids,
common::ObTabletID tablet_id)
common::ObTabletID tablet_id,
transaction::ObTxReadSnapshot *snapshot)
{
int ret = OB_SUCCESS;
blocksstable::ObDatumRow *insert_row = NULL;
int64_t affected_rows = 0;
if (OB_FAIL(ObDMLService::init_dml_param(*ins_ctdef,
*ins_rtdef,
*snapshot_,
*snapshot,
write_branch_id_,
op_alloc_,
store_ctx_guard,
@ -195,7 +195,6 @@ int ObDASInsertOp::insert_index_with_fetch(ObDMLBaseParam &dml_param,
LOG_WARN("fail to push duplicated_row iter", K(ret));
} else {
LOG_DEBUG("insert one row and conflicted", KPC(insert_row));
ins_rtdef_->is_duplicated_ = true;
is_duplicated_ = true;
}
}
@ -221,14 +220,36 @@ int ObDASInsertOp::insert_row_with_fetch()
ObDMLBaseParam dml_param;
ObDASDMLIterator dml_iter(ins_ctdef_, insert_buffer_, op_alloc_);
storage::ObStoreCtxGuard store_ctx_guard;
transaction::ObTxReadSnapshot *snapshot = snapshot_;
if (das_gts_opt_info_.get_specify_snapshot()) {
transaction::ObTransService *txs = nullptr;
if (das_gts_opt_info_.isolation_level_ != transaction::ObTxIsolationLevel::RC) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected isolation_level", K(ret), K(das_gts_opt_info_));
} else if (OB_ISNULL(txs = MTL_WITH_CHECK_TENANT(transaction::ObTransService*, MTL_ID()))) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("get_tx_service", K(ret), K(MTL_ID()));
} else if (OB_FAIL(txs->get_ls_read_snapshot(*trans_desc_,
das_gts_opt_info_.isolation_level_,
ls_id_,
THIS_WORKER.get_timeout_ts(),
*das_gts_opt_info_.get_response_snapshot()))) {
LOG_WARN("fail to get ls read_snapshot", K(ret), K(ls_id_), K(THIS_WORKER.get_timeout_ts()));
} else {
snapshot = das_gts_opt_info_.get_response_snapshot();
LOG_TRACE("succ get ls snaoshot", K(ls_id_), K(tablet_id_), KPC(snapshot));
}
}
if (ins_ctdef_->table_rowkey_types_.empty()) {
if (OB_FAIL(ret)) {
// do nothing
} else if (ins_ctdef_->table_rowkey_types_.empty()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("table_rowkey_types is invalid", K(ret));
} else if (OB_FAIL(as->get_write_store_ctx_guard(ls_id_,
ins_rtdef_->timeout_ts_,
*trans_desc_,
*snapshot_,
*snapshot,
write_branch_id_,
store_ctx_guard))) {
LOG_WARN("fail to get_write_store_ctx_guard", K(ret), K(ls_id_));
@ -252,7 +273,8 @@ int ObDASInsertOp::insert_row_with_fetch()
ins_rtdef_,
store_ctx_guard,
&ins_ctdef_->table_rowkey_cids_,
tablet_id_))) {
tablet_id_,
snapshot))) {
LOG_WARN("fail to insert primary table", K(ret));
}
@ -275,7 +297,8 @@ int ObDASInsertOp::insert_row_with_fetch()
index_ins_rtdef,
store_ctx_guard,
&ins_ctdef_->table_rowkey_cids_,
index_tablet_id))) {
index_tablet_id,
snapshot))) {
LOG_WARN("fail to insert local unique index", K(ret), K(index_ins_ctdef->table_param_.get_data_table()));
}
}
@ -309,7 +332,8 @@ int ObDASInsertOp::insert_row_with_fetch()
index_ins_rtdef,
store_ctx_guard,
&(index_ins_ctdef->column_ids_),
index_tablet_id))) {
index_tablet_id,
snapshot))) {
// For non-unique local index,
// We check for duplications on all columns because the partition key is not stored in storage level
LOG_WARN("fail to insert non_unique index", K(ret), K(index_ins_ctdef->table_param_.get_data_table()));
@ -365,6 +389,27 @@ int ObDASInsertOp::release_op()
return ret;
}
int ObDASInsertOp::record_task_result_to_rtdef()
{
int ret = OB_SUCCESS;
ins_rtdef_->affected_rows_ += affected_rows_;
ins_rtdef_->is_duplicated_ |= is_duplicated_;
return ret;
}
int ObDASInsertOp::assign_task_result(ObIDASTaskOp *other)
{
int ret = OB_SUCCESS;
if (other->get_type() != get_type()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected task type", K(ret), KPC(other));
} else {
ObDASInsertOp *ins_op = static_cast<ObDASInsertOp *>(other);
affected_rows_ = ins_op->get_affected_rows();
is_duplicated_ = ins_op->get_is_duplicated();
}
return ret;
}
int ObDASInsertOp::decode_task_result(ObIDASTaskResult *task_result)
{
int ret = OB_SUCCESS;
@ -380,13 +425,18 @@ int ObDASInsertOp::decode_task_result(ObIDASTaskResult *task_result)
LOG_WARN("init insert result iterator failed", K(ret));
} else {
result_ = insert_result;
ins_rtdef_->affected_rows_ += insert_result->get_affected_rows();
ins_rtdef_->is_duplicated_ |= insert_result->is_duplicated();
affected_rows_ = insert_result->get_affected_rows();
is_duplicated_ = insert_result->is_duplicated();
if (das_gts_opt_info_.get_specify_snapshot()) {
if (OB_FAIL(das_gts_opt_info_.get_response_snapshot()->assign(insert_result->get_response_snapshot()))) {
LOG_WARN("fail to assign snapshot", K(ret));
}
}
}
} else {
result_ = insert_result;
ins_rtdef_->affected_rows_ += insert_result->get_affected_rows();
ins_rtdef_->is_duplicated_ |= insert_result->is_duplicated();
affected_rows_ = insert_result->get_affected_rows();
is_duplicated_ = insert_result->is_duplicated();
}
}
return ret;
@ -410,6 +460,11 @@ int ObDASInsertOp::fill_task_result(ObIDASTaskResult &task_result, bool &has_mor
ins_result.set_is_duplicated(is_duplicated_);
has_more = false;
memory_limit -= ins_result.get_result_buffer().get_mem_used();
if (das_gts_opt_info_.get_specify_snapshot()) {
if (OB_FAIL(ins_result.get_response_snapshot().assign(*das_gts_opt_info_.get_response_snapshot()))) {
LOG_WARN("fail to assign snapshot", K(ret));
}
}
}
} else {
ins_result.set_affected_rows(affected_rows_);
@ -433,29 +488,26 @@ int ObDASInsertOp::init_task_info(uint32_t row_extend_size)
int ObDASInsertOp::swizzling_remote_task(ObDASRemoteInfo *remote_info)
{
int ret = OB_SUCCESS;
if (remote_info != nullptr) {
if (OB_FAIL(ObIDASTaskOp::swizzling_remote_task(remote_info))) {
LOG_WARN("fail to swizzling remote task", K(ret));
} else if (remote_info != nullptr) {
//DAS insert is executed remotely
trans_desc_ = remote_info->trans_desc_;
snapshot_ = &remote_info->snapshot_;
}
return ret;
}
int ObDASInsertOp::write_row(const ExprFixedArray &row,
ObEvalCtx &eval_ctx,
ObChunkDatumStore::StoredRow *&stored_row,
bool &buffer_full)
ObChunkDatumStore::StoredRow *&stored_row)
{
int ret = OB_SUCCESS;
bool added = false;
buffer_full = false;
if (!insert_buffer_.is_inited()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("buffer not inited", K(ret));
} else if (OB_FAIL(insert_buffer_.try_add_row(row, &eval_ctx, das::OB_DAS_MAX_PACKET_SIZE, stored_row, added, true))) {
LOG_WARN("try add row to insert buffer failed", K(ret), K(row), K(insert_buffer_));
} else if (!added) {
buffer_full = true;
} else if (OB_FAIL(insert_buffer_.add_row(row, &eval_ctx, stored_row, true))) {
LOG_WARN("add row to insert buffer failed", K(ret), K(row), K(insert_buffer_));
}
return ret;
}

15
src/sql/das/ob_das_insert_op.h
View File

@ -59,6 +59,8 @@ public:
virtual int open_op() override;
virtual int release_op() override;
virtual int record_task_result_to_rtdef() override;
virtual int assign_task_result(ObIDASTaskOp *other) override;
virtual int decode_task_result(ObIDASTaskResult *task_result) override;
virtual int fill_task_result(ObIDASTaskResult &task_result, bool &has_more, int64_t &memory_limit) override;
virtual int init_task_info(uint32_t row_extend_size) override;
@ -67,8 +69,7 @@ public:
virtual ObDASBaseRtDef *get_rtdef() override { return ins_rtdef_; }
int write_row(const ExprFixedArray &row,
ObEvalCtx &eval_ctx,
ObChunkDatumStore::StoredRow *&stored_row,
bool &buffer_full);
ObChunkDatumStore::StoredRow *&stored_row);
int64_t get_row_cnt() const { return insert_buffer_.get_row_cnt(); }
void set_das_ctdef(const ObDASInsCtDef *ins_ctdef) { ins_ctdef_ = ins_ctdef; }
void set_das_rtdef(ObDASInsRtDef *ins_rtdef) { ins_rtdef_ = ins_rtdef; }
@ -80,6 +81,9 @@ public:
blocksstable::ObDatumRowIterator *get_duplicated_result()
{ return result_; }
int64_t get_affected_rows() { return affected_rows_; }
bool get_is_duplicated() { return is_duplicated_; }
INHERIT_TO_STRING_KV("parent", ObIDASTaskOp,
KPC_(ins_ctdef),
KPC_(ins_rtdef),
@ -98,7 +102,8 @@ private:
ObDASInsRtDef *ins_rtdef,
storage::ObStoreCtxGuard &store_ctx_guard,
const UIntFixedArray *duplicated_column_ids,
common::ObTabletID tablet_id);
common::ObTabletID tablet_id,
transaction::ObTxReadSnapshot *snapshot);
private:
const ObDASInsCtDef *ins_ctdef_;
@ -130,7 +135,9 @@ public:
transaction::ObTxReadSnapshot &get_response_snapshot() { return response_snapshot_; }
INHERIT_TO_STRING_KV("ObIDASTaskResult", ObIDASTaskResult,
K_(affected_rows));
K_(affected_rows),
K_(is_duplicated),
K_(response_snapshot));
private:
int64_t affected_rows_;
ObDASWriteBuffer result_buffer_;

45
src/sql/das/ob_das_lock_op.cpp
View File

@ -79,7 +79,6 @@ int ObDASLockOp::open_op()
LOG_WARN("lock row to partition storage failed", K(ret));
}
} else {
lock_rtdef_->affected_rows_ += affected_rows;
affected_rows_ = affected_rows;
}
return ret;
@ -91,6 +90,26 @@ int ObDASLockOp::release_op()
return ret;
}
int ObDASLockOp::record_task_result_to_rtdef()
{
int ret = OB_SUCCESS;
lock_rtdef_->affected_rows_ += affected_rows_;
return ret;
}
int ObDASLockOp::assign_task_result(ObIDASTaskOp *other)
{
int ret = OB_SUCCESS;
if (other->get_type() != get_type()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected task type", K(ret), KPC(other));
} else {
ObDASLockOp *lock_op = static_cast<ObDASLockOp *>(other);
affected_rows_ = lock_op->get_affected_rows();
}
return ret;
}
int ObDASLockOp::decode_task_result(ObIDASTaskResult *task_result)
{
int ret = OB_SUCCESS;
@ -100,7 +119,7 @@ int ObDASLockOp::decode_task_result(ObIDASTaskResult *task_result)
#endif
if (OB_SUCC(ret)) {
ObDASLockResult *lock_result = static_cast<ObDASLockResult*>(task_result);
lock_rtdef_->affected_rows_ += lock_result->get_affected_rows();
affected_rows_ = lock_result->get_affected_rows();
}
return ret;
}
@ -136,29 +155,25 @@ int ObDASLockOp::init_task_info(uint32_t row_extend_size)
int ObDASLockOp::swizzling_remote_task(ObDASRemoteInfo *remote_info)
{
int ret = OB_SUCCESS;
if (remote_info != nullptr) {
//DAS lock is executed remotely
trans_desc_ = remote_info->trans_desc_;
snapshot_ = &remote_info->snapshot_;
}
if (OB_FAIL(ObIDASTaskOp::swizzling_remote_task(remote_info))) {
LOG_WARN("fail to swizzling remote task", K(ret));
} else if (remote_info != nullptr) {
//DAS lock is executed remotely
trans_desc_ = remote_info->trans_desc_;
}
return ret;
}
int ObDASLockOp::write_row(const ExprFixedArray &row,
ObEvalCtx &eval_ctx,
ObChunkDatumStore::StoredRow *&stored_row,
bool &buffer_full)
ObChunkDatumStore::StoredRow *&stored_row)
{
int ret = OB_SUCCESS;
bool added = false;
buffer_full = false;
if (!lock_buffer_.is_inited()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("buffer not inited", K(ret));
} else if (OB_FAIL(lock_buffer_.try_add_row(row, &eval_ctx, das::OB_DAS_MAX_PACKET_SIZE, stored_row, added, true))) {
LOG_WARN("try add row to lock buffer failed", K(ret), K(row), K(lock_buffer_));
} else if (!added) {
buffer_full = true;
} else if (OB_FAIL(lock_buffer_.add_row(row, &eval_ctx, stored_row, true))) {
LOG_WARN("add row to lock buffer failed", K(ret), K(row), K(lock_buffer_));
}
return ret;
}

6
src/sql/das/ob_das_lock_op.h
View File

@ -29,6 +29,8 @@ public:
virtual int open_op() override;
virtual int release_op() override;
virtual int assign_task_result(ObIDASTaskOp *other) override;
virtual int record_task_result_to_rtdef() override;
virtual int decode_task_result(ObIDASTaskResult *task_result) override;
virtual int fill_task_result(ObIDASTaskResult &task_result, bool &has_more, int64_t &memory_limit) override;
virtual int init_task_info(uint32_t row_extend_size) override;
@ -37,8 +39,7 @@ public:
virtual ObDASBaseRtDef *get_rtdef() override { return lock_rtdef_; }
int write_row(const ExprFixedArray &row,
ObEvalCtx &eval_ctx,
ObChunkDatumStore::StoredRow *&stored_row,
bool &buffer_full);
ObChunkDatumStore::StoredRow *&stored_row);
int64_t get_row_cnt() const { return lock_buffer_.get_row_cnt(); }
void set_das_ctdef(const ObDASLockCtDef *del_ctdef) { lock_ctdef_ = del_ctdef; }
void set_das_rtdef(ObDASLockRtDef *del_rtdef) { lock_rtdef_ = del_rtdef; }
@ -46,6 +47,7 @@ public:
{
return lock_buffer_.dump_data(*lock_ctdef_);
}
int64_t get_affected_rows() { return affected_rows_; }
INHERIT_TO_STRING_KV("parent", ObIDASTaskOp,
KPC_(lock_ctdef),

237
src/sql/das/ob_das_parallel_handler.cpp Normal file
View File

@ -0,0 +1,237 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL_DAS
#include "sql/das/ob_das_parallel_handler.h"
#include "sql/das/ob_data_access_service.h"
#include "share/resource_manager/ob_cgroup_ctrl.h"
#include "lib/profile/ob_trace_id.h"
using namespace oceanbase;
using namespace oceanbase::common;
using namespace oceanbase::sql;
using namespace oceanbase::lib;
using namespace oceanbase::share;
int64_t ObDASParallelTaskFactory::alloc_count_;
int64_t ObDASParallelTaskFactory::free_count_;
void __attribute__((weak)) request_finish_callback();
int ObDASParallelHandler::init(observer::ObSrvTask *task)
{
int ret = OB_SUCCESS;
if (NULL == task) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KPC(task));
} else {
task_ = task;
}
return ret;
}
int ObDASParallelHandler::deep_copy_all_das_tasks(ObDASTaskFactory &das_factory,
ObIAllocator &alloc,
ObIArray<ObIDASTaskOp*> &src_task_list,
ObIArray<ObIDASTaskOp*> &new_task_list,
ObDASRemoteInfo &remote_info,
ObDasAggregatedTask &das_task_wrapper)
{
int ret = OB_SUCCESS;
if (OB_FAIL(MTL(ObDataAccessService *)->collect_das_task_info(src_task_list, remote_info))) {
LOG_WARN("fail to collect das task info", K(ret));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < src_task_list.count(); i++) {
ObIDASTaskOp *das_op = nullptr;
if (OB_FAIL(deep_copy_das_task(das_factory, src_task_list.at(i), das_op, alloc))) {
LOG_WARN("fail to deep copy das_op",K(ret), K(src_task_list.at(i)));
} else if (OB_ISNULL(das_op)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null ptr", K(ret));
} else if (OB_FAIL(new_task_list.push_back(das_op))) {
LOG_WARN("fail to push back task list", K(ret));
} else if (OB_FAIL(das_task_wrapper.push_back_task(das_op))) {
LOG_WARN("fail to push back das_op", K(ret));
}
}
}
return ret;
}
int ObDASParallelHandler::deep_copy_das_task(ObDASTaskFactory &das_factory,
ObIDASTaskOp *src_op,
ObIDASTaskOp *&dst_op,
ObIAllocator &alloc)
{
int ret = OB_SUCCESS;
ObIDASTaskOp *das_op = nullptr;
if (OB_ISNULL(src_op->get_agg_task()) || OB_ISNULL(src_op->get_cur_agg_list())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null agg_task", K(ret), K(src_op->get_agg_task()), K(src_op->get_cur_agg_list()));
} else if (OB_FAIL(das_factory.create_das_task_op(src_op->get_type(), das_op))) {
LOG_WARN("fail to create das_op", K(ret));
} else if (OB_FAIL(das_op->init_task_info(ObDASWriteBuffer::DAS_ROW_DEFAULT_EXTEND_SIZE))) {
LOG_WARN("fail to init das_op info", K(ret));
} else {
int64_t ser_pos = 0;
int64_t des_pos = 0;
void *ser_ptr = NULL;
das_op->trans_desc_ = src_op->trans_desc_;
das_op->snapshot_ = src_op->snapshot_;
int64_t ser_arg_len = src_op->get_serialize_size();
if (OB_ISNULL(ser_ptr = alloc.alloc(ser_arg_len))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail alloc memory", K(ser_arg_len), KP(ser_ptr), K(ret));
} else if (OB_FAIL(src_op->serialize(static_cast<char *>(ser_ptr), ser_arg_len, ser_pos))) {
LOG_WARN("fail serialzie init task arg", KP(ser_ptr), K(ser_arg_len), K(ser_pos), K(ret));
} else if (OB_FAIL(das_op->deserialize(static_cast<const char *>(ser_ptr), ser_pos, des_pos))) {
LOG_WARN("fail des task arg", KP(ser_ptr), K(ser_pos), K(des_pos), K(ret));
} else if (ser_pos != des_pos) {
ret = OB_DESERIALIZE_ERROR;
LOG_WARN("data_len and pos mismatch", K(ser_arg_len), K(ser_pos), K(des_pos), K(ret));
} else {
das_op->set_tablet_loc(src_op->get_tablet_loc());
dst_op = das_op;
}
}
return ret;
}
int ObDASParallelHandler::record_status_and_op_result(ObIDASTaskOp *src_op, ObIDASTaskOp *dst_op)
{
int ret = OB_SUCCESS;
// record all affected_row and other info
if (dst_op->get_task_status() == ObDasTaskStatus::UNSTART) {
src_op->set_task_status(ObDasTaskStatus::UNSTART);
} else if (dst_op->get_task_status() == ObDasTaskStatus::FINISHED) {
src_op->set_task_status(ObDasTaskStatus::FINISHED);
src_op->errcode_ = OB_SUCCESS;
if (OB_FAIL(src_op->state_advance())) {
LOG_WARN("failed to advance das task state.",K(ret));
} else if (OB_FAIL(src_op->assign_task_result(dst_op))) {
LOG_WARN("failed to assign das task result.",K(ret));
}
} else if (dst_op->get_task_status() == ObDasTaskStatus::FAILED) {
src_op->set_task_status(ObDasTaskStatus::FAILED);
src_op->errcode_ = dst_op->errcode_;
if (OB_FAIL(src_op->state_advance())) {
LOG_WARN("failed to advance das task state.",K(ret));
}
}
return ret;
}
int ObDASParallelHandler::run()
{
int ret = OB_SUCCESS;
// execute all das_tasks
ObDASParallelTask *task = static_cast<ObDASParallelTask *>(task_);
common::ObSEArray<ObIDASTaskOp*, 4> new_task_list;
common::ObSEArray<ObIDASTaskOp*, 4> src_task_list;
lib::MemoryContext mem_context = nullptr;
common::ObCurTraceId::set(task->get_trace_id());
CREATE_WITH_TEMP_ENTITY(RESOURCE_OWNER, MTL_ID()) {
int interrupted_code = task->get_das_ref_count_ctx().get_interrupted_err_code();
if (interrupted_code != OB_SUCCESS) {
task->get_agg_task()->set_save_ret(interrupted_code);
LOG_WARN("this task is interrupted,ret_code is", K(interrupted_code));
} else if (OB_FAIL(ROOT_CONTEXT->CREATE_CONTEXT(mem_context,
lib::ContextParam().set_mem_attr(MTL_ID(), "DASParallelTask")))) {
LOG_WARN("create memory entity failed", K(ret));
} else {
WITH_CONTEXT(mem_context) {
ObDASRemoteInfo remote_info;
ObArenaAllocator tmp_alloc;
ObDASTaskFactory das_factory(mem_context->get_arena_allocator());
ObDasAggregatedTask das_task_wrapper(tmp_alloc);
ObDASRemoteInfo::get_remote_info() = &remote_info;
if (OB_FAIL(task->get_agg_task()->get_aggregated_tasks(src_task_list))) {
LOG_WARN("fail to get all das tasks", K(ret), KPC(task));
} else if (OB_FAIL(deep_copy_all_das_tasks(das_factory,
mem_context->get_arena_allocator(),
src_task_list,
new_task_list,
remote_info,
das_task_wrapper))) {
LOG_WARN("fail to deep copy all das tasks", K(ret));
} else if (OB_FAIL(MTL(ObDataAccessService *)->parallel_execute_das_task(new_task_list))) {
LOG_WARN("fail to parallel execute das task", K(ret), KPC(task));
} else {
// do nothing
}
// close new_task_list and copy all task execute result
int last_ret = OB_SUCCESS;
for (int64_t i = 0; i < new_task_list.count(); i++) {
int tmp_ret = OB_SUCCESS;
if (OB_SUCCESS != (tmp_ret = record_status_and_op_result(src_task_list.at(i), new_task_list.at(i)))) {
LOG_WARN("fail to record task status and result", K(tmp_ret));
}
last_ret = OB_SUCCESS == last_ret ? tmp_ret : last_ret;
if (OB_SUCCESS != (tmp_ret = new_task_list.at(i)->end_das_task())) {
LOG_WARN("end das task failed", K(ret), K(tmp_ret), KPC(new_task_list.at(i)));
}
last_ret = OB_SUCCESS == last_ret ? tmp_ret : last_ret;
}
ret = OB_SUCCESS == ret ? last_ret : ret;
} // end for mem_context
}
}
if (nullptr != mem_context) {
DESTROY_CONTEXT(mem_context);
mem_context = NULL;
}
task->get_agg_task()->set_save_ret(ret);
if (ret != OB_SUCCESS) {
task->get_das_ref_count_ctx().interrupt_other_workers(ret);
}
// whether success or failure,we must dec the reference_count
task->get_das_ref_count_ctx().inc_concurrency_limit_with_signal();
request_finish_callback();
// cover the error code
ret = OB_SUCCESS;
return ret;
}
int ObDASParallelTask::init(ObDasAggregatedTask *agg_task, int32_t group_id)
{
int ret = OB_SUCCESS;
if (NULL == agg_task) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("task is null, unexpected error", K(ret));
} else if (OB_FAIL(handler_.init(this))) {
LOG_WARN("init handler failed", K(ret));
} else {
set_group_id(group_id);
agg_task_ = agg_task;
trace_id_.set(*ObCurTraceId::get_trace_id());
set_type(ObRequest::OB_DAS_PARALLEL_TASK);
}
return ret;
}
ObDASParallelTask *ObDASParallelTaskFactory::alloc(DASRefCountContext &ref_count_ctx)
{
ObDASParallelTask *task = NULL;
if (NULL != (task = op_alloc_args(ObDASParallelTask, ref_count_ctx))) {
(void)ATOMIC_FAA(&alloc_count_, 1);
alloc_count_++;
if (REACH_TIME_INTERVAL(3 * 1000 * 1000)) {
LOG_INFO("ts response task statistics", K_(alloc_count), K_(free_count));
}
}
return task;
}
void ObDASParallelTaskFactory::free(ObDASParallelTask *task)
{
if (NULL != task) {
op_reclaim_free(task);
task = NULL;
(void)ATOMIC_FAA(&free_count_, 1);
}
}

86
src/sql/das/ob_das_parallel_handler.h Normal file
View File

@ -0,0 +1,86 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OBDEV_SRC_SQL_DAS_OB_DAS_PARALLEL_HANDLER_H_
#define OBDEV_SRC_SQL_DAS_OB_DAS_PARALLEL_HANDLER_H_
#include "sql/engine/dml/ob_dml_ctx_define.h"
namespace oceanbase
{
namespace sql
{
class ObDASParallelHandler : public rpc::frame::ObReqProcessor
{
public:
ObDASParallelHandler()
: task_(nullptr)
{ }
~ObDASParallelHandler() {}
int init(observer::ObSrvTask *task);
void reset()
{
task_ = nullptr;
}
protected:
int run();
int deep_copy_all_das_tasks(ObDASTaskFactory &das_factory,
ObIAllocator &alloc,
ObIArray<ObIDASTaskOp*> &src_task_list,
ObIArray<ObIDASTaskOp*> &new_task_list,
ObDASRemoteInfo &remote_info,
ObDasAggregatedTask &das_task_wrapper);
int deep_copy_das_task(ObDASTaskFactory &das_factory,
ObIDASTaskOp *src_op,
ObIDASTaskOp *&dst_op,
ObIAllocator &alloc);
int record_status_and_op_result(ObIDASTaskOp *src_op, ObIDASTaskOp *dst_op);
private:
DISALLOW_COPY_AND_ASSIGN(ObDASParallelHandler);
observer::ObSrvTask *task_;
}; // end of class ObDASParallelHandler
class ObDASParallelTask : public observer::ObSrvTask
{
public:
ObDASParallelTask(DASRefCountContext &das_ref_count_ctx)
: agg_task_(nullptr),
das_ref_count_ctx_(das_ref_count_ctx),
trace_id_(),
handler_()
{}
~ObDASParallelTask() {}
int init(ObDasAggregatedTask *agg_task, int32_t group_id);
const ObCurTraceId::TraceId &get_trace_id() const { return trace_id_; }
void reset()
{
agg_task_ = nullptr;
handler_.reset();
}
ObDasAggregatedTask *get_agg_task() const { return agg_task_; }
rpc::frame::ObReqProcessor &get_processor() { return handler_; }
DASRefCountContext &get_das_ref_count_ctx() { return das_ref_count_ctx_; }
TO_STRING_KV(KP(this), KPC(agg_task_));
private:
ObDasAggregatedTask *agg_task_;
DASRefCountContext &das_ref_count_ctx_;
ObCurTraceId::TraceId trace_id_;
ObDASParallelHandler handler_;
};
class ObDASParallelTaskFactory
{
public:
static ObDASParallelTask *alloc(DASRefCountContext &ref_count_ctx);
static void free(ObDASParallelTask *task);
private:
static int64_t alloc_count_;
static int64_t free_count_;
};
} // namespace sql
} // namespace oceanbase
#endif /* OBDEV_SRC_SQL_DAS_OB_DAS_PARALLEL_HANDLER_H_ */

754
src/sql/das/ob_das_ref.cpp
View File

@ -29,6 +29,76 @@ using namespace common;
using namespace observer;
namespace sql
{
DASRefCountContext::DASRefCountContext()
: max_das_task_concurrency_(INT32_MAX),
das_task_concurrency_limit_(INT32_MAX),
err_ret_(OB_SUCCESS),
cond_(),
need_wait_(false),
is_inited_(false)
{
}
void DASRefCountContext::inc_concurrency_limit_with_signal()
{
ObThreadCondGuard guard(cond_);
if (__sync_add_and_fetch(&das_task_concurrency_limit_, 1) == max_das_task_concurrency_) {
cond_.signal();
LOG_TRACE("inc currency with signal", K(get_current_concurrency()));
}
}
void DASRefCountContext::inc_concurrency_limit()
{
ATOMIC_INC(&das_task_concurrency_limit_);
}
int DASRefCountContext::dec_concurrency_limit()
{
int ret = OB_SUCCESS;
int32_t cur = get_current_concurrency();
int32_t next = cur - 1;
if (OB_UNLIKELY(0 == cur)) {
ret = OB_SIZE_OVERFLOW;
} else {
while (ATOMIC_CAS(&das_task_concurrency_limit_, cur, next) != cur) {
cur = get_current_concurrency();
next = cur - 1;
if (OB_UNLIKELY(0 == cur)) {
ret = OB_SIZE_OVERFLOW;
break;
}
}
}
return ret;
}
// not thread safe.
int DASRefCountContext::acquire_task_execution_resource(int64_t timeout_ts)
{
int ret = OB_SUCCESS;
OB_ASSERT(get_current_concurrency() >= 0);
set_need_wait(true);
if (!is_inited_ && OB_FAIL(cond_.init(ObWaitEventIds::DAS_ASYNC_RPC_LOCK_WAIT))) {
LOG_WARN("fail to init condition", K(ret));
} else if (FALSE_IT(is_inited_ = true)) {
// do nothing
} else if (OB_FAIL(dec_concurrency_limit())) {
LOG_WARN("failed to acquire das execution resource", K(ret), K(get_current_concurrency()));
}
if (OB_UNLIKELY(OB_SIZE_OVERFLOW == ret)) {
ret = OB_SUCCESS;
ObThreadCondGuard guard(cond_);
if (OB_FAIL(cond_.wait(timeout_ts - ObTimeUtility::current_time()))) {
LOG_WARN("failed to acquire das task execution resource", K(ret), K(get_current_concurrency()));
} else if (OB_FAIL(dec_concurrency_limit())) {
LOG_WARN("failed to acquire das execution resource", K(ret), K(get_current_concurrency()));
}
}
return ret;
}
bool DasRefKey::operator==(const DasRefKey &other) const
{
return (tablet_loc_ == other.tablet_loc_ && op_type_ == other.op_type_);
@ -49,27 +119,19 @@ ObDASRef::ObDASRef(ObEvalCtx &eval_ctx, ObExecContext &exec_ctx)
batched_tasks_(das_alloc_),
exec_ctx_(exec_ctx),
eval_ctx_(eval_ctx),
frozen_op_node_(nullptr),
expr_frame_info_(nullptr),
wild_datum_info_(eval_ctx),
del_aggregated_tasks_(das_alloc_),
aggregated_tasks_(das_alloc_),
lookup_cnt_(0),
task_cnt_(0),
init_mem_used_(exec_ctx.get_allocator().used()),
task_map_(),
max_das_task_concurrency_(1),
das_task_concurrency_limit_(1),
cond_(),
async_cb_list_(das_alloc_),
das_ref_count_ctx_(),
das_parallel_ctx_(),
flags_(0)
{
int ret = OB_SUCCESS;
max_das_task_concurrency_ = MTL(ObDataAccessService *)->get_das_concurrency_limit();
OB_ASSERT(max_das_task_concurrency_ > 0);
das_task_concurrency_limit_ = max_das_task_concurrency_;
if (OB_FAIL(cond_.init(ObWaitEventIds::DAS_ASYNC_RPC_LOCK_WAIT))) {
LOG_ERROR("Failed to init thread cond", K(ret), K(MTL_ID()));
}
}
DASOpResultIter ObDASRef::begin_result_iter()
@ -81,9 +143,6 @@ ObIDASTaskOp* ObDASRef::find_das_task(const ObDASTabletLoc *tablet_loc, ObDASOpT
{
int ret = OB_SUCCESS;
ObIDASTaskOp *das_task = nullptr;
if (nullptr == frozen_op_node_) {
frozen_op_node_ = batched_tasks_.get_header_node();
}
lookup_cnt_++;
if (task_map_.created()) {
DasRefKey key(tablet_loc, op_type);
@ -98,12 +157,14 @@ ObIDASTaskOp* ObDASRef::find_das_task(const ObDASTabletLoc *tablet_loc, ObDASOpT
} else if (OB_HASH_NOT_EXIST == ret) {
// key not found
} else {
DASTaskIter task_iter(frozen_op_node_->get_next(), batched_tasks_.get_header_node());
DASTaskIter task_iter(batched_tasks_.get_header_node()->get_next(), batched_tasks_.get_header_node());
for (; nullptr == das_task && !task_iter.is_end(); ++task_iter) {
ObIDASTaskOp *tmp_task = *task_iter;
if (tmp_task != nullptr &&
tmp_task->get_tablet_loc() == tablet_loc &&
tmp_task->get_type() == op_type) {
tmp_task->get_type() == op_type &&
!(tmp_task->is_write_buff_full()) &&
tmp_task->get_agg_task()->start_status_ == DAS_AGG_TASK_UNSTART) {
das_task = tmp_task;
}
}
@ -127,12 +188,15 @@ int ObDASRef::create_task_map()
} else if (OB_FAIL(task_map_.create(DAS_REF_MAP_BUCKET_SIZE, ObModIds::OB_HASH_BUCKET))) {
LOG_WARN("create task map failed", KR(ret));
} else {
DASTaskIter task_iter(frozen_op_node_->get_next(), batched_tasks_.get_header_node());
DASTaskIter task_iter(batched_tasks_.get_header_node()->get_next(), batched_tasks_.get_header_node());
for (; OB_SUCC(ret) && !task_iter.is_end(); ++task_iter) {
ObIDASTaskOp *task = *task_iter;
if (OB_ISNULL(task)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("task is null", KR(ret), KP(task));
} else if (task->is_write_buff_full() ||
task->get_agg_task()->start_status_ != DAS_AGG_TASK_UNSTART) {
LOG_TRACE("this task is submitted or write_buffer is full", K(ret));
} else {
DasRefKey key(task->get_tablet_loc(), task->get_type());
if (OB_FAIL(task_map_.set_refactored(key, task))) {
@ -184,8 +248,8 @@ int ObDASRef::pick_del_task_to_first()
{
int ret = OB_SUCCESS;
#if !defined(NDEBUG)
LOG_DEBUG("print all das_task before sort");
print_all_das_task();
// LOG_DEBUG("print all das_task before sort");
// print_all_das_task();
#endif
DasOpNode *head_node = batched_tasks_.get_obj_list().get_header();
DasOpNode *curr_node = batched_tasks_.get_obj_list().get_first();
@ -229,100 +293,150 @@ bool ObDASRef::is_all_local_task() const
return bret;
}
// In order to solve the problem that tx_desc will be modified concurrently by concurrent threads,
// when the remote das_task is sent, the get_serialize_size and serialize of tx_desc will report an error 4019 due to non-atomicity.
// So we use the SQL main thread to copy a tx_desc_bak_ to remote_das_task for use when submitting the first concurrent task.
// When the savepoint rollback occurs,tx_desc.op_sn_ will change.
// tx_desc_bak_ must be refreshed to ensure that subsequent execution will not report errors.
// The refresh_tx_desc_bak function handles the above logic.
int ObDASRef::parallel_submit_agg_task(ObDasAggregatedTask *agg_task)
{
int ret = OB_SUCCESS;
ObSQLSessionInfo *session = exec_ctx_.get_my_session();
agg_task->set_start_status(DAS_AGG_TASK_PARALLEL_EXEC);
if (OB_ISNULL(session)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null ptr", K(ret));
} else if (OB_FAIL(get_das_parallel_ctx().refresh_tx_desc_bak(get_das_alloc(), session->get_tx_desc()))) {
LOG_WARN("fail to check and refresh tx_desc", K(ret));
} else if (OB_FAIL(MTL(ObDataAccessService *)->parallel_submit_das_task(*this, *agg_task))) {
LOG_WARN("fail to execute parallel_das_task", K(ret));
} else {
LOG_TRACE("succeed submit parallel task", K(ret), K(agg_task));
}
return ret;
}
int ObDASRef::execute_all_task()
{
int ret = OB_SUCCESS;
const bool async = GET_MIN_CLUSTER_VERSION() >= CLUSTER_VERSION_4_1_0_0;
// move local aggregated tasks to last for better concurrency.
if (OB_FAIL(move_local_tasks_to_last())) {
LOG_WARN("failed to move local tasks to last.", K(ret));
if (OB_FAIL(execute_all_task(del_aggregated_tasks_))) {
LOG_WARN("fail to execute all delete agg_tasks", K(ret));
} else if (OB_FAIL(execute_all_task(aggregated_tasks_))) {
LOG_WARN("fail to execute all agg_tasks", K(ret));
} else {
uint32_t finished_cnt = 0;
uint32_t high_priority_task_execution_cnt = 0;
bool has_unstart_high_priority_tasks = true;
while (finished_cnt < aggregated_tasks_.get_size() && OB_SUCC(ret)) {
finished_cnt = 0;
// execute tasks follows aggregated task state machine.
if (has_unstart_high_priority_tasks) {
high_priority_task_execution_cnt = 0;
DLIST_FOREACH_X(curr, aggregated_tasks_.get_obj_list(), OB_SUCC(ret)) {
ObDasAggregatedTasks* aggregated_task = curr->get_obj();
if (aggregated_task->has_unstart_high_priority_tasks()) {
if (OB_FAIL(MTL(ObDataAccessService *)->execute_das_task(*this, *aggregated_task, async))) {
LOG_WARN("failed to execute high priority aggregated das task", KR(ret), KPC(aggregated_task), K(async));
} else {
++high_priority_task_execution_cnt;
LOG_DEBUG("successfully executing aggregated task", "server", aggregated_task->server_);
}
}
}
if (high_priority_task_execution_cnt == 0) {
has_unstart_high_priority_tasks = false;
}
DASTaskIter task_iter = begin_task_iter();
while (OB_SUCC(ret) && !task_iter.is_end()) {
int end_ret = OB_SUCCESS;
ObIDASTaskOp *das_op = nullptr;
if (OB_ISNULL(das_op = *task_iter)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null ptr", K(ret));
} else if (OB_FAIL(das_op->record_task_result_to_rtdef())) {
LOG_WARN("fail to record task result", K(ret), KPC(das_op));
} else {
++task_iter;
}
if (!has_unstart_high_priority_tasks) {
DLIST_FOREACH_X(curr, aggregated_tasks_.get_obj_list(), OB_SUCC(ret)) {
ObDasAggregatedTasks* aggregated_task = curr->get_obj();
if (aggregated_task->has_unstart_tasks()) {
if (OB_FAIL(MTL(ObDataAccessService *)->execute_das_task(*this, *aggregated_task, async))) {
LOG_WARN("failed to execute aggregated das task", KR(ret), KPC(aggregated_task), K(async));
} else {
LOG_DEBUG("successfully executing aggregated task", "server", aggregated_task->server_);
}
}
}
}
// wait all existing tasks to be finished
int tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(wait_executing_tasks())) {
LOG_WARN("failed to process all async remote tasks", KR(ret));
}
ret = COVER_SUCC(tmp_ret);
if (OB_FAIL(ret) && check_rcode_can_retry(ret)) {
ret = OB_SUCCESS;
}
if (OB_SUCC(ret)) {
// check das task status.
DLIST_FOREACH_X(curr, aggregated_tasks_.get_obj_list(), OB_SUCC(ret)) {
ObDasAggregatedTasks* aggregated_task = curr->get_obj();
if (aggregated_task->has_unstart_tasks()) {
if (aggregated_task->has_failed_tasks()) {
// retry all failed tasks.
common::ObSEArray<ObIDASTaskOp *, 2> failed_tasks;
int tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(aggregated_task->get_failed_tasks(failed_tasks))) {
LOG_WARN("failed to get failed tasks", K(ret));
} else if (failed_tasks.count() == 0) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to get failed tasks");
} else {
for (int i = 0; OB_SUCC(ret) && i < failed_tasks.count(); i++) {
ObIDASTaskOp *failed_task = failed_tasks.at(i);
if (!GCONF._enable_partition_level_retry || !failed_task->can_part_retry()) {
ret = failed_task->errcode_;
} else if (OB_FAIL(MTL(ObDataAccessService *)->retry_das_task(*this, *failed_tasks.at(i)))) {
LOG_WARN("Failed to retry das task", K(ret));
}
}
}
} else {
// proceed while loop for other unfinished tasks.
}
}
}
return ret;
}
bool ObDASRef::check_agg_task_can_retry(ObDasAggregatedTask *agg_task)
{
bool bret = false;
if (agg_task->get_save_ret() != OB_SUCCESS) {
if (agg_task->has_parallel_submiitted()) {
bret = false;
// parallel submit task can't do retry
} else if (check_rcode_can_retry(agg_task->get_save_ret())) {
bret = true;
}
} else {
bret = true;
}
return bret;
}
int ObDASRef::retry_all_fail_tasks(common::ObIArray<ObIDASTaskOp *> &failed_tasks)
{
int ret = OB_SUCCESS;
for (int i = 0; OB_SUCC(ret) && i < failed_tasks.count(); i++) {
ObIDASTaskOp *failed_task = failed_tasks.at(i);
if (!GCONF._enable_partition_level_retry || !failed_task->can_part_retry()) {
ret = failed_task->errcode_;
LOG_WARN("can't do task level retry", K(ret), KPC(failed_task));
} else if (OB_FAIL(MTL(ObDataAccessService *)->retry_das_task(*this, *failed_tasks.at(i)))) {
LOG_WARN("Failed to retry das task", K(ret));
}
}
return ret;
}
int ObDASRef::execute_all_task(DasAggregatedTaskList &agg_task_list)
{
int ret = OB_SUCCESS;
const bool async = GET_MIN_CLUSTER_VERSION() >= CLUSTER_VERSION_4_1_0_0;
uint32_t finished_cnt = 0;
while (finished_cnt < agg_task_list.get_size() && OB_SUCC(ret)) {
finished_cnt = 0;
// execute tasks follows aggregated task state machine.
DLIST_FOREACH_X(curr, agg_task_list.get_obj_list(), OB_SUCC(ret)) {
ObDasAggregatedTask* agg_task = curr->get_obj();
if (agg_task->has_unstart_tasks() && !agg_task->has_parallel_submiitted()) {
if (get_parallel_type() == DAS_SERIALIZATION) {
if (OB_FAIL(MTL(ObDataAccessService *)->execute_das_task(*this, *agg_task, async))) {
LOG_WARN("failed to execute aggregated das task", K(ret), KPC(agg_task), K(async));
} else {
++finished_cnt;
#if !defined(NDEBUG)
OB_ASSERT(aggregated_task->high_priority_tasks_.get_size() == 0);
OB_ASSERT(aggregated_task->tasks_.get_size() == 0);
OB_ASSERT(aggregated_task->failed_tasks_.get_size() == 0);
OB_ASSERT(aggregated_task->success_tasks_.get_size() != 0);
DLIST_FOREACH_X(curr_task, aggregated_task->success_tasks_, true) {
ObIDASTaskOp *tmp_task = curr_task->get_data();
OB_ASSERT(ObDasTaskStatus::FINISHED == tmp_task->get_task_status());
}
#endif
LOG_DEBUG("successfully executing aggregated task", "server", agg_task->server_);
}
} else {
if (OB_FAIL(parallel_submit_agg_task(agg_task))) {
LOG_WARN("failed to execute aggregated das task", K(ret), KPC(agg_task));
} else {
LOG_DEBUG("successfully parallel submit agg_task", KPC(agg_task));
}
}
LOG_DEBUG("current das task status", K(finished_cnt), K(aggregated_tasks_.get_size()));
}
}
// wait all existing tasks to be finished
int tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(wait_tasks_and_process_response())) {
LOG_WARN("failed to process all async remote tasks", K(ret));
}
ret = COVER_SUCC(tmp_ret);
if (OB_FAIL(ret) && check_rcode_can_retry(ret)) {
ret = OB_SUCCESS;
}
// check das task status.
DLIST_FOREACH_X(curr, agg_task_list.get_obj_list(), OB_SUCC(ret)) {
ObDasAggregatedTask* aggregated_task = curr->get_obj();
if (aggregated_task->has_parallel_submiitted() && aggregated_task->get_save_ret() != OB_SUCCESS) {
// all parallel_submit task can't retry
ret = aggregated_task->get_save_ret();
LOG_WARN("can't retry for this error_ret", K(ret), KPC(aggregated_task));
} else if (aggregated_task->has_not_execute_task()) {
// 还有任务没执行完毕
if (aggregated_task->has_failed_tasks()) {
// retry all failed tasks.
common::ObSEArray<ObIDASTaskOp *, 2> failed_tasks;
int tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(aggregated_task->get_failed_tasks(failed_tasks))) {
LOG_WARN("failed to get failed tasks", K(ret));
} else if (failed_tasks.count() == 0) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to get failed tasks");
} else if (OB_FAIL(retry_all_fail_tasks(failed_tasks))) {
LOG_WARN("fail to retry das tasks", K(ret), K(failed_tasks));
}
}
} else {
++finished_cnt;
LOG_DEBUG("check finish agg_task print agg_list", K(finished_cnt), K(agg_task_list.get_size()), KPC(aggregated_task));
}
}
}
@ -343,26 +457,40 @@ bool ObDASRef::check_rcode_can_retry(int ret)
return bret;
}
int ObDASRef::wait_executing_tasks()
int ObDASRef::wait_all_executing_tasks()
{
int ret = OB_SUCCESS;
{
ObThreadCondGuard guard(cond_);
while (OB_SUCC(ret) && get_current_concurrency() < max_das_task_concurrency_) {
if (das_ref_count_ctx_.is_need_wait()) {
ObThreadCondGuard guard(das_ref_count_ctx_.get_cond());
while (OB_SUCC(ret) &&
das_ref_count_ctx_.get_current_concurrency() < das_ref_count_ctx_.get_max_das_task_concurrency()) {
// we cannot use ObCond here because it can not explicitly lock mutex, causing concurrency problem.
if (OB_FAIL(cond_.wait())) {
if (OB_FAIL(das_ref_count_ctx_.get_cond().wait())) {
LOG_WARN("failed to wait all das tasks to be finished.", K(ret));
}
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(process_remote_task_resp())) {
LOG_WARN("failed to process remote task resp", K(ret));
}
return ret;
}
int ObDASRef::wait_tasks_and_process_response()
{
int ret = OB_SUCCESS;
if (OB_FAIL(wait_all_executing_tasks())) {
LOG_WARN("fail to wait all executing tasks", K(ret));
} else if (OB_FAIL(process_remote_task_resp())) {
LOG_WARN("failed to process remote task resp", K(ret));
}
return ret;
}
void ObDASRef::clear_task_map()
{
if (task_map_.created()) {
task_map_.clear();
}
}
int ObDASRef::wait_all_tasks()
{
// won't implement until das async execution.
@ -434,7 +562,7 @@ int ObDASRef::move_local_tasks_to_last()
bool found_local_tasks = false;
const common::ObAddr &ctrl_addr = MTL(ObDataAccessService *)->get_ctrl_addr();
DLIST_FOREACH_X(curr, aggregated_tasks_.get_obj_list(), !found_local_tasks) {
ObDasAggregatedTasks* aggregated_task = curr->get_obj();
ObDasAggregatedTask* aggregated_task = curr->get_obj();
if (aggregated_task->server_ == ctrl_addr) {
if (!aggregated_tasks_.get_obj_list().move_to_last(curr)) {
ret = OB_ERR_UNEXPECTED;
@ -446,16 +574,6 @@ int ObDASRef::move_local_tasks_to_last()
return ret;
}
void ObDASRef::set_frozen_node()
{
frozen_op_node_ = batched_tasks_.get_last_node();
lookup_cnt_ = 0;
task_cnt_ = 0;
if (task_map_.created()) {
task_map_.clear();
}
}
int ObDASRef::close_all_task()
{
int ret = OB_SUCCESS;
@ -463,9 +581,17 @@ int ObDASRef::close_all_task()
if (has_task()) {
FLTSpanGuard(close_das_task);
ObSQLSessionInfo *session = nullptr;
int wait_ret = OB_SUCCESS;
if (get_parallel_type() != DAS_SERIALIZATION) {
// parallel submit das_task maybe some task is executing, must wait all task end
// and maybe remote_task need mereg_trans_result
if (OB_SUCCESS != (wait_ret = wait_tasks_and_process_response())) {
LOG_WARN("fail to wait all executing tasks", K(wait_ret));
}
}
ret = COVER_SUCC(wait_ret);
DASTaskIter task_iter = begin_task_iter();
while (OB_SUCC(ret) && !task_iter.is_end()) {
while (!task_iter.is_end()) {
int end_ret = OB_SUCCESS;
if (OB_SUCCESS != (end_ret = MTL(ObDataAccessService*)->end_das_task(*this, **task_iter))) {
LOG_WARN("execute das task failed", K(end_ret));
@ -492,6 +618,7 @@ int ObDASRef::close_all_task()
session->get_trans_result().set_incomplete();
}
batched_tasks_.destroy();
del_aggregated_tasks_.destroy();
aggregated_tasks_.destroy();
if (task_map_.created()) {
task_map_.destroy();
@ -522,64 +649,140 @@ int ObDASRef::create_das_task(const ObDASTabletLoc *tablet_loc,
task_op->set_tablet_id(tablet_loc->tablet_id_);
task_op->set_ls_id(tablet_loc->ls_id_);
task_op->set_tablet_loc(tablet_loc);
if (is_do_gts_opt() && OB_FAIL(task_op->init_das_gts_opt_info(session->get_tx_isolation()))) {
LOG_WARN("fail to init gts opt info", K(ret), K(session->get_tx_isolation()));
} else if (OB_FAIL(add_aggregated_task(task_op, op_type))) {
LOG_WARN("failed to add aggregated task", K(ret));
}
ObDiagnosticInfo *di = ObLocalDiagnosticInfo::get();
if (OB_NOT_NULL(di)) {
task_op->set_plan_line_id(di->get_ash_stat().plan_line_id_);
}
if (OB_FAIL(add_aggregated_task(task_op))) {
LOG_WARN("failed to add aggregated task", KR(ret));
}
}
return ret;
}
int ObDASRef::add_aggregated_task(ObIDASTaskOp *das_task)
int ObDASRef::find_agg_task(const ObDASTabletLoc *tablet_loc, ObDASOpType op_type, ObDasAggregatedTask *&agg_task)
{
int ret = OB_SUCCESS;
bool aggregated = false;
DLIST_FOREACH_X(curr, aggregated_tasks_.get_obj_list(), !aggregated && OB_SUCC(ret)) {
ObDasAggregatedTasks* aggregated_task = curr->get_obj();
if (aggregated_task->server_ == das_task->tablet_loc_->server_) {
if (OB_FAIL(aggregated_task->push_back_task(das_task))) {
LOG_WARN("failed to add aggregated tasks", KR(ret));
} else {
if (DAS_OP_TABLE_DELETE == op_type) {
DLIST_FOREACH_X(curr, del_aggregated_tasks_.get_obj_list(), !aggregated && OB_SUCC(ret)) {
ObDasAggregatedTask* aggregated_task = curr->get_obj();
if (aggregated_task->server_ == tablet_loc->server_ &&
aggregated_task->start_status_ == DAS_AGG_TASK_UNSTART) {
agg_task = aggregated_task;
aggregated = true;
}
}
} else {
DLIST_FOREACH_X(curr, aggregated_tasks_.get_obj_list(), !aggregated && OB_SUCC(ret)) {
ObDasAggregatedTask* aggregated_task = curr->get_obj();
if (aggregated_task->server_ == tablet_loc->server_ &&
aggregated_task->start_status_ == DAS_AGG_TASK_UNSTART) {
agg_task = aggregated_task;
aggregated = true;
}
}
}
if (OB_FAIL(ret)) {
} else if (!aggregated) {
void *buf = das_alloc_.alloc(sizeof(ObDasAggregatedTasks));
ObDasAggregatedTasks *agg_tasks = nullptr;
if (OB_ISNULL(buf)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to allocate memory for aggregated tasks", KR(ret));
} else if (FALSE_IT(agg_tasks = new(buf) ObDasAggregatedTasks(das_alloc_))) {
} else if (OB_FAIL(aggregated_tasks_.store_obj(agg_tasks))) {
LOG_WARN("failed to add aggregated tasks", KR(ret));
} else if (OB_FAIL(agg_tasks->push_back_task(das_task))) {
LOG_WARN("failed to add task", K(ret));
return ret;
}
int ObDASRef::create_agg_task(ObDASOpType op_type, const ObDASTabletLoc *tablet_loc, ObDasAggregatedTask *&agg_task)
{
int ret = OB_SUCCESS;
// create agg_task
void *buf = das_alloc_.alloc(sizeof(ObDasAggregatedTask));
ObDasAggregatedTask *tmp_agg_task = nullptr;
agg_task = nullptr;
if (OB_ISNULL(buf)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to allocate memory for aggregated tasks", KR(ret));
} else if (FALSE_IT(tmp_agg_task = new(buf) ObDasAggregatedTask(das_alloc_))) {
} else if (DAS_OP_TABLE_DELETE == op_type &&
OB_FAIL(del_aggregated_tasks_.store_obj(tmp_agg_task))) {
LOG_WARN("failed to add aggregated tasks", KR(ret));
} else if (DAS_OP_TABLE_DELETE != op_type &&
OB_FAIL(aggregated_tasks_.store_obj(tmp_agg_task))) {
LOG_WARN("failed to add aggregated tasks", KR(ret));
} else {
agg_task = tmp_agg_task;
agg_task->server_ = tablet_loc->server_;
}
return ret;
}
int ObDASRef::add_aggregated_task(ObIDASTaskOp *das_task, ObDASOpType op_type)
{
int ret = OB_SUCCESS;
bool aggregated = false;
ObDasAggregatedTask *agg_task = nullptr;
if (OB_FAIL(find_agg_task(das_task->tablet_loc_, op_type, agg_task))) {
LOG_WARN("fail to find agg_task", K(ret), K(op_type));
} else if (OB_NOT_NULL(agg_task)) {
if (OB_FAIL(OB_FAIL(agg_task->push_back_task(das_task)))) {
LOG_WARN("fail to push back das_task", K(ret), KPC(das_task));
}
} else {
// create agg_task
if (OB_FAIL(create_agg_task(op_type, das_task->tablet_loc_, agg_task))) {
LOG_WARN("fail to create agg_task", K(ret), K(op_type));
} else if (OB_FAIL(agg_task->push_back_task(das_task))) {
LOG_WARN("fail to push back das_task", K(ret), KPC(das_task));
}
}
if (OB_SUCC(ret) && OB_FAIL(add_batched_task(das_task))) {
LOG_WARN("add batched task failed", KR(ret), KPC(das_task));
}
return ret;
}
bool ObDASRef::check_tasks_same_ls_and_is_local(share::ObLSID &ls_id)
{
ObIDASTaskOp *first_das_op = nullptr;
bool is_all_same = true;
DASTaskIter task_iter = begin_task_iter();
const common::ObAddr &ctrl_addr = MTL(ObDataAccessService *)->get_ctrl_addr();
if (!has_task()) {
is_all_same = false;
}
while (is_all_same && !task_iter.is_end()) {
ObIDASTaskOp *das_op = *task_iter;
if (OB_ISNULL(first_das_op)) {
first_das_op = das_op;
if (first_das_op->get_tablet_loc()->server_ != ctrl_addr) {
is_all_same = false;
}
} else if (first_das_op->get_ls_id() != das_op->get_ls_id() ||
first_das_op->get_tablet_loc()->server_ != das_op->get_tablet_loc()->server_) {
is_all_same = false;
}
++task_iter;
}
if (is_all_same) {
ls_id = first_das_op->get_ls_id();
}
return is_all_same;
}
void ObDASRef::reset()
{
das_factory_.cleanup();
batched_tasks_.destroy();
del_aggregated_tasks_.destroy();
aggregated_tasks_.destroy();
lookup_cnt_ = 0;
task_cnt_ = 0;
init_mem_used_ = 0;
das_ref_count_ctx_.reuse();
das_parallel_ctx_.reset();
if (task_map_.created()) {
task_map_.destroy();
}
flags_ = false;
frozen_op_node_ = nullptr;
expr_frame_info_ = nullptr;
if (reuse_alloc_ != nullptr) {
reuse_alloc_->reset();
@ -591,14 +794,16 @@ void ObDASRef::reuse()
{
das_factory_.cleanup();
batched_tasks_.destroy();
del_aggregated_tasks_.destroy();
aggregated_tasks_.destroy();
lookup_cnt_ = 0;
task_cnt_ = 0;
init_mem_used_ = 0;
das_ref_count_ctx_.reuse();
das_parallel_ctx_.reuse();
if (task_map_.created()) {
task_map_.destroy();
}
frozen_op_node_ = nullptr;
if (reuse_alloc_ != nullptr) {
reuse_alloc_->reset_remain_one_page();
} else {
@ -608,115 +813,45 @@ void ObDASRef::reuse()
}
}
int32_t ObDASRef::get_current_concurrency() const
{
return ATOMIC_LOAD(&das_task_concurrency_limit_);
};
void ObDASRef::inc_concurrency_limit()
{
ATOMIC_INC(&das_task_concurrency_limit_);
}
void ObDASRef::inc_concurrency_limit_with_signal()
{
ObThreadCondGuard guard(cond_);
if (__sync_add_and_fetch(&das_task_concurrency_limit_, 1) == max_das_task_concurrency_) {
cond_.signal();
}
}
int ObDASRef::dec_concurrency_limit()
{
int ret = OB_SUCCESS;
int32_t cur = get_current_concurrency();
int32_t next = cur - 1;
if (OB_UNLIKELY(0 == cur)) {
ret = OB_SIZE_OVERFLOW;
} else {
while (ATOMIC_CAS(&das_task_concurrency_limit_, cur, next) != cur) {
cur = get_current_concurrency();
next = cur - 1;
if (OB_UNLIKELY(0 == cur)) {
ret = OB_SIZE_OVERFLOW;
break;
}
}
}
return ret;
}
// not thread safe.
int ObDASRef::acquire_task_execution_resource()
{
int ret = OB_SUCCESS;
OB_ASSERT(get_current_concurrency() >= 0);
if (OB_FAIL(dec_concurrency_limit())) {
LOG_WARN("failed to acquire das execution resource", K(ret), K(get_current_concurrency()));
}
if (OB_UNLIKELY(OB_SIZE_OVERFLOW == ret)) {
ret = OB_SUCCESS;
ObThreadCondGuard guard(cond_);
if (OB_FAIL(cond_.wait(get_exec_ctx().get_my_session()->get_query_timeout_ts() -
ObTimeUtility::current_time()))) {
LOG_WARN("failed to acquire das task execution resource", K(ret), K(get_current_concurrency()));
} else if (OB_FAIL(dec_concurrency_limit())) {
LOG_WARN("failed to acquire das execution resource", K(ret), K(get_current_concurrency()));
}
}
return ret;
}
void ObDasAggregatedTasks::reset()
void ObDasAggregatedTask::reset()
{
server_.reset();
high_priority_tasks_.reset();
tasks_.reset();
failed_tasks_.reset();
success_tasks_.reset();
}
void ObDasAggregatedTasks::reuse()
void ObDasAggregatedTask::reuse()
{
server_.reset();
high_priority_tasks_.reset();
tasks_.reset();
failed_tasks_.reset();
success_tasks_.reset();
}
int ObDasAggregatedTasks::push_back_task(ObIDASTaskOp *das_task)
int ObDasAggregatedTask::push_back_task(ObIDASTaskOp *das_task)
{
int ret = OB_SUCCESS;
if (das_task->get_cur_agg_list()) {
// if task already have linked list (in task retry), remove it first
das_task->get_cur_agg_list()->remove(&das_task->get_node());
}
if (high_priority_tasks_.get_size() == 0 && tasks_.get_size() == 0) {
if (tasks_.get_size() == 0) {
server_ = das_task->get_tablet_loc()->server_;
}
if (ObDASOpType::DAS_OP_TABLE_DELETE == das_task->get_type()) {
// we move all DELETE das op to high priority tasks anyway.
if (OB_UNLIKELY(!high_priority_tasks_.add_last(&das_task->get_node()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to push back high priority task", K(ret));
} else {
das_task->set_cur_agg_list(&high_priority_tasks_);
}
} else if (OB_UNLIKELY(!tasks_.add_last(&das_task->get_node()))) {
if (OB_UNLIKELY(!tasks_.add_last(&das_task->get_node()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to push back normal das task", K(ret));
} else {
das_task->set_cur_agg_list(&tasks_);
}
if (OB_SUCC(ret) && !das_task->get_agg_tasks()) {
das_task->set_agg_tasks(this);
if (OB_SUCC(ret) && !das_task->get_agg_task()) {
das_task->set_agg_task(this);
}
return ret;
}
int ObDasAggregatedTasks::get_aggregated_tasks(
common::ObSEArray<ObIDASTaskOp *, 2> &tasks) {
int ObDasAggregatedTask::get_aggregated_tasks(common::ObIArray<ObIDASTaskOp *> &tasks) {
int ret = OB_SUCCESS;
ObIDASTaskOp *cur_task = nullptr;
// 1. if have failed tasks, should explicitly get failed task via get_failed_tasks().
@ -725,27 +860,12 @@ int ObDasAggregatedTasks::get_aggregated_tasks(
LOG_WARN("das aggregated failed task exist. couldn't get unstarted tasks.", K(ret));
}
// 2. if no failed tasks exist and have unfinished high priority aggregated tasks, return all high priority tasks.
if (tasks.count() == 0 && OB_SUCC(ret)) {
DLIST_FOREACH_X(curr, high_priority_tasks_, OB_SUCC(ret)) {
cur_task = curr->get_data();
OB_ASSERT(cur_task != nullptr);
OB_ASSERT(cur_task->get_cur_agg_list() == &high_priority_tasks_);
OB_ASSERT(ObDASOpType::DAS_OP_TABLE_DELETE == cur_task->get_type());
OB_ASSERT(ObDasTaskStatus::UNSTART == cur_task->get_task_status());
if (OB_FAIL(tasks.push_back(cur_task))) {
LOG_WARN("failed to push back high prio tasks", KR(ret), K(cur_task));
}
}
}
// 3. if no unfinished high priority aggregated tasks exist, return all normal aggregated tasks.
if (tasks.count() == 0 && OB_SUCC(ret)) {
DLIST_FOREACH_X(curr, tasks_, OB_SUCC(ret)) {
cur_task = curr->get_data();
OB_ASSERT(cur_task != nullptr);
OB_ASSERT(cur_task->get_cur_agg_list() == &tasks_);
OB_ASSERT(ObDASOpType::DAS_OP_TABLE_DELETE != cur_task->get_type());
OB_ASSERT(ObDasTaskStatus::UNSTART == cur_task->get_task_status());
if (OB_FAIL(tasks.push_back(cur_task))) {
LOG_WARN("failed to push back high prio tasks", KR(ret), K(cur_task));
@ -755,71 +875,7 @@ int ObDasAggregatedTasks::get_aggregated_tasks(
return ret;
}
int ObDasAggregatedTasks::get_aggregated_tasks(
common::ObSEArray<common::ObSEArray<ObIDASTaskOp *, 2>, 2> &task_groups,
int64_t count)
{
int ret = OB_SUCCESS;
ObIDASTaskOp *cur_task = nullptr;
// 1. if have failed tasks, should explicitly get failed task via get_failed_tasks().
if (OB_UNLIKELY(failed_tasks_.get_size() != 0)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("das aggregated failed task exist. couldn't get unstarted tasks.", K(ret));
} else if (OB_LIKELY(count == 1)) {
if (OB_FAIL(task_groups.push_back(common::ObSEArray<ObIDASTaskOp *, 2>()))) {
LOG_WARN("failed to push back", K(ret));
} else if (OB_FAIL(get_aggregated_tasks(task_groups.at(0)))) {
LOG_WARN("Failed to get aggregated das task", KR(ret));
}
} else {
// 2. if no failed tasks exist and have unfinished high priority aggregated tasks, return all high priority tasks.
if (task_groups.count() == 0 && high_priority_tasks_.get_size() != 0 && OB_SUCC(ret)) {
int idx = 0;
if (high_priority_tasks_.get_size() < count) {
count = high_priority_tasks_.get_size();
}
for (int i = 0; OB_SUCC(ret) && i < count; i++) {
if (OB_FAIL(task_groups.push_back(common::ObSEArray<ObIDASTaskOp *, 2>()))) {
LOG_WARN("failed to push back", K(ret));
}
}
DLIST_FOREACH_X(curr, high_priority_tasks_, OB_SUCC(ret)) {
cur_task = curr->get_data();
OB_ASSERT(cur_task != nullptr);
OB_ASSERT(ObDASOpType::DAS_OP_TABLE_DELETE == cur_task->get_type());
OB_ASSERT(ObDasTaskStatus::UNSTART == cur_task->get_task_status());
if (OB_FAIL(task_groups.at(idx++ % count).push_back(cur_task))) {
LOG_WARN("failed to push back high prio tasks", KR(ret), K(cur_task));
}
}
}
// 3. if no unfinished high priority aggregated tasks exist, return all normal aggregated tasks.
if (task_groups.count() == 0 && tasks_.get_size() != 0 && OB_SUCC(ret)) {
int idx = 0;
if (tasks_.get_size() < count) {
count = tasks_.get_size();
}
for (int i = 0; OB_SUCC(ret) && i < count; i++) {
if (OB_FAIL(task_groups.push_back(common::ObSEArray<ObIDASTaskOp *, 2>()))) {
LOG_WARN("failed to push back", K(ret));
}
}
DLIST_FOREACH_X(curr, tasks_, OB_SUCC(ret)) {
cur_task = curr->get_data();
OB_ASSERT(cur_task != nullptr);
OB_ASSERT(ObDASOpType::DAS_OP_TABLE_DELETE != cur_task->get_type());
OB_ASSERT(ObDasTaskStatus::UNSTART == cur_task->get_task_status());
if (OB_FAIL(task_groups.at(idx++ % count).push_back(cur_task))) {
LOG_WARN("failed to push back high prio tasks", KR(ret), K(cur_task));
}
}
}
}
return ret;
}
int ObDasAggregatedTasks::move_to_success_tasks(ObIDASTaskOp *das_task)
int ObDasAggregatedTask::move_to_success_tasks(ObIDASTaskOp *das_task)
{
int ret = OB_SUCCESS;
das_task->get_cur_agg_list()->remove(&das_task->get_node());
@ -832,7 +888,7 @@ int ObDasAggregatedTasks::move_to_success_tasks(ObIDASTaskOp *das_task)
return ret;
}
int ObDasAggregatedTasks::move_to_failed_tasks(ObIDASTaskOp *das_task)
int ObDasAggregatedTask::move_to_failed_tasks(ObIDASTaskOp *das_task)
{
int ret = OB_SUCCESS;
das_task->get_cur_agg_list()->remove(&das_task->get_node());
@ -845,7 +901,7 @@ int ObDasAggregatedTasks::move_to_failed_tasks(ObIDASTaskOp *das_task)
return ret;
}
int ObDasAggregatedTasks::get_failed_tasks(common::ObSEArray<ObIDASTaskOp *, 2> &tasks)
int ObDasAggregatedTask::get_failed_tasks(common::ObSEArray<ObIDASTaskOp *, 2> &tasks)
{
int ret = OB_SUCCESS;
ObIDASTaskOp *cur_task = nullptr;
@ -860,21 +916,81 @@ int ObDasAggregatedTasks::get_failed_tasks(common::ObSEArray<ObIDASTaskOp *, 2>
return ret;
}
bool ObDasAggregatedTasks::has_unstart_tasks() const
bool ObDasAggregatedTask::has_unstart_tasks() const
{
return high_priority_tasks_.get_size() != 0 ||
tasks_.get_size() != 0 ||
failed_tasks_.get_size() != 0;
return tasks_.get_size() != 0 ;
}
bool ObDasAggregatedTasks::has_unstart_high_priority_tasks() const
int32_t ObDasAggregatedTask::get_unstart_task_size() const
{
return high_priority_tasks_.get_size() != 0;
return tasks_.get_size();
}
int32_t ObDasAggregatedTasks::get_unstart_task_size() const
int DASParallelContext::deep_copy_tx_desc(ObIAllocator &alloc, transaction::ObTxDesc *src_tx_desc)
{
return tasks_.get_size() + high_priority_tasks_.get_size();
int ret = OB_SUCCESS;
int64_t tx_desc_length = 0;
int64_t ser_pos = 0;
int64_t des_pos = 0;
void *buf = nullptr;
transaction::ObTxDesc *dst_tx_desc = nullptr;
if (OB_ISNULL(src_tx_desc)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null ptr", K(ret));
} else if (FALSE_IT(tx_desc_length = src_tx_desc->get_serialize_size())) {
// do nothing
} else if (OB_ISNULL(buf = alloc.alloc(tx_desc_length))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("alloca memory failed", K(ret));
} else if (OB_FAIL(src_tx_desc->serialize(static_cast<char *>(buf), tx_desc_length, ser_pos))) {
LOG_WARN("serialized tx_desc failed", K(ser_pos), K(tx_desc_length), K(ret));
} else if (OB_FAIL(MTL(transaction::ObTransService*)->acquire_tx(static_cast<const char *>(buf), ser_pos, des_pos, dst_tx_desc))) {
LOG_WARN("acquire tx_desc by deserialized failed", K(ser_pos), K(des_pos), K(ret));
} else if (OB_ISNULL(dst_tx_desc)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null", K(ret), K(ser_pos), K(des_pos));
} else if (ser_pos != des_pos) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("copy trans_desc failed", K(ret), K(ser_pos), K(des_pos));
} else {
tx_desc_bak_ = dst_tx_desc;
}
return ret;
}
int DASParallelContext::release_tx_desc()
{
int ret = OB_SUCCESS;
if (OB_NOT_NULL(tx_desc_bak_)) {
transaction::ObTransService *txs = MTL(transaction::ObTransService*);
txs->release_tx(*tx_desc_bak_);
tx_desc_bak_ = NULL;
}
return ret;
}
int DASParallelContext::refresh_tx_desc_bak(ObIAllocator &alloc, transaction::ObTxDesc *src_tx_desc)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(tx_desc_bak_)) {
if (OB_FAIL(deep_copy_tx_desc(alloc, src_tx_desc))) {
LOG_WARN("fail to deep_copy tx_desc", K(ret));
}
} else if (tx_desc_bak_->get_op_sn() != src_tx_desc->get_op_sn()) {
if (!has_refreshed_tx_desc_scn_) {
if (OB_FAIL(release_tx_desc())) {
LOG_WARN("fail to release tx_desc", K(ret));
} else if (OB_FAIL(deep_copy_tx_desc(alloc, src_tx_desc))) {
LOG_WARN("fail to deep copy tx_desc", K(ret), KPC(src_tx_desc));
} else {
has_refreshed_tx_desc_scn_ = true;
}
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected tx_desc op_scn", K(ret), K(tx_desc_bak_->get_op_sn()), K(src_tx_desc->get_op_sn()));
}
}
return ret;
}
} // namespace sql

204
src/sql/das/ob_das_ref.h
View File

@ -26,16 +26,124 @@ class ObDASScanOp;
class ObDASInsertOp;
class ObRpcDasAsyncAccessCallBack;
struct ObDasAggregatedTasks
enum ObDasAggTaskStartStatus
{
DAS_AGG_TASK_UNSTART = 0,
DAS_AGG_TASK_REACH_MEM_LIMIT,
DAS_AGG_TASK_PARALLEL_EXEC,
DAS_AGG_TASK_REMOTE_EXEC
};
enum ObDasParallelType
{
DAS_SERIALIZATION = 0,
DAS_STREAMING_PARALLEL,
DAS_BLOCKING_PARALLEL
};
struct DASParallelContext
{
public:
DASParallelContext()
: parallel_type_(DAS_SERIALIZATION),
submitted_task_count_(0),
das_dop_(0),
tx_desc_bak_(nullptr),
has_refreshed_tx_desc_scn_(false)
{
}
~DASParallelContext() {}
void set_das_dop(int64_t das_dop) { das_dop_ = das_dop; }
void set_das_parallel_type(ObDasParallelType v) { parallel_type_ = v; }
void add_submitted_task_count(int64_t v) { submitted_task_count_ += v; }
void reset_task_count() { submitted_task_count_ = 0; }
ObDasParallelType get_parallel_type() { return parallel_type_; }
int64_t get_submitted_task_count() { return submitted_task_count_; }
int64_t get_das_dop() { return das_dop_; }
void reuse()
{
submitted_task_count_ = 0;
has_refreshed_tx_desc_scn_ = false;
}
void reset()
{
parallel_type_ = DAS_SERIALIZATION;
submitted_task_count_ = 0;
if (OB_NOT_NULL(tx_desc_bak_)) {
transaction::ObTransService *txs = MTL(transaction::ObTransService*);
txs->release_tx(*tx_desc_bak_);
tx_desc_bak_ = NULL;
}
has_refreshed_tx_desc_scn_ = false;
}
void set_tx_desc_bak(transaction::ObTxDesc *v) { tx_desc_bak_ = v; }
transaction::ObTxDesc *get_tx_desc_bak() { return tx_desc_bak_; }
int deep_copy_tx_desc(ObIAllocator &alloc, transaction::ObTxDesc *src_tx_desc);
int refresh_tx_desc_bak(ObIAllocator &alloc, transaction::ObTxDesc *src_tx_desc);
int release_tx_desc();
TO_STRING_KV(K_(parallel_type),
K_(submitted_task_count),
K_(das_dop),
K_(tx_desc_bak),
K_(has_refreshed_tx_desc_scn));
public:
ObDasParallelType parallel_type_;
int64_t submitted_task_count_;
int64_t das_dop_;
transaction::ObTxDesc *tx_desc_bak_;
bool has_refreshed_tx_desc_scn_;
};
struct DASRefCountContext
{
public:
DASRefCountContext();
void inc_concurrency_limit_with_signal();
void inc_concurrency_limit();
int32_t get_max_das_task_concurrency() const { return max_das_task_concurrency_; };
~DASRefCountContext() {}
int32_t get_current_concurrency() const
{
return ATOMIC_LOAD(&das_task_concurrency_limit_);
}
int dec_concurrency_limit();
common::ObThreadCond &get_cond() { return cond_; }
int acquire_task_execution_resource(int64_t timeout_ts);
void interrupt_other_workers(int ret)
{
ObThreadCondGuard guard(cond_);
if (err_ret_ == OB_SUCCESS) {
err_ret_ = ret;
}
}
int get_interrupted_err_code() { return ATOMIC_LOAD(&err_ret_); }
void reuse() { need_wait_ = false; }
void set_need_wait(bool v) { need_wait_ = v; }
bool is_need_wait() { return need_wait_; }
TO_STRING_KV(K(max_das_task_concurrency_), K(das_task_concurrency_limit_));
public:
int32_t max_das_task_concurrency_;
int32_t das_task_concurrency_limit_;
int err_ret_;
common::ObThreadCond cond_;
bool need_wait_;
bool is_inited_;
};
struct ObDasAggregatedTask
{
public:
ObDasAggregatedTasks(common::ObIAllocator &allocator)
: server_(),
high_priority_tasks_(),
tasks_(),
failed_tasks_(),
success_tasks_() {}
~ObDasAggregatedTasks() { reset(); };
ObDasAggregatedTask(common::ObIAllocator &allocator)
: server_(),
tasks_(),
failed_tasks_(),
success_tasks_(),
start_status_(DAS_AGG_TASK_UNSTART),
mem_used_(0),
save_ret_(OB_SUCCESS)
{
}
~ObDasAggregatedTask() { reset(); };
void reset();
void reuse();
int push_back_task(ObIDASTaskOp *das_task);
@ -47,28 +155,43 @@ struct ObDasAggregatedTasks
* get aggregated tasks.
* @param tasks: task array to output aggregated tasks.
*/
int get_aggregated_tasks(common::ObSEArray<ObIDASTaskOp *, 2> &tasks);
/**
* get aggregated tasks.
* @param task_groups: task array's array to output aggregated tasks.
* @param count: how many aggregated tasks should be generated.
*/
int get_aggregated_tasks(common::ObSEArray<common::ObSEArray<ObIDASTaskOp *, 2>, 2> &task_groups, int64_t count);
int get_aggregated_tasks(common::ObIArray<ObIDASTaskOp *> &tasks);
int get_failed_tasks(common::ObSEArray<ObIDASTaskOp *, 2> &tasks);
bool has_failed_tasks() const { return failed_tasks_.get_size() > 0; }
bool has_unstart_tasks() const;
bool has_unstart_high_priority_tasks() const;
bool has_not_execute_task() const
{
return tasks_.get_size() != 0 || failed_tasks_.get_size() != 0;
}
int32_t get_unstart_task_size() const;
TO_STRING_KV(K_(server),
K(high_priority_tasks_.get_size()),
void set_start_status(ObDasAggTaskStartStatus status) { start_status_ = status; }
void add_mem_used(int64_t mem_used) { mem_used_ += mem_used; }
int64_t get_mem_used() { return mem_used_; }
void set_save_ret(int ret) { save_ret_ = ret; }
int get_save_ret() { return save_ret_; }
bool execute_success()
{
return save_ret_ == OB_SUCCESS && (!has_not_execute_task());
}
bool has_parallel_submiitted()
{
return (start_status_ == DAS_AGG_TASK_PARALLEL_EXEC || start_status_ == DAS_AGG_TASK_REMOTE_EXEC);
}
ObDasAggTaskStartStatus get_start_status() { return start_status_; }
TO_STRING_KV(K(server_),
K(start_status_),
K(tasks_.get_size()),
K(failed_tasks_.get_size()),
K(success_tasks_.get_size()));
K(success_tasks_.get_size()),
K(save_ret_));
common::ObAddr server_;
DasTaskLinkedList high_priority_tasks_;
DasTaskLinkedList tasks_;
DasTaskLinkedList failed_tasks_;
DasTaskLinkedList success_tasks_;
ObDasAggTaskStartStatus start_status_;
int64_t mem_used_;
int save_ret_;
};
struct DasRefKey
@ -97,6 +220,7 @@ static const int64_t DAS_REF_TASK_LOOKUP_THRESHOLD = 1000;
static const int64_t DAS_REF_TASK_SIZE_THRESHOLD = 1000;
static const int64_t DAS_REF_MAP_BUCKET_SIZE = 5000;
typedef common::hash::ObHashMap<DasRefKey, ObIDASTaskOp *, common::hash::NoPthreadDefendMode> ObDASRefMap;
typedef common::ObObjStore<ObDasAggregatedTask *, common::ObIAllocator&> DasAggregatedTaskList;
class ObDASRef
{
@ -104,6 +228,7 @@ public:
explicit ObDASRef(ObEvalCtx &eval_ctx, ObExecContext &exec_ctx);
~ObDASRef() { reset(); }
bool check_tasks_same_ls_and_is_local(share::ObLSID &ls_id);
DASOpResultIter begin_result_iter();
DASTaskIter begin_task_iter() { return batched_tasks_.begin(); }
ObDASTaskFactory &get_das_factory() { return das_factory_; }
@ -113,7 +238,9 @@ public:
template <typename DASOp>
bool has_das_op(const ObDASTabletLoc *tablet_loc, DASOp *&das_op);
ObIDASTaskOp* find_das_task(const ObDASTabletLoc *tablet_loc, ObDASOpType op_type);
int add_aggregated_task(ObIDASTaskOp *das_task);
int add_aggregated_task(ObIDASTaskOp *das_task, ObDASOpType op_type);
int create_agg_task(ObDASOpType op_type, const ObDASTabletLoc *tablet_loc, ObDasAggregatedTask *&agg_task);
int find_agg_task(const ObDASTabletLoc *tablet_loc, ObDASOpType op_type, ObDasAggregatedTask *&agg_task);
int add_batched_task(ObIDASTaskOp *das_task);
//创建一个DAS Task，并由das_ref持有
template <typename DASOp>
@ -123,9 +250,21 @@ public:
ObIDASTaskOp *&task_op);
bool has_task() const { return !batched_tasks_.empty(); }
int32_t get_das_task_cnt() const { return batched_tasks_.get_size(); }
ObDasParallelType get_parallel_type() { return das_parallel_ctx_.get_parallel_type(); }
bool is_parallel_submit() { return get_parallel_type() != DAS_SERIALIZATION; }
int64_t get_submitted_task_count() { return das_parallel_ctx_.get_submitted_task_count(); }
int64_t get_das_dop() { return das_parallel_ctx_.get_das_dop(); }
DASParallelContext &get_das_parallel_ctx() { return das_parallel_ctx_; }
int parallel_submit_agg_task(ObDasAggregatedTask *agg_task);
int execute_all_task();
int execute_all_task(DasAggregatedTaskList &agg_task_list);
bool check_agg_task_can_retry(ObDasAggregatedTask *agg_task);
int retry_all_fail_tasks(common::ObIArray<ObIDASTaskOp *> &failed_tasks);
int close_all_task();
bool is_all_local_task() const;
bool is_do_gts_opt() { return do_gts_opt_; }
void set_do_gts_opt(bool v) { do_gts_opt_ = v; }
void set_execute_directly(bool v) { execute_directly_ = v; }
bool is_execute_directly() const { return execute_directly_; }
common::ObIAllocator &get_das_alloc() { return das_alloc_; }
@ -133,30 +272,25 @@ public:
int pick_del_task_to_first();
void print_all_das_task();
void set_frozen_node();
const ObExprFrameInfo *get_expr_frame_info() const { return expr_frame_info_; }
void set_expr_frame_info(const ObExprFrameInfo *info) { expr_frame_info_ = info; }
ObEvalCtx &get_eval_ctx() { return eval_ctx_; };
void reset();
void reuse();
void set_lookup_iter(DASOpResultIter *lookup_iter) { wild_datum_info_.lookup_iter_ = lookup_iter; }
int32_t get_current_concurrency() const;
void inc_concurrency_limit();
void inc_concurrency_limit_with_signal();
int dec_concurrency_limit();
int32_t get_max_concurrency() const { return max_das_task_concurrency_; };
int acquire_task_execution_resource();
int get_aggregated_tasks_count() const { return aggregated_tasks_.get_size(); }
int wait_all_tasks();
int allocate_async_das_cb(ObRpcDasAsyncAccessCallBack *&async_cb,
const common::ObSEArray<ObIDASTaskOp*, 2> &task_ops,
int64_t timeout_ts);
void remove_async_das_cb(ObRpcDasAsyncAccessCallBack *das_async_cb);
DASRefCountContext &get_das_ref_count_ctx() { return das_ref_count_ctx_; }
void clear_task_map();
int wait_tasks_and_process_response();
private:
DISABLE_COPY_ASSIGN(ObDASRef);
int create_task_map();
int move_local_tasks_to_last();
int wait_executing_tasks();
int wait_all_executing_tasks();
int process_remote_task_resp();
bool check_rcode_can_retry(int ret);
private:
@ -173,20 +307,19 @@ private:
DasTaskList batched_tasks_;
ObExecContext &exec_ctx_;
ObEvalCtx &eval_ctx_;
DasOpNode *frozen_op_node_; // 初始为链表的head节点，冻结一次之后为链表的最后一个节点
const ObExprFrameInfo *expr_frame_info_;
DASOpResultIter::WildDatumPtrInfo wild_datum_info_;
typedef common::ObObjStore<ObDasAggregatedTasks *, common::ObIAllocator&> DasAggregatedTaskList;
DasAggregatedTaskList del_aggregated_tasks_; // we must execute all delete tasks firstly
DasAggregatedTaskList aggregated_tasks_;
int64_t lookup_cnt_;
int64_t task_cnt_;
int64_t init_mem_used_;
ObDASRefMap task_map_;
int32_t max_das_task_concurrency_;
int32_t das_task_concurrency_limit_;
common::ObThreadCond cond_;
typedef common::ObObjStore<ObRpcDasAsyncAccessCallBack *, common::ObIAllocator&> DasAsyncCbList;
DasAsyncCbList async_cb_list_;
DASRefCountContext das_ref_count_ctx_;
DASParallelContext das_parallel_ctx_;
public:
//all flags
union {
@ -194,6 +327,7 @@ public:
struct {
uint64_t execute_directly_ : 1;
uint64_t enable_rich_format_ : 1;
uint64_t do_gts_opt_ : 1;
uint64_t reserved_ : 62;
};
};

6
src/sql/das/ob_das_rpc_processor.cpp
View File

@ -263,7 +263,7 @@ void ObRpcDasAsyncAccessCallBack::on_timeout()
LOG_WARN("das async task timeout", KR(ret), K(get_task_ops()));
result_.set_err_code(ret);
result_.get_op_results().reuse();
context_->get_das_ref().inc_concurrency_limit_with_signal();
context_->get_ref_count_ctx().inc_concurrency_limit_with_signal();
}
void ObRpcDasAsyncAccessCallBack::on_invalid()
@ -275,7 +275,7 @@ void ObRpcDasAsyncAccessCallBack::on_invalid()
LOG_WARN("das async task invalid", K(get_task_ops()));
result_.set_err_code(ret);
result_.get_op_results().reuse();
context_->get_das_ref().inc_concurrency_limit_with_signal();
context_->get_ref_count_ctx().inc_concurrency_limit_with_signal();
}
void ObRpcDasAsyncAccessCallBack::set_args(const Request &arg)
@ -293,7 +293,7 @@ int ObRpcDasAsyncAccessCallBack::process()
result_.get_op_results().reuse();
LOG_WARN("das async rpc execution failed", K(get_rcode()), K_(result));
}
context_->get_das_ref().inc_concurrency_limit_with_signal();
context_->get_ref_count_ctx().inc_concurrency_limit_with_signal();
return ret;
}

21
src/sql/das/ob_das_rpc_processor.h
View File

@ -90,18 +90,23 @@ class ObDASAsyncAccessP final : public ObDASBaseAccessP<obrpc::OB_DAS_ASYNC_ACCE
class ObDasAsyncRpcCallBackContext
{
public:
ObDasAsyncRpcCallBackContext(ObDASRef &das_ref,
const common::ObSEArray<ObIDASTaskOp*, 2> &task_ops,
int64_t timeout_ts)
: das_ref_(das_ref), task_ops_(task_ops), alloc_(), timeout_ts_(timeout_ts) {}
ObDasAsyncRpcCallBackContext(DASRefCountContext &ref_count_ctx,
const common::ObSEArray<ObIDASTaskOp*, 2> &task_ops,
int64_t timeout_ts)
: ref_count_ctx_(ref_count_ctx),
task_ops_(task_ops),
alloc_(),
timeout_ts_(timeout_ts)
{
}
~ObDasAsyncRpcCallBackContext() = default;
int init(const ObMemAttr &attr);
ObDASRef &get_das_ref() { return das_ref_; };
const common::ObSEArray<ObIDASTaskOp*, 2> &get_task_ops() const { return task_ops_; };
common::ObArenaAllocator &get_alloc() { return alloc_; };
int64_t get_timeout_ts() const { return timeout_ts_; }
DASRefCountContext &get_ref_count_ctx() { return ref_count_ctx_; }
private:
ObDASRef &das_ref_;
DASRefCountContext &ref_count_ctx_;
const common::ObSEArray<ObIDASTaskOp*, 2> task_ops_;
common::ObArenaAllocator alloc_; // used for async rpc result allocation.
int64_t timeout_ts_;
@ -111,11 +116,11 @@ class ObRpcDasAsyncAccessCallBack
: public obrpc::ObDASRpcProxy::AsyncCB<obrpc::OB_DAS_ASYNC_ACCESS>
{
public:
ObRpcDasAsyncAccessCallBack(ObDasAsyncRpcCallBackContext *context)
ObRpcDasAsyncAccessCallBack(ObDasAsyncRpcCallBackContext *context, ObDASTaskFactory *factory)
: context_(context)
{
// we need das_factory to allocate task op result on receiving rpc response.
result_.set_das_factory(&context->get_das_ref().get_das_factory());
result_.set_das_factory(factory);
}
~ObRpcDasAsyncAccessCallBack() = default;
void on_timeout() override;

5
src/sql/das/ob_das_scan_op.cpp
View File

@ -219,10 +219,11 @@ ObDASScanOp::~ObDASScanOp()
int ObDASScanOp::swizzling_remote_task(ObDASRemoteInfo *remote_info)
{
int ret = OB_SUCCESS;
if (remote_info != nullptr) {
if (OB_FAIL(ObIDASTaskOp::swizzling_remote_task(remote_info))) {
LOG_WARN("fail to swizzling remote task", K(ret));
} else if (remote_info != nullptr) {
//DAS scan is executed remotely
trans_desc_ = remote_info->trans_desc_;
snapshot_ = &remote_info->snapshot_;
scan_rtdef_->stmt_allocator_.set_alloc(&CURRENT_CONTEXT->get_arena_allocator());
scan_rtdef_->scan_allocator_.set_alloc(&CURRENT_CONTEXT->get_arena_allocator());
scan_rtdef_->tsc_monitor_info_ = remote_info->tsc_monitor_info_;

2
src/sql/das/ob_das_scan_op.h
View File

@ -253,6 +253,8 @@ public:
virtual int open_op() override;
virtual int release_op() override;
virtual int record_task_result_to_rtdef() override { return OB_SUCCESS; }
virtual int assign_task_result(ObIDASTaskOp *other) override { return OB_SUCCESS; }
storage::ObTableScanParam &get_scan_param() { return scan_param_; }
const storage::ObTableScanParam &get_scan_param() const { return scan_param_; }

3
src/sql/das/ob_das_simple_op.h
View File

@ -32,7 +32,8 @@ public:
virtual int open_op() = 0;
virtual int fill_task_result(ObIDASTaskResult &task_result, bool &has_more, int64_t &memory_limit) = 0;
virtual int decode_task_result(ObIDASTaskResult *task_result) = 0;
virtual int record_task_result_to_rtdef() override { return OB_SUCCESS; }
virtual int assign_task_result(ObIDASTaskOp *other) override { return OB_SUCCESS; }
virtual int release_op() override;
virtual int init_task_info(uint32_t row_extend_size) override;
virtual int swizzling_remote_task(ObDASRemoteInfo *remote_info) override;

47
src/sql/das/ob_das_task.cpp
View File

@ -36,6 +36,7 @@ struct EnumEncoder<false, sql::ObDASBaseRtDef*> : sql::DASRtEncoder<sql::ObDASBa
} // end namespace common
using namespace common;
using namespace transaction;
namespace sql
{
OB_DEF_SERIALIZE(ObDASRemoteInfo)
@ -317,6 +318,23 @@ OB_DEF_SERIALIZE_SIZE(ObDASRemoteInfo)
return len;
}
int ObIDASTaskOp::swizzling_remote_task(ObDASRemoteInfo *remote_info)
{
int ret = OB_SUCCESS;
if (remote_info != nullptr) {
snapshot_ = &remote_info->snapshot_;
if (das_gts_opt_info_.use_specify_snapshot_) {
if (OB_ISNULL(das_gts_opt_info_.get_specify_snapshot())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr of specify_snapshot", K(ret));
} else {
snapshot_ = das_gts_opt_info_.get_specify_snapshot();
}
}
}
return ret;
}
int ObIDASTaskOp::start_das_task()
{
int &ret = errcode_;
@ -368,6 +386,17 @@ int ObIDASTaskOp::end_das_task()
return ret;
}
int ObIDASTaskOp::init_das_gts_opt_info(transaction::ObTxIsolationLevel isolation_level)
{
int ret = OB_SUCCESS;
if (OB_FAIL(get_das_gts_opt_info().init(isolation_level))) {
LOG_WARN("fail to init das gts opt", K(ret), K(isolation_level));
} else {
snapshot_ = get_das_gts_opt_info().get_specify_snapshot();
}
return ret;
}
OB_SERIALIZE_MEMBER(ObIDASTaskOp,
tenant_id_,
task_id_,
@ -433,17 +462,18 @@ int ObDASGTSOptInfo::init(transaction::ObTxIsolationLevel isolation_level)
int ret = OB_SUCCESS;
void *buf = nullptr;
void *buf2 = nullptr;
if (OB_ISNULL(buf = alloc_.alloc(sizeof(ObTxReadSnapshot)))) {
int64_t mem_size = sizeof(transaction::ObTxReadSnapshot);
if (OB_ISNULL(buf = alloc_.alloc(mem_size))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to allocate memory for ObTxReadSnapshot", K(ret), K(sizeof(ObTxReadSnapshot)));
} else if (OB_ISNULL(buf2 = alloc_.alloc(sizeof(ObTxReadSnapshot)))) {
LOG_WARN("failed to allocate memory for ObTxReadSnapshot", K(ret), K(mem_size));
} else if (OB_ISNULL(buf2 = alloc_.alloc(mem_size))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to allocate memory for ObTxReadSnapshot", K(ret), K(sizeof(ObTxReadSnapshot)));
LOG_WARN("failed to allocate memory for ObTxReadSnapshot", K(ret), K(mem_size));
} else {
use_specify_snapshot_ = true;
isolation_level_ = isolation_level;
specify_snapshot_ = new(buf) ObTxReadSnapshot();
response_snapshot_ = new(buf2) ObTxReadSnapshot();
specify_snapshot_ = new(buf) transaction::ObTxReadSnapshot();
response_snapshot_ = new(buf2) transaction::ObTxReadSnapshot();
}
return ret;
}
@ -549,11 +579,11 @@ int ObIDASTaskOp::state_advance()
OB_ASSERT(cur_agg_list_ != nullptr);
OB_ASSERT(task_status_ != ObDasTaskStatus::UNSTART);
if (task_status_ == ObDasTaskStatus::FINISHED) {
if (OB_FAIL(get_agg_tasks()->move_to_success_tasks(this))) {
if (OB_FAIL(get_agg_task()->move_to_success_tasks(this))) {
LOG_WARN("failed to move task to success tasks", KR(ret));
}
} else if (task_status_ == ObDasTaskStatus::FAILED) {
if (OB_FAIL(get_agg_tasks()->move_to_failed_tasks(this))) {
if (OB_FAIL(get_agg_task()->move_to_failed_tasks(this))) {
LOG_WARN("failed to move task to success tasks", KR(ret));
}
} else {
@ -827,5 +857,6 @@ int DASOpResultIter::reset_wild_datums_ptr()
}
return ret;
}
} // namespace sql
} // namespace oceanbase

50
src/sql/das/ob_das_task.h
View File

@ -37,7 +37,7 @@ class ObDASExtraData;
class ObExprFrameInfo;
class ObDASScanOp;
class ObDASTaskFactory;
class ObDasAggregatedTasks;
class ObDasAggregatedTask;
typedef ObDLinkNode<ObIDASTaskOp*> DasTaskNode;
typedef ObDList<DasTaskNode> DasTaskLinkedList;
@ -147,16 +147,17 @@ class ObIDASTaskOp
friend class ObDASRef;
friend class ObRpcDasAsyncAccessCallBack;
friend class ObDataAccessService;
friend class ObDASParallelHandler;
OB_UNIS_VERSION_V(1);
public:
ObIDASTaskOp(common::ObIAllocator &op_alloc)
: tenant_id_(common::OB_INVALID_ID),
task_id_(common::OB_INVALID_ID),
op_type_(DAS_OP_INVALID),
errcode_(OB_SUCCESS),
task_flag_(0),
: errcode_(OB_SUCCESS),
trans_desc_(nullptr),
snapshot_(nullptr),
tenant_id_(common::OB_INVALID_ID),
task_id_(common::OB_INVALID_ID),
op_type_(DAS_OP_INVALID),
task_flag_(0),
write_branch_id_(0),
tablet_loc_(nullptr),
op_alloc_(op_alloc),
@ -165,7 +166,7 @@ public:
related_tablet_ids_(op_alloc),
task_status_(ObDasTaskStatus::UNSTART),
das_task_node_(),
agg_tasks_(nullptr),
agg_task_(nullptr),
cur_agg_list_(nullptr),
op_result_(nullptr),
attach_ctdef_(nullptr),
@ -180,6 +181,8 @@ public:
virtual int open_op() = 0; //执行具体的DAS Task Op逻辑，由实例化的TaskOp自定义自己的执行逻辑
virtual int release_op() = 0; //close DAS Task Op,释放对应的资源
virtual int record_task_result_to_rtdef() = 0;
virtual int assign_task_result(ObIDASTaskOp *other) = 0;
void set_tablet_id(const common::ObTabletID &tablet_id) { tablet_id_ = tablet_id; }
const common::ObTabletID &get_tablet_id() const { return tablet_id_; }
void set_task_id(const int64_t task_id) { task_id_ = task_id; }
@ -206,7 +209,7 @@ public:
return common::OB_NOT_IMPLEMENT;
}
virtual int init_task_info(uint32_t row_extend_size) = 0;
virtual int swizzling_remote_task(ObDASRemoteInfo *remote_info) = 0;
virtual int swizzling_remote_task(ObDASRemoteInfo *remote_info);
virtual const ObDASBaseCtDef *get_ctdef() const { return nullptr; }
virtual ObDASBaseRtDef *get_rtdef() { return nullptr; }
virtual void reset_access_datums_ptr() { }
@ -260,13 +263,13 @@ public:
bool is_in_retry() const { return in_part_retry_ || in_stmt_retry_; }
void set_task_status(ObDasTaskStatus status);
ObDasTaskStatus get_task_status() const { return task_status_; };
const ObDasAggregatedTasks *get_agg_tasks() const { return agg_tasks_; };
ObDasAggregatedTasks *get_agg_tasks() { return agg_tasks_; };
void set_agg_tasks(ObDasAggregatedTasks *agg_tasks)
const ObDasAggregatedTask *get_agg_task() const { return agg_task_; };
ObDasAggregatedTask *get_agg_task() { return agg_task_; };
void set_agg_task(ObDasAggregatedTask *agg_task)
{
OB_ASSERT(agg_tasks != nullptr);
OB_ASSERT(agg_tasks_ == nullptr);
agg_tasks_ = agg_tasks;
OB_ASSERT(agg_task != nullptr);
OB_ASSERT(agg_task_ == nullptr);
agg_task_ = agg_task;
};
// NOT THREAD SAFE. We only advance state on das controller.
int state_advance();
@ -278,16 +281,24 @@ public:
bool get_inner_rescan() { return inner_rescan_; }
void set_inner_rescan(bool flag) { inner_rescan_ = flag; }
void set_write_buff_full(bool v) { write_buff_full_ = v; }
bool is_write_buff_full() { return write_buff_full_; }
ObDASGTSOptInfo &get_das_gts_opt_info() { return das_gts_opt_info_; }
int init_das_gts_opt_info(transaction::ObTxIsolationLevel isolation_level);
protected:
int start_das_task();
int end_das_task();
public:
int errcode_; //don't need serialize it
transaction::ObTxDesc *trans_desc_; //trans desc，事务是全局信息，由RPC框架管理，这里不维护其内存
transaction::ObTxReadSnapshot *snapshot_; // Mvcc snapshot
protected:
uint64_t tenant_id_;
int64_t task_id_;
ObDASOpType op_type_; //DAS提供的operation type
public:
int errcode_; //don't need serialize it
protected:
//事务相关信息
union
@ -304,11 +315,10 @@ protected:
uint16_t in_stmt_retry_ : 1;
uint16_t need_switch_param_ : 1; //need to switch param in gi table rescan, this parameter has been deprecated
uint16_t inner_rescan_ : 1; //disable das retry for inner_rescan
uint16_t status_reserved_ : 11;
uint16_t write_buff_full_ : 1;
uint16_t status_reserved_ : 10;
};
};
transaction::ObTxDesc *trans_desc_; //trans desc，事务是全局信息，由RPC框架管理，这里不维护其内存
transaction::ObTxReadSnapshot *snapshot_; // Mvcc snapshot
int16_t write_branch_id_; // branch id for parallel write, required for partially rollback
common::ObTabletID tablet_id_;
share::ObLSID ls_id_;
@ -330,7 +340,7 @@ protected:
ObTabletIDFixedArray related_tablet_ids_;
ObDasTaskStatus task_status_; // do not serialize
DasTaskNode das_task_node_; // tasks's linked list node, do not serialize
ObDasAggregatedTasks *agg_tasks_; //task's agg task, do not serialize
ObDasAggregatedTask *agg_task_; //task's agg task, do not serialize
DasTaskLinkedList *cur_agg_list_; //task's agg_list, do not serialize
ObIDASTaskResult *op_result_;
//The attach_ctdef describes the computations that are pushed down and executed as an attachment to the ObDASTaskOp,

39
src/sql/das/ob_das_update_op.cpp
View File

@ -308,7 +308,6 @@ int ObDASUpdateOp::open_op()
LOG_WARN("update row to partition storage failed", K(ret));
}
} else {
upd_rtdef_->affected_rows_ += affected_rows;
affected_rows_ = affected_rows;
}
return ret;
@ -320,6 +319,26 @@ int ObDASUpdateOp::release_op()
return ret;
}
int ObDASUpdateOp::assign_task_result(ObIDASTaskOp *other)
{
int ret = OB_SUCCESS;
if (other->get_type() != get_type()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected task type", K(ret), KPC(other));
} else {
ObDASUpdateOp *upd_op = static_cast<ObDASUpdateOp *>(other);
affected_rows_ = upd_op->get_affected_rows();
}
return ret;
}
int ObDASUpdateOp::record_task_result_to_rtdef()
{
int ret = OB_SUCCESS;
upd_rtdef_->affected_rows_ += affected_rows_;
return ret;
}
int ObDASUpdateOp::decode_task_result(ObIDASTaskResult *task_result)
{
int ret = OB_SUCCESS;
@ -329,7 +348,7 @@ int ObDASUpdateOp::decode_task_result(ObIDASTaskResult *task_result)
#endif
if (OB_SUCC(ret)) {
ObDASUpdateResult *del_result = static_cast<ObDASUpdateResult*>(task_result);
upd_rtdef_->affected_rows_ += del_result->get_affected_rows();
affected_rows_ = del_result->get_affected_rows();
}
return ret;
}
@ -362,29 +381,25 @@ int ObDASUpdateOp::init_task_info(uint32_t row_extend_size)
int ObDASUpdateOp::swizzling_remote_task(ObDASRemoteInfo *remote_info)
{
int ret = OB_SUCCESS;
if (remote_info != nullptr) {
if (OB_FAIL(ObIDASTaskOp::swizzling_remote_task(remote_info))) {
LOG_WARN("fail to swizzling remote task", K(ret));
} else if (remote_info != nullptr) {
//DAS update is executed remotely
trans_desc_ = remote_info->trans_desc_;
snapshot_ = &remote_info->snapshot_;
}
return ret;
}
int ObDASUpdateOp::write_row(const ExprFixedArray &row,
ObEvalCtx &eval_ctx,
ObChunkDatumStore::StoredRow *&stored_row,
bool &buffer_full)
ObChunkDatumStore::StoredRow *&stored_row)
{
int ret = OB_SUCCESS;
bool added = false;
buffer_full = false;
if (!write_buffer_.is_inited()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("buffer not inited", K(ret));
} else if (OB_FAIL(write_buffer_.try_add_row(row, &eval_ctx, das::OB_DAS_MAX_PACKET_SIZE, stored_row, added, true))) {
LOG_WARN("try add row to datum store failed", K(ret), K(row), K(write_buffer_));
} else if (!added) {
buffer_full = true;
} else if (OB_FAIL(write_buffer_.add_row(row, &eval_ctx, stored_row, true))) {
LOG_WARN("add row to write buffer failed", K(ret), K(row), K(write_buffer_));
}
return ret;
}

6
src/sql/das/ob_das_update_op.h
View File

@ -29,6 +29,8 @@ public:
virtual int open_op() override;
virtual int release_op() override;
virtual int record_task_result_to_rtdef() override;
virtual int assign_task_result(ObIDASTaskOp *other) override;
virtual int decode_task_result(ObIDASTaskResult *task_result) override;
virtual int fill_task_result(ObIDASTaskResult &task_result, bool &has_more, int64_t &memory_limit) override;
virtual int init_task_info(uint32_t row_extend_size) override;
@ -37,13 +39,13 @@ public:
virtual ObDASBaseRtDef *get_rtdef() override { return upd_rtdef_; }
int write_row(const ExprFixedArray &row,
ObEvalCtx &eval_ctx,
ObChunkDatumStore::StoredRow *&stored_row,
bool &buffer_full);
ObChunkDatumStore::StoredRow *&stored_row);
int64_t get_row_cnt() const { return write_buffer_.get_row_cnt(); }
void set_das_ctdef(const ObDASUpdCtDef *upd_ctdef) { upd_ctdef_ = upd_ctdef; }
void set_das_rtdef(ObDASUpdRtDef *upd_rtdef) { upd_rtdef_ = upd_rtdef; }
virtual int dump_data() const override
{ return write_buffer_.dump_data(*upd_ctdef_); }
int64_t get_affected_rows() { return affected_rows_; }
INHERIT_TO_STRING_KV("parent", ObIDASTaskOp,
KPC_(upd_ctdef),

223
src/sql/das/ob_data_access_service.cpp
View File

@ -23,6 +23,10 @@
#include "sql/engine/ob_exec_context.h"
#include "sql/das/ob_das_retry_ctrl.h"
#include "storage/tx/ob_trans_service.h"
#include "sql/das/ob_das_parallel_handler.h"
#include "observer/omt/ob_multi_tenant.h"
#include "lib/allocator/ob_sql_mem_leak_checker.h"
namespace oceanbase
{
using namespace share;
@ -85,7 +89,7 @@ int ObDataAccessService::init(rpc::frame::ObReqTransport *transport, const ObAdd
}
int ObDataAccessService::execute_das_task(
ObDASRef &das_ref, ObDasAggregatedTasks &task_ops, bool async) {
ObDASRef &das_ref, ObDasAggregatedTask &task_ops, bool async) {
int ret = OB_SUCCESS;
if (OB_LIKELY(das_ref.is_execute_directly())) {
common::ObSEArray<ObIDASTaskOp *, 2> task_wrapper;
@ -148,7 +152,7 @@ int ObDataAccessService::get_das_task_id(int64_t &das_id)
}
void ObDataAccessService::calc_das_task_parallelism(const ObDASRef &das_ref,
const ObDasAggregatedTasks &task_ops, int &target_parallelism)
const ObDasAggregatedTask &task_ops, int &target_parallelism)
{
// const int divide_async_task_threshold = 1024;
// if (task_ops.get_unstart_task_size() > divide_async_task_threshold) {
@ -160,45 +164,38 @@ void ObDataAccessService::calc_das_task_parallelism(const ObDASRef &das_ref,
}
OB_NOINLINE int ObDataAccessService::execute_dist_das_task(
ObDASRef &das_ref, ObDasAggregatedTasks &task_ops, bool async) {
ObDASRef &das_ref, ObDasAggregatedTask &task_ops, bool async) {
int ret = OB_SUCCESS;
ObExecContext &exec_ctx = das_ref.get_exec_ctx();
ObSQLSessionInfo *session = exec_ctx.get_my_session();
ObDASTaskArg task_arg;
task_arg.set_timeout_ts(session->get_query_timeout_ts());
int64_t timeout_ts = session->get_query_timeout_ts();
task_arg.set_timeout_ts(timeout_ts);
task_arg.set_ctrl_svr(ctrl_addr_);
task_arg.get_runner_svr() = task_ops.server_;
int target_parallelism = 0;
calc_das_task_parallelism(das_ref, task_ops, target_parallelism);
common::ObSEArray<common::ObSEArray<ObIDASTaskOp *, 2>, 2> task_groups;
if (OB_FAIL(task_ops.get_aggregated_tasks(task_groups, target_parallelism))) {
LOG_WARN("failed to get das task groups", K(ret));
}
OB_ASSERT(target_parallelism >= task_groups.count());
LOG_DEBUG("current dist das task group", K(ret), K(task_groups), K(task_ops));
for (int i = 0; OB_SUCC(ret) && i < task_groups.count(); i++) {
task_arg.get_task_ops() = task_groups.at(i);
if (OB_FAIL(task_arg.get_task_ops().get_copy_assign_ret())) {
LOG_WARN("failed to copy das task", K(ret));
} else if (task_arg.is_local_task()) {
if (OB_FAIL(do_local_das_task(das_ref, task_arg))) {
LOG_WARN("do local das task failed", K(ret));
common::ObIArray<ObIDASTaskOp *> &task_list = task_arg.get_task_ops();
if (OB_FAIL(task_ops.get_aggregated_tasks(task_list))) {
LOG_WARN("fail to get agg tasks", K(ret));
} else if (task_list.empty()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected empty task_list", K(ret));
} else if (task_arg.is_local_task()) {
if (OB_FAIL(do_local_das_task(task_list))) {
LOG_WARN("do local das task failed", K(ret), K(task_arg));
}
} else {
if (async) {
if (OB_FAIL(das_ref.get_das_ref_count_ctx().acquire_task_execution_resource(timeout_ts))) {
LOG_WARN("failed to acquire execution resource", K(ret));
} else if (OB_FAIL(do_async_remote_das_task(das_ref, task_ops, task_arg, THIS_WORKER.get_group_id()))) {
das_ref.get_das_ref_count_ctx().inc_concurrency_limit();
LOG_WARN("do remote das task failed", K(ret));
}
} else if (OB_FAIL(das_ref.acquire_task_execution_resource())) {
LOG_WARN("failed to acquire execution resource", K(ret));
} else {
if (async) {
if (OB_FAIL(do_async_remote_das_task(das_ref, task_ops, task_arg))) {
das_ref.inc_concurrency_limit();
LOG_WARN("do remote das task failed", K(ret));
}
} else {
if (OB_FAIL(do_sync_remote_das_task(das_ref, task_ops, task_arg))) {
LOG_WARN("do remote das task failed", K(ret));
}
if (OB_FAIL(do_sync_remote_das_task(das_ref, task_ops, task_arg))) {
LOG_WARN("do remote das task failed", K(ret));
}
}
task_arg.get_task_ops().reuse();
}
return ret;
}
@ -244,7 +241,7 @@ int ObDataAccessService::retry_das_task(ObDASRef &das_ref, ObIDASTaskOp &task_op
{
int ret = task_op.errcode_;
ObArenaAllocator tmp_alloc;
ObDasAggregatedTasks das_task_wrapper(tmp_alloc);
ObDasAggregatedTask das_task_wrapper(tmp_alloc);
bool retry_continue = false;
ObDASLocationRouter &location_router = DAS_CTX(das_ref.get_exec_ctx()).get_location_router();
location_router.reset_cur_retry_cnt();
@ -354,7 +351,7 @@ int ObDataAccessService::rescan_das_task(ObDASRef &das_ref, ObDASScanOp &scan_op
FLTSpanGuard(rescan_das_task);
ObArenaAllocator tmp_alloc;
ObDasAggregatedTasks das_task_wrapper(tmp_alloc);
ObDasAggregatedTask das_task_wrapper(tmp_alloc);
if (scan_op.is_local_task()) {
if (OB_FAIL(scan_op.rescan())) {
LOG_WARN("rescan das task failed", K(ret));
@ -375,24 +372,21 @@ int ObDataAccessService::rescan_das_task(ObDASRef &das_ref, ObDASScanOp &scan_op
return ret;
}
int ObDataAccessService::do_local_das_task(ObDASRef &das_ref,
ObDASTaskArg &task_arg) {
UNUSED(das_ref);
int ObDataAccessService::do_local_das_task(ObIArray<ObIDASTaskOp*> &task_list) {
int ret = OB_SUCCESS;
FLTSpanGuard(do_local_das_task);
const common::ObSEArray<ObIDASTaskOp*, 2> &task_ops = task_arg.get_task_ops();
LOG_DEBUG("begin to do local das task", K(task_arg));
for (int64_t i = 0; OB_SUCC(ret) && i < task_ops.count(); i++) {
if (OB_FAIL(task_ops.at(i)->start_das_task())) {
LOG_DEBUG("begin to do local das task", K(task_list));
for (int64_t i = 0; OB_SUCC(ret) && i < task_list.count(); i++) {
if (OB_FAIL(task_list.at(i)->start_das_task())) {
LOG_WARN("start local das task failed", K(ret));
int tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(task_ops.at(i)->state_advance())) {
if (OB_TMP_FAIL(task_list.at(i)->state_advance())) {
LOG_WARN("failed to advance das task state.",K(ret));
}
break;
} else {
if (OB_FAIL(task_ops.at(i)->state_advance())) {
if (OB_FAIL(task_list.at(i)->state_advance())) {
LOG_WARN("failed to advance das task state.",K(ret));
}
}
@ -400,11 +394,14 @@ int ObDataAccessService::do_local_das_task(ObDASRef &das_ref,
return ret;
}
int ObDataAccessService::do_async_remote_das_task(
ObDASRef &das_ref, ObDasAggregatedTasks &aggregated_tasks,
ObDASTaskArg &task_arg) {
int ObDataAccessService::do_async_remote_das_task(ObDASRef &das_ref,
ObDasAggregatedTask &aggregated_tasks,
ObDASTaskArg &task_arg,
int32_t group_id) {
int ret = OB_SUCCESS;
void *resp_buf = nullptr;
ObIDASTaskResult *op_result = nullptr;
ObRpcDasAsyncAccessCallBack *das_async_cb = nullptr;
FLTSpanGuard(do_async_remote_das_task);
ObSQLSessionInfo *session = das_ref.get_exec_ctx().get_my_session();
ObPhysicalPlanCtx *plan_ctx = das_ref.get_exec_ctx().get_physical_plan_ctx();
@ -423,11 +420,6 @@ int ObDataAccessService::do_async_remote_das_task(
ObDASRemoteInfo remote_info;
remote_info.exec_ctx_ = &das_ref.get_exec_ctx();
remote_info.frame_info_ = das_ref.get_expr_frame_info();
remote_info.trans_desc_ = session->get_tx_desc();
if (OB_FAIL(remote_info.snapshot_.assign(*task_arg.get_task_op()->get_snapshot()))) {
LOG_WARN("assign snapshot fail", K(ret));
}
remote_info.need_tx_ = (remote_info.trans_desc_ != nullptr);
session->get_cur_sql_id(remote_info.sql_id_, sizeof(remote_info.sql_id_));
remote_info.user_id_ = session->get_user_id();
remote_info.session_id_ = session->get_sessid();
@ -438,16 +430,30 @@ int ObDataAccessService::do_async_remote_das_task(
}
task_arg.set_remote_info(&remote_info);
ObDASRemoteInfo::get_remote_info() = &remote_info;
ObIDASTaskResult *op_result = nullptr;
ObRpcDasAsyncAccessCallBack *das_async_cb = nullptr;
if (FAILEDx(das_ref.allocate_async_das_cb(das_async_cb, task_ops, timeout_ts))) {
if (das_ref.is_parallel_submit()) {
if (OB_ISNULL(das_ref.get_das_parallel_ctx().get_tx_desc_bak())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null ptr of tx_desc", K(ret));
} else {
remote_info.trans_desc_ = das_ref.get_das_parallel_ctx().get_tx_desc_bak();
}
} else {
remote_info.trans_desc_ = session->get_tx_desc();
}
remote_info.need_tx_ = (remote_info.trans_desc_ != nullptr);
if (OB_FAIL(ret)) {
// do nothing
} else if (OB_FAIL(remote_info.snapshot_.assign(*task_arg.get_task_op()->get_snapshot()))) {
LOG_WARN("assign snapshot fail", K(ret));
} else if (OB_FAIL(das_ref.allocate_async_das_cb(das_async_cb, task_ops, timeout_ts))) {
LOG_WARN("failed to allocate das async cb", K(ret));
}
// prepare op result in advance avoiding racing condition.
for (int64_t i = 0; OB_SUCC(ret) && i < task_ops.count(); i++) {
if (OB_UNLIKELY(ObDasTaskStatus::UNSTART != task_ops.at(i)->get_task_status())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("task status unexpected", KR(ret), K(task_ops.at(i)->get_task_status()), KPC(task_ops.at(i)));
LOG_WARN("task status unexpected", K(ret), K(task_ops.at(i)->get_task_status()), KPC(task_ops.at(i)));
} else if (NULL != (op_result = task_ops.at(i)->get_op_result())) {
if (OB_FAIL(op_result->reuse())) {
LOG_WARN("reuse task result failed", K(ret));
@ -470,10 +476,10 @@ int ObDataAccessService::do_async_remote_das_task(
}
}
}
LOG_DEBUG("begin to do remote das task", K(task_arg));
LOG_DEBUG("begin to do remote das task", K(task_ops));
if (OB_FAIL(ret)) {
// do nothing
} else if (OB_FAIL(collect_das_task_info(task_arg, remote_info))) {
} else if (OB_FAIL(collect_das_task_info(task_ops, remote_info))) {
LOG_WARN("collect das task info failed", K(ret));
} else if (OB_UNLIKELY(timeout <= 0)) {
ret = OB_TIMEOUT;
@ -483,7 +489,7 @@ int ObDataAccessService::do_async_remote_das_task(
.by(tenant_id)
.timeout(timeout)
.das_async_access(task_arg, das_async_cb))) {
LOG_WARN("rpc remote sync access failed", K(ret), K(task_arg));
LOG_WARN("rpc remote sync access failed", K(ret), K(task_arg), K(group_id));
// RPC fail, add task's LSID to trans_result
// indicate some transaction participant may touched
for (int i = 0; i < task_ops.count(); i++) {
@ -507,7 +513,7 @@ int ObDataAccessService::do_async_remote_das_task(
}
int ObDataAccessService::do_sync_remote_das_task(
ObDASRef &das_ref, ObDasAggregatedTasks &aggregated_tasks,
ObDASRef &das_ref, ObDasAggregatedTask &aggregated_tasks,
ObDASTaskArg &task_arg) {
int ret = OB_SUCCESS;
void *resp_buf = nullptr;
@ -522,20 +528,31 @@ int ObDataAccessService::do_sync_remote_das_task(
ObDASRemoteInfo remote_info;
remote_info.exec_ctx_ = &das_ref.get_exec_ctx();
remote_info.frame_info_ = das_ref.get_expr_frame_info();
remote_info.trans_desc_ = session->get_tx_desc();
if (OB_FAIL(remote_info.snapshot_.assign(*task_arg.get_task_op()->get_snapshot()))) {
LOG_WARN("assign snapshot fail", K(ret));
}
remote_info.need_tx_ = (remote_info.trans_desc_ != nullptr);
session->get_cur_sql_id(remote_info.sql_id_, sizeof(remote_info.sql_id_));
remote_info.user_id_ = session->get_user_id();
remote_info.session_id_ = session->get_sessid();
remote_info.plan_id_ = session->get_current_plan_id();
if (das_ref.is_parallel_submit()) {
if (OB_ISNULL(das_ref.get_das_parallel_ctx().get_tx_desc_bak())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null ptr of tx_desc", K(ret));
} else {
remote_info.trans_desc_ = das_ref.get_das_parallel_ctx().get_tx_desc_bak();
}
} else {
remote_info.trans_desc_ = session->get_tx_desc();
}
remote_info.need_tx_ = (remote_info.trans_desc_ != nullptr);
if (OB_NOT_NULL(plan_ctx->get_phy_plan())) {
remote_info.plan_id_ = plan_ctx->get_phy_plan()->get_plan_id();
remote_info.plan_hash_ = plan_ctx->get_phy_plan()->get_plan_hash_value();
remote_info.need_subschema_ctx_ = plan_ctx->is_subschema_ctx_inited();
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(remote_info.snapshot_.assign(*task_arg.get_task_op()->get_snapshot()))) {
LOG_WARN("assign snapshot fail", K(ret));
}
task_arg.set_remote_info(&remote_info);
ObDASRemoteInfo::get_remote_info() = &remote_info;
@ -573,7 +590,7 @@ int ObDataAccessService::do_sync_remote_das_task(
}
if (OB_FAIL(ret)) {
// do nothing
} else if (OB_FAIL(collect_das_task_info(task_arg, remote_info))) {
} else if (OB_FAIL(collect_das_task_info(task_ops, remote_info))) {
LOG_WARN("collect das task info failed", K(ret));
} else if (OB_UNLIKELY(timeout <= 0)) {
ret = OB_TIMEOUT;
@ -603,7 +620,6 @@ int ObDataAccessService::do_sync_remote_das_task(
LOG_WARN("failed to process task resp", K(ret), K(task_arg), K(task_resp));
}
}
das_ref.inc_concurrency_limit();
return ret;
}
@ -709,10 +725,83 @@ int ObDataAccessService::process_task_resp(ObDASRef &das_ref, const ObDASTaskRes
return ret;
}
int ObDataAccessService::collect_das_task_info(ObDASTaskArg &task_arg, ObDASRemoteInfo &remote_info)
int ObDataAccessService::push_parallel_task(ObDASRef &das_ref, ObDasAggregatedTask &agg_task, int32_t group_id)
{
int ret = OB_SUCCESS;
ObDASParallelTask *task = nullptr;
omt::ObMultiTenant *omt = GCTX.omt_;
DISABLE_SQL_MEMLEAK_GUARD;
if (NULL == omt) {
ret = OB_ERR_UNEXPECTED;
TRANS_LOG(ERROR, "unexpected error, omt is null", KR(ret), KP(omt));
} else if (OB_ISNULL(task = ObDASParallelTaskFactory::alloc(das_ref.get_das_ref_count_ctx()))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("alloc memory failed", K(ret));
} else if (OB_FAIL(task->init(&agg_task, group_id))) {
LOG_WARN("init parallel task failed", K(ret), K(agg_task));
} else if (OB_FAIL(omt->recv_request(MTL_ID(), *task))) {
LOG_WARN("fail to push parallel_das_task", K(ret), KPC(task));
} else {
LOG_TRACE("push parallel task succeed", K(agg_task));
}
if (OB_FAIL(ret)) {
ObDASParallelTaskFactory::free(task);
task = NULL;
}
return ret;
}
int ObDataAccessService::parallel_execute_das_task(common::ObIArray<ObIDASTaskOp *> &task_list)
{
int ret = OB_SUCCESS;
if (task_list.empty()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected empty task_list", K(ret));
} else if (OB_FAIL(do_local_das_task(task_list))) {
LOG_WARN("failed to do local das task", K(ret), K(task_list));
}
return ret;
}
int ObDataAccessService::parallel_submit_das_task(ObDASRef &das_ref, ObDasAggregatedTask &agg_task)
{
int ret = OB_SUCCESS;
ObSQLSessionInfo *session = das_ref.get_exec_ctx().get_my_session();
int64_t timeout_ts = session->get_query_timeout_ts();
int32_t group_id = THIS_WORKER.get_group_id();
int32_t das_group_id = group_id | das::OB_DAS_PARALLEL_POOL_MARK;
LOG_TRACE("print group_id", K(group_id), K(das::OB_DAS_PARALLEL_POOL_MARK), K(das_group_id));
if (agg_task.server_ == ctrl_addr_) {
// 并发提交 local das 任务
ObDASParallelTask *task = nullptr;
omt::ObMultiTenant *omt = GCTX.omt_;
if (OB_FAIL(das_ref.get_das_ref_count_ctx().acquire_task_execution_resource(timeout_ts))) {
LOG_WARN("fail to acquire resource", K(ret));
} else if (OB_FAIL(push_parallel_task(das_ref, agg_task, das_group_id))) {
// NOTICE: if error occur, must release the reference count
das_ref.get_das_ref_count_ctx().inc_concurrency_limit();
LOG_WARN("fail to push parallel task", K(ret));
}
} else {
ObDASTaskArg task_arg;
ObExecContext &exec_ctx = das_ref.get_exec_ctx();
ObSQLSessionInfo *session = exec_ctx.get_my_session();
task_arg.set_timeout_ts(session->get_query_timeout_ts());
task_arg.set_ctrl_svr(ctrl_addr_);
task_arg.get_runner_svr() = agg_task.server_;
if (OB_FAIL(agg_task.get_aggregated_tasks(task_arg.get_task_ops()))) {
LOG_WARN("failed to get das task groups", K(ret));
} else if (OB_FAIL(das_ref.get_das_ref_count_ctx().acquire_task_execution_resource(timeout_ts))) {
LOG_WARN("fail to acquire resource", K(ret));
} else if (OB_FAIL(do_async_remote_das_task(das_ref, agg_task, task_arg, das_group_id))) {
das_ref.get_das_ref_count_ctx().inc_concurrency_limit();
LOG_WARN("do remote das task failed", K(ret), K(task_arg));
}
}
return ret;
}
int ObDataAccessService::collect_das_task_info(ObIArray<ObIDASTaskOp*> &task_ops, ObDASRemoteInfo &remote_info)
{
int ret = OB_SUCCESS;
common::ObSEArray<ObIDASTaskOp*, 2> &task_ops = task_arg.get_task_ops();
ObIDASTaskOp *task_op = nullptr;
for (int i = 0; OB_SUCC(ret) && i < task_ops.count(); i++) {
task_op = task_ops.at(i);

20
src/sql/das/ob_data_access_service.h
View File

@ -37,7 +37,7 @@ public:
const common::ObAddr &self_addr);
//开启DAS Task分区相关的事务控制，并执行task对应的op
int execute_das_task(ObDASRef &das_ref,
ObDasAggregatedTasks &task_ops, bool async = true);
ObDasAggregatedTask &task_ops, bool async = true);
//关闭DAS Task的执行流程，并释放task持有的资源，并结束相关的事务控制
int end_das_task(ObDASRef &das_ref, ObIDASTaskOp &task_op);
int get_das_task_id(int64_t &das_id);
@ -54,18 +54,24 @@ public:
const ObIDASTaskOp *task_op,
ObDASExtraData *&extra_result);
int process_task_resp(ObDASRef &das_ref, const ObDASTaskResp &task_resp, const common::ObSEArray<ObIDASTaskOp*, 2> &task_ops);
int parallel_execute_das_task(common::ObIArray<ObIDASTaskOp *> &task_list);
int parallel_submit_das_task(ObDASRef &das_ref, ObDasAggregatedTask &agg_task);
int push_parallel_task(ObDASRef &das_ref, ObDasAggregatedTask &agg_task, int32_t group_id);
int collect_das_task_info(ObIArray<ObIDASTaskOp*> &task_list, ObDASRemoteInfo &remote_info);
private:
int execute_dist_das_task(ObDASRef &das_ref,
ObDasAggregatedTasks &task_ops, bool async = true);
ObDasAggregatedTask &task_ops, bool async = true);
int clear_task_exec_env(ObDASRef &das_ref, ObIDASTaskOp &task_op);
int refresh_task_location_info(ObDASRef &das_ref, ObIDASTaskOp &task_op);
int do_local_das_task(ObDASRef &das_ref, ObDASTaskArg &task_arg);
int do_async_remote_das_task(ObDASRef &das_ref, ObDasAggregatedTasks &aggregated_tasks, ObDASTaskArg &task_arg);
int do_sync_remote_das_task(ObDASRef &das_ref, ObDasAggregatedTasks &aggregated_tasks, ObDASTaskArg &task_arg);
int collect_das_task_info(ObDASTaskArg &task_arg, ObDASRemoteInfo &remote_info);
int do_local_das_task(ObIArray<ObIDASTaskOp*> &task_list);
int do_async_remote_das_task(ObDASRef &das_ref,
ObDasAggregatedTask &aggregated_tasks,
ObDASTaskArg &task_arg,
int32_t group_id);
int do_sync_remote_das_task(ObDASRef &das_ref, ObDasAggregatedTask &aggregated_tasks, ObDASTaskArg &task_arg);
void calc_das_task_parallelism(const ObDASRef &das_ref, const ObDasAggregatedTask &task_ops, int &target_parallelism);
int collect_das_task_attach_info(ObDASRemoteInfo &remote_info,
ObDASBaseRtDef *attach_rtdef);
void calc_das_task_parallelism(const ObDASRef &das_ref, const ObDasAggregatedTasks &task_ops, int &target_parallelism);
private:
obrpc::ObDASRpcProxy das_rpc_proxy_;
common::ObAddr ctrl_addr_;

3
src/sql/engine/cmd/ob_variable_set_executor.cpp
View File

@ -1369,6 +1369,9 @@ int ObVariableSetExecutor::is_support(const share::ObSetVar &set_var)
(SYS_VAR_INNODB_READ_ONLY <= var_id && SYS_VAR_SUPER_READ_ONLY >= var_id)) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("This variable not support, just mock", K(set_var.var_name_), K(var_id), K(ret));
} else if (SYS_VAR_LOW_PRIORITY_UPDATES <= var_id && SYS_VAR_MAX_INSERT_DELAYED_THREADS >= var_id) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("This variable not support, just mock", K(set_var.var_name_), K(var_id), K(ret));
}
return ret;
}

185
src/sql/engine/dml/ob_conflict_checker.cpp
View File

@ -29,6 +29,17 @@ using namespace share::schema;
namespace sql
{
int ObTabletSnapshotMaping::assign(const ObTabletSnapshotMaping& other)
{
int ret = OB_SUCCESS;
this->ls_id_ = other.ls_id_;
this->tablet_id_ = other.tablet_id_;
if (OB_FAIL(this->snapshot_.assign(other.snapshot_))) {
LOG_WARN("assign snapshot fail", K(ret), K(other));
}
return ret;
}
OB_SERIALIZE_MEMBER(ObRowkeyCstCtdef,
constraint_name_,
rowkey_expr_,
@ -115,6 +126,12 @@ OB_DEF_SERIALIZE_SIZE(ObConflictCheckerCtdef)
return len;
}
bool ObTabletSnapshotMaping::operator==(const ObTabletSnapshotMaping &other) const
{
return ls_id_ == other.ls_id_ && tablet_id_ == other.tablet_id_;
}
bool ObConflictValue::operator==(const ObConflictValue &other) const
{
return current_datum_row_ == other.current_datum_row_;
@ -178,10 +195,38 @@ ObConflictChecker::ObConflictChecker(common::ObIAllocator &allocator,
das_ref_(eval_ctx, eval_ctx.exec_ctx_),
local_tablet_loc_(nullptr),
table_loc_(nullptr),
tmp_mem_ctx_()
tmp_mem_ctx_(),
se_rowkey_dist_ctx_(nullptr),
snapshot_maping_()
{
}
int ObConflictChecker::create_rowkey_check_hashset(int64_t replace_row_cnt)
{
int ret = OB_SUCCESS;
SeRowkeyDistCtx *rowkey_dist_ctx = nullptr;
if (OB_ISNULL(se_rowkey_dist_ctx_)) {
void *buf = allocator_.alloc(sizeof(SeRowkeyDistCtx));
if (OB_ISNULL(buf)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("allocate memory failed", K(ret), "size", sizeof(SeRowkeyDistCtx));
} else {
rowkey_dist_ctx = new (buf) SeRowkeyDistCtx();
const int64_t max_bucket_num = replace_row_cnt > MAX_ROWKEY_CHECKER_DISTINCT_BUCKET_NUM ?
MAX_ROWKEY_CHECKER_DISTINCT_BUCKET_NUM : replace_row_cnt;
if (OB_FAIL(rowkey_dist_ctx->create(max_bucket_num,
"DmlConflictDisBu",
"DmlConflictDisNo",
MTL_ID()))) {
LOG_WARN("create rowkey distinct context failed", K(ret), "rows", replace_row_cnt, K(max_bucket_num));
} else {
se_rowkey_dist_ctx_ = rowkey_dist_ctx;
}
}
}
return ret;
}
//初始conflict_map
int ObConflictChecker::create_conflict_map(int64_t replace_row_cnt)
{
@ -199,7 +244,9 @@ int ObConflictChecker::create_conflict_map(int64_t replace_row_cnt)
}
//初始化map array， map创建hash_bucket将会在延后
int ObConflictChecker::init_conflict_checker(const ObExprFrameInfo *expr_frame_info, ObDASTableLoc *table_loc)
int ObConflictChecker::init_conflict_checker(const ObExprFrameInfo *expr_frame_info,
ObDASTableLoc *table_loc,
bool use_partition_gts_opt)
{
int ret = OB_SUCCESS;
ObSQLSessionInfo *session = eval_ctx_.exec_ctx_.get_my_session();
@ -214,6 +261,7 @@ int ObConflictChecker::init_conflict_checker(const ObExprFrameInfo *expr_frame_i
// 这里需要注意
das_ref_.set_execute_directly(!checker_ctdef_.use_dist_das_);
das_ref_.set_mem_attr(mem_attr);
das_ref_.set_do_gts_opt(use_partition_gts_opt);
}
OZ(init_das_scan_rtdef());
return ret;
@ -664,6 +712,10 @@ int ObConflictChecker::reuse()
if (tmp_mem_ctx_ != nullptr) {
tmp_mem_ctx_->reset_remain_one_page();
}
if (se_rowkey_dist_ctx_ != nullptr) {
se_rowkey_dist_ctx_->reuse();
}
snapshot_maping_.reuse();
return ret;
}
@ -682,6 +734,39 @@ int ObConflictChecker::destroy()
DESTROY_CONTEXT(tmp_mem_ctx_);
tmp_mem_ctx_ = nullptr;
}
if (se_rowkey_dist_ctx_ != nullptr) {
se_rowkey_dist_ctx_->destroy();
se_rowkey_dist_ctx_ = nullptr;
}
snapshot_maping_.reset();
return ret;
}
int ObConflictChecker::add_lookup_range_no_dup(storage::ObTableScanParam &scan_param, ObNewRange &lookup_range)
{
int ret = OB_SUCCESS;
if (OB_NOT_NULL(se_rowkey_dist_ctx_)) {
const ObRowkey &tmp_table_rowkey = lookup_range.get_start_key();;
ret = se_rowkey_dist_ctx_->exist_refactored(tmp_table_rowkey);
if (OB_HASH_EXIST == ret) {
ret = OB_SUCCESS;
} else if (OB_HASH_NOT_EXIST == ret) {
//step3: if not exist, deep copy data and add ObRowkey to hash set
//step3.1: Init the buffer of ObObj Array
ret = OB_SUCCESS;
if (OB_FAIL(se_rowkey_dist_ctx_->set_refactored(tmp_table_rowkey))) {
LOG_WARN("set rowkey item failed", K(ret));
} else if (OB_FAIL(scan_param.key_ranges_.push_back(lookup_range))) {
LOG_WARN("push_back lookup_range failed", K(ret), K(lookup_range));
} else {
LOG_TRACE("print add lookup_range succ", K(lookup_range));
}
} else {
LOG_WARN("check if rowkey item exists failed", K(ret));
}
} else if (OB_FAIL(add_var_to_array_no_dup(scan_param.key_ranges_, lookup_range))) {
LOG_WARN("store lookup key range failed", K(ret), K(lookup_range), K(scan_param));
}
return ret;
}
@ -705,7 +790,7 @@ int ObConflictChecker::build_primary_table_lookup_das_task()
storage::ObTableScanParam &scan_param = das_scan_op->get_scan_param();
if (OB_FAIL(build_data_table_range(lookup_range))) {
LOG_WARN("build data table range failed", K(ret), KPC(tablet_loc));
} else if (OB_FAIL(add_var_to_array_no_dup(scan_param.key_ranges_, lookup_range))) {
} else if (OB_FAIL(add_lookup_range_no_dup(scan_param, lookup_range))) {
LOG_WARN("store lookup key range failed", K(ret), K(lookup_range), K(scan_param));
} else {
LOG_TRACE("after build conflict rowkey", K(scan_param.tablet_id_),
@ -829,8 +914,9 @@ int ObConflictChecker::build_data_table_range(ObNewRange &lookup_range)
int ObConflictChecker::do_lookup_and_build_base_map(int64_t replace_row_cnt)
{
int ret = OB_SUCCESS;
NG_TRACE_TIMES(2, start_fetch_conflict_row);
const ExprFixedArray &storage_output = checker_ctdef_.das_scan_ctdef_.pd_expr_spec_.access_exprs_;
if (OB_FAIL(das_ref_.execute_all_task())) {
if (OB_FAIL(post_all_das_scan_tasks())) {
LOG_WARN("execute all delete das task failed", K(ret));
} else {
DASOpResultIter result_iter = das_ref_.begin_result_iter();
@ -1101,5 +1187,96 @@ int ObConflictChecker::attach_related_taskinfo(ObDASScanOp &target_op, ObDASBase
}
return ret;
}
int ObConflictChecker::collect_all_snapshot(transaction::ObTxReadSnapshot &snapshot, const ObDASTabletLoc *tablet_loc)
{
int ret = OB_SUCCESS;
ObTabletSnapshotMaping tablet_snapshot_maping;
tablet_snapshot_maping.tablet_id_ = tablet_loc->tablet_id_;
tablet_snapshot_maping.ls_id_ = tablet_loc->ls_id_;
if (OB_ISNULL(tablet_loc)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null", K(ret));
} else if (FALSE_IT(tablet_snapshot_maping.tablet_id_ = tablet_loc->tablet_id_)) {
// do nothing
} else if (FALSE_IT(tablet_snapshot_maping.ls_id_ = tablet_loc->ls_id_)) {
// do nothing
} else if (OB_FAIL(tablet_snapshot_maping.snapshot_.assign(snapshot))) {
LOG_WARN("fail to assign snapshot", K(ret), K(snapshot));
} else if (OB_FAIL(has_exist_in_array(snapshot_maping_, tablet_snapshot_maping))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected duplicate snapshot_version", K(ret), K(tablet_snapshot_maping), K(snapshot_maping_));
} else if (OB_FAIL(snapshot_maping_.push_back(tablet_snapshot_maping))) {
LOG_WARN("fail to push back snapshot", K(ret), K(tablet_snapshot_maping));
} else {
LOG_TRACE("collect snaoshot after try_insert", K(tablet_snapshot_maping));
}
return ret;
}
int ObConflictChecker::get_snapshot_by_ids(ObTabletID tablet_id, share::ObLSID ls_id, bool &founded, transaction::ObTxReadSnapshot &snapshot)
{
int ret = OB_SUCCESS;
founded = false;
for (int64_t i = 0; OB_SUCC(ret) && !founded && i < snapshot_maping_.count(); i++) {
ObTabletSnapshotMaping &snapshot_maping = snapshot_maping_.at(i);
if (snapshot_maping.tablet_id_ == tablet_id && snapshot_maping.ls_id_ == ls_id) {
if (OB_FAIL(snapshot.assign(snapshot_maping.snapshot_))) {
LOG_WARN("fail to assign snapshot", K(ret));
} else {
founded = true;
}
}
}
return ret;
}
int ObConflictChecker::post_all_das_scan_tasks()
{
int ret = OB_SUCCESS;
if (OB_FAIL(set_partition_snapshot_for_das_task(das_ref_))) {
LOG_WARN("fail to set partition snapshot", K(ret));
} else if (OB_FAIL(das_ref_.execute_all_task())) {
LOG_WARN("execute all delete das task failed", K(ret));
}
return ret;
}
int ObConflictChecker::set_partition_snapshot_for_das_task(ObDASRef &das_ref)
{
int ret = OB_SUCCESS;
if (!das_ref.is_do_gts_opt()) {
// do nothing
} else {
DASTaskIter task_iter = das_ref.begin_task_iter();
while (OB_SUCC(ret) && !task_iter.is_end()) {
bool founded = false;
transaction::ObTxReadSnapshot *snapshot = nullptr;;
ObIDASTaskOp *das_op = *task_iter;
if (OB_ISNULL(das_op)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null", K(ret));
} else if (OB_ISNULL(snapshot = das_op->get_das_gts_opt_info().get_specify_snapshot())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null", K(ret), KPC(snapshot));
} else if (OB_FAIL(get_snapshot_by_ids(das_op->get_tablet_id(), das_op->get_ls_id(), founded, *snapshot))) {
LOG_WARN("fail to get snapshot by ids", K(das_op->get_tablet_id()), K(das_op->get_ls_id()), K(snapshot_maping_));
} else if (!founded) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("fail to found snapshot", K(ret), K(das_op->get_tablet_id()), K(das_op->get_type()), K(das_op->get_ls_id()));
} else if (!snapshot->is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("snapshot is invalid", K(ret), K(das_op->get_tablet_id()), K(das_op->get_type()));
} else {
LOG_TRACE("set specify snapshot for current das_task", KPC(snapshot),
K(das_op->get_tablet_id()), K(das_op->get_ls_id()), K(das_op->get_type()));
}
++task_iter;
}
}
return ret;
}
} // namespace sql
} // namespace oceanbase

37
src/sql/engine/dml/ob_conflict_checker.h
View File

@ -25,6 +25,25 @@ class ObTableModifyOp;
struct ObDASScanCtDef;
struct ObDASScanRtDef;
class ObTabletSnapshotMaping
{
public:
ObTabletSnapshotMaping()
: snapshot_(),
tablet_id_(),
ls_id_()
{}
~ObTabletSnapshotMaping() = default;
int assign(const ObTabletSnapshotMaping &other);
bool operator==(const ObTabletSnapshotMaping &other) const;
TO_STRING_KV(K_(snapshot), K_(tablet_id), K_(ls_id));
transaction::ObTxReadSnapshot snapshot_;
common::ObTabletID tablet_id_;
share::ObLSID ls_id_;
private:
DISALLOW_COPY_AND_ASSIGN(ObTabletSnapshotMaping);
};
struct ObRowkeyCstCtdef
{
OB_UNIS_VERSION_V(1);
@ -116,6 +135,7 @@ public:
{}
virtual ~ObConflictCheckerCtdef() = default;
TO_STRING_KV(K_(cst_ctdefs), K_(das_scan_ctdef), KPC_(calc_part_id_expr), K_(attach_spec));
static const int64_t MIN_ROW_COUNT_USE_HASHSET_DO_DISTICT = 50;
// must constraint_infos_.count() == conflict_map_array_.count()
// constraint_infos_ 用于生成ObConflictRowMap的key
ObRowkeyCstCtdefArray cst_ctdefs_;
@ -147,12 +167,15 @@ public:
//初始conflict_checker
int init_conflict_checker(const ObExprFrameInfo *expr_frame_info,
ObDASTableLoc *table_loc);
ObDASTableLoc *table_loc,
bool use_partition_gts_opt);
void set_local_tablet_loc(ObDASTabletLoc *tablet_loc) { local_tablet_loc_ = tablet_loc; }
//初始conflict_map
int create_conflict_map(int64_t replace_row_cnt);
int create_rowkey_check_hashset(int64_t replace_row_cnt);
// 检查当前的主键是否冲突
int check_duplicate_rowkey(const ObChunkDatumStore::StoredRow *replace_row,
ObIArray<ObConflictValue> &constraint_values,
@ -190,14 +213,23 @@ public:
// 向主表做回表，根据冲突行的主键，查询出所有对应主表的冲突行, 构建冲突map
int do_lookup_and_build_base_map(int64_t replace_row_cnt);
int post_all_das_scan_tasks();
// todo @kaizhan.dkz 构建回表的das scan task
int build_primary_table_lookup_das_task();
int add_lookup_range_no_dup(storage::ObTableScanParam &scan_param, ObNewRange &lookup_range);
//会被算子的inner_close函数调用
int close();
int reuse();
int collect_all_snapshot(transaction::ObTxReadSnapshot &snapshot, const ObDASTabletLoc *tablet_loc);
int get_snapshot_by_ids(ObTabletID tablet_id, share::ObLSID ls_id, bool &founded, transaction::ObTxReadSnapshot &snapshot);
int set_partition_snapshot_for_das_task(ObDASRef &das_ref);
int destroy();
private:
@ -226,6 +258,7 @@ private:
int get_tmp_string_buffer(common::ObIAllocator *&allocator);
public:
static const int64_t MAX_ROWKEY_CHECKER_DISTINCT_BUCKET_NUM = 1 * 128 * 1024;
common::ObArrayWrap<ObConflictRowMapCtx> conflict_map_array_;
ObEvalCtx &eval_ctx_; // 用于表达式的计算
const ObConflictCheckerCtdef &checker_ctdef_;
@ -240,6 +273,8 @@ public:
ObDASTabletLoc *local_tablet_loc_;
ObDASTableLoc *table_loc_;
lib::MemoryContext tmp_mem_ctx_;
SeRowkeyDistCtx *se_rowkey_dist_ctx_;
ObSEArray<ObTabletSnapshotMaping, 16> snapshot_maping_;
};
} // namespace sql
} // namespace oceanbase

31
src/sql/engine/dml/ob_dml_ctx_define.h
View File

@ -1015,13 +1015,16 @@ struct ObDMLRtCtx
: das_ref_(eval_ctx, exec_ctx),
das_task_status_(),
op_(op),
cached_row_size_(0)
exec_ctx_(exec_ctx),
das_task_memory_size_(0),
das_parallel_task_size_(0)
{ }
void reuse()
{
das_ref_.reuse();
cached_row_size_ = 0;
das_task_memory_size_ = 0;
das_parallel_task_size_ = 0;
}
void cleanup()
@ -1038,13 +1041,31 @@ struct ObDMLRtCtx
{ return das_task_status_.need_pick_del_task_first(); }
bool need_non_sub_full_task()
{ return das_task_status_.need_non_sub_full_task(); }
void add_cached_row_size(const int64_t row_size) { cached_row_size_ += row_size; }
int64_t get_row_buffer_size() const { return cached_row_size_; }
void add_das_task_memory_size(const int64_t row_size) { das_task_memory_size_ += row_size; }
int64_t get_das_task_memory_size() const { return das_task_memory_size_; }
void add_das_parallel_task_size(const int64_t row_size) { das_parallel_task_size_ += row_size; }
int64_t get_das_parallel_task_size() const { return das_parallel_task_size_; }
bool need_submit_all_tasks()
{
bool bret = false;
int64_t simulate_buffer_size = - EVENT_CALL(EventTable::EN_DAS_DML_BUFFER_OVERFLOW);
int64_t buffer_size_limit = is_meta_tenant(MTL_ID()) ? das::OB_DAS_MAX_META_TENANT_PACKET_SIZE : das::OB_DAS_MAX_TOTAL_PACKET_SIZE;
if (OB_UNLIKELY(simulate_buffer_size > 0)) {
buffer_size_limit = simulate_buffer_size;
}
if ((das_task_memory_size_ - das_parallel_task_size_) >= buffer_size_limit) {
bret = true;
}
return bret;
}
ObDASRef das_ref_;
DasTaskStatus das_task_status_;
ObTableModifyOp &op_;
int64_t cached_row_size_;
ObExecContext &exec_ctx_;
int64_t das_task_memory_size_;
int64_t das_parallel_task_size_;
};
template <typename T>

162
src/sql/engine/dml/ob_dml_service.cpp
View File

@ -1093,6 +1093,9 @@ int ObDMLService::update_row(const ObUpdCtDef &upd_ctdef,
}
} else if (OB_UNLIKELY(old_tablet_loc != new_tablet_loc)) {
//the updated row may be moved across partitions
if (dml_rtctx.das_ref_.get_parallel_type() == DAS_STREAMING_PARALLEL) {
dml_rtctx.das_ref_.get_das_parallel_ctx().set_das_parallel_type(DAS_BLOCKING_PARALLEL);
}
if (upd_ctdef.dupd_ctdef_.is_ignore_) {
ret = OB_NOT_SUPPORTED;
LOG_USER_ERROR(OB_NOT_SUPPORTED, "Cross-partition update ignore");
@ -1711,6 +1714,64 @@ int ObDMLService::init_lock_rtdef(ObDMLRtCtx &dml_rtctx,
return init_das_dml_rtdef(dml_rtctx, lock_ctdef.das_ctdef_, lock_rtdef.das_rtdef_, loc_meta);
}
int ObDMLService::check_agg_task_state(ObDMLRtCtx &dml_rtctx, ObIDASTaskOp *das_op, int64_t row_size, bool &reach_mem_limit)
{
int ret = OB_SUCCESS;
ObDasAggregatedTask *agg_task = nullptr;
reach_mem_limit = false;
int64_t simulate_buffer_size = - EVENT_CALL(EventTable::EN_DAS_SIMULATE_AGG_TASK_BUFF_LIMIT);
int64_t buffer_size_limit = is_meta_tenant(MTL_ID()) ? das::OB_DAS_MAX_META_TENANT_PACKET_SIZE : das::OB_DAS_MAX_PACKET_SIZE;
if (OB_UNLIKELY(simulate_buffer_size > 0)) {
buffer_size_limit = simulate_buffer_size;
}
if (dml_rtctx.das_ref_.get_parallel_type() != DAS_SERIALIZATION) {
if (OB_ISNULL(agg_task = das_op->get_agg_task())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null ptr", K(ret), KPC(das_op));
} else if (FALSE_IT(agg_task->add_mem_used(row_size))) {
// do nothing
} else if (agg_task->get_mem_used() >= buffer_size_limit) {
agg_task->start_status_ = DAS_AGG_TASK_REACH_MEM_LIMIT;
reach_mem_limit = true;
LOG_TRACE("this agg_task buff is reach memory limit", K(agg_task->get_mem_used()), K(buffer_size_limit));
}
}
return ret;
}
int ObDMLService::parallel_submit_das_task(ObDMLRtCtx &dml_rtctx, ObDasAggregatedTask *agg_task)
{
int ret = OB_SUCCESS;
if (dml_rtctx.das_ref_.get_parallel_type() == DAS_SERIALIZATION) {
// not support parallel_submit
LOG_TRACE("can't parallel submit", K(dml_rtctx.das_ref_.get_parallel_type()));
} else if (dml_rtctx.das_ref_.get_parallel_type() == DAS_STREAMING_PARALLEL) {
if (dml_rtctx.das_ref_.get_submitted_task_count() >= dml_rtctx.das_ref_.get_das_dop()) {
LOG_INFO("submitted tasks reach dop", "submitted_cnt", dml_rtctx.das_ref_.get_submitted_task_count(),
"das_dop", dml_rtctx.das_ref_.get_das_dop());
} else if (OB_FAIL(dml_rtctx.das_ref_.parallel_submit_agg_task(agg_task))) {
LOG_WARN("fail to parallel exec this agg_task", K(ret));
} else {
dml_rtctx.add_das_parallel_task_size(agg_task->get_mem_used());
dml_rtctx.das_ref_.get_das_parallel_ctx().add_submitted_task_count(1);
LOG_TRACE("succeed parallel submit one agg_task", K(agg_task), K(agg_task->get_start_status()),
K(dml_rtctx.get_das_task_memory_size()), K(dml_rtctx.get_das_parallel_task_size()));
}
} else if (dml_rtctx.das_ref_.get_parallel_type() == DAS_BLOCKING_PARALLEL) {
// 阻塞性提交，暂时不真正提交，但是会等到足够并行度的task数量才会真正全量提交
if (dml_rtctx.das_ref_.get_submitted_task_count() >= dml_rtctx.das_ref_.get_das_dop()) {
LOG_INFO("submitted tasks reach dop", "submitted_cnt", dml_rtctx.das_ref_.get_submitted_task_count(),
"das_dop", dml_rtctx.das_ref_.get_das_dop());
} else {
dml_rtctx.add_das_parallel_task_size(agg_task->get_mem_used());
dml_rtctx.das_ref_.get_das_parallel_ctx().add_submitted_task_count(1);
LOG_TRACE("block parallel submit one agg_task", K(agg_task), K(agg_task->get_start_status()),
K(dml_rtctx.get_das_task_memory_size()), K(dml_rtctx.get_das_parallel_task_size()));
}
}
return ret;
}
template <int N>
int ObDMLService::write_row_to_das_op(const ObDASDMLBaseCtDef &ctdef,
ObDASDMLBaseRtDef &rtdef,
@ -1722,6 +1783,7 @@ int ObDMLService::write_row_to_das_op(const ObDASDMLBaseCtDef &ctdef,
{
int ret = OB_SUCCESS;
bool need_retry = false;
bool reach_agg_mem_limit = false;
bool is_strict_defensive_check = trans_info_expr == nullptr ? false : true;
typedef typename das_reg::ObDASOpTypeTraits<N>::DASCtDef CtDefType;
typedef typename das_reg::ObDASOpTypeTraits<N>::DASRtDef RtDefType;
@ -1729,59 +1791,59 @@ int ObDMLService::write_row_to_das_op(const ObDASDMLBaseCtDef &ctdef,
OB_ASSERT(typeid(ctdef) == typeid(CtDefType));
OB_ASSERT(typeid(rtdef) == typeid(RtDefType));
int64_t extend_size = is_strict_defensive_check ?
ObDASWriteBuffer::DAS_WITH_TRANS_INFO_EXTEND_SIZE :
ObDASWriteBuffer::DAS_ROW_DEFAULT_EXTEND_SIZE;
do {
bool buffer_full = false;
need_retry = false;
ACTIVE_SESSION_RETRY_DIAG_INFO_SETTER(table_id_, ctdef.table_id_);
ACTIVE_SESSION_RETRY_DIAG_INFO_SETTER(table_schema_version_, ctdef.schema_version_);
//1. find das dml op
OpType *dml_op = nullptr;
if (OB_UNLIKELY(!dml_rtctx.das_ref_.has_das_op(tablet_loc, dml_op))) {
if (OB_FAIL(dml_rtctx.das_ref_.prepare_das_task(tablet_loc, dml_op))) {
LOG_WARN("prepare das task failed", K(ret), K(N));
} else if (OB_FAIL(dml_op->init_task_info(extend_size))) {
LOG_WARN("fail to init das write buff", K(ret), K(extend_size));
} else {
dml_op->set_das_ctdef(static_cast<const CtDefType*>(&ctdef));
dml_op->set_das_rtdef(static_cast<RtDefType*>(&rtdef));
rtdef.table_loc_->is_writing_ = true;
}
if (OB_SUCC(ret) &&
rtdef.related_ctdefs_ != nullptr && !rtdef.related_ctdefs_->empty()) {
if (OB_FAIL(add_related_index_info(*tablet_loc,
*rtdef.related_ctdefs_,
*rtdef.related_rtdefs_,
*dml_op))) {
LOG_WARN("add related index info failed", K(ret));
}
ObDASWriteBuffer::DAS_WITH_TRANS_INFO_EXTEND_SIZE : ObDASWriteBuffer::DAS_ROW_DEFAULT_EXTEND_SIZE;
ACTIVE_SESSION_RETRY_DIAG_INFO_SETTER(table_id_, ctdef.table_id_);
ACTIVE_SESSION_RETRY_DIAG_INFO_SETTER(table_schema_version_, ctdef.schema_version_);
// 1. find das dml op
OpType *dml_op = nullptr;
if (OB_UNLIKELY(!dml_rtctx.das_ref_.has_das_op(tablet_loc, dml_op))) {
if (OB_FAIL(dml_rtctx.das_ref_.prepare_das_task(tablet_loc, dml_op))) {
LOG_WARN("prepare das task failed", K(ret), K(N));
} else if (OB_FAIL(dml_op->init_task_info(extend_size))) {
LOG_WARN("fail to init das write buff", K(ret), K(extend_size));
} else {
dml_op->set_das_ctdef(static_cast<const CtDefType*>(&ctdef));
dml_op->set_das_rtdef(static_cast<RtDefType*>(&rtdef));
rtdef.table_loc_->is_writing_ = true;
}
if (OB_SUCC(ret) &&
rtdef.related_ctdefs_ != nullptr && !rtdef.related_ctdefs_->empty()) {
if (OB_FAIL(add_related_index_info(*tablet_loc,
*rtdef.related_ctdefs_,
*rtdef.related_rtdefs_,
*dml_op))) {
LOG_WARN("add related index info failed", K(ret));
}
}
//2. try add row to das dml buffer
if (OB_SUCC(ret)) {
if (OB_FAIL(dml_op->write_row(row, dml_rtctx.get_eval_ctx(), stored_row, buffer_full))) {
LOG_WARN("insert row to das dml op buffer failed", K(ret), K(ctdef), K(rtdef));
} else if (!buffer_full &&
OB_NOT_NULL(trans_info_expr) &&
OB_FAIL(ObDMLService::add_trans_info_datum(trans_info_expr, dml_rtctx.get_eval_ctx(), stored_row))) {
LOG_WARN("fail to add trans info datum", K(ret));
} else if (OB_NOT_NULL(stored_row)) {
dml_rtctx.add_cached_row_size(stored_row->row_size_);
LOG_DEBUG("write row to das op", K(ret), K(buffer_full),
"op_type", N, "table_id", ctdef.table_id_, "index_tid", ctdef.index_tid_,
"row", ROWEXPR2STR(dml_rtctx.get_eval_ctx(), row), "row_size", stored_row->row_size_);
}
}
// 2. add row to das dml buffer
if (OB_FAIL(ret)) {
// do nothing
} else if (OB_FAIL(dml_op->write_row(row, dml_rtctx.get_eval_ctx(), stored_row))) {
LOG_WARN("insert row to das dml op buffer failed", K(ret), K(ctdef), K(rtdef));
} else if (OB_ISNULL(stored_row)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null ptr", K(ret));
} else if (FALSE_IT(dml_rtctx.add_das_task_memory_size(stored_row->row_size_))) {
// do nothing
} else if (OB_NOT_NULL(trans_info_expr) &&
OB_FAIL(ObDMLService::add_trans_info_datum(trans_info_expr, dml_rtctx.get_eval_ctx(), stored_row))) {
LOG_WARN("fail to add trans info datum", K(ret));
} else if (dml_rtctx.das_ref_.get_parallel_type() != DAS_SERIALIZATION) {
// parallel submit
if (OB_FAIL(check_agg_task_state(dml_rtctx, dml_op, stored_row->row_size_, reach_agg_mem_limit))) {
LOG_WARN("fail to check agg_task state", K(ret), K(stored_row->row_size_));
} else if (!reach_agg_mem_limit) {
LOG_TRACE("not reach agg_task memory limit");
} else if (OB_FAIL(parallel_submit_das_task(dml_rtctx, dml_op->get_agg_task()))) {
LOG_WARN("fail to parallel submit this agg_task", K(ret));
} else {
dml_rtctx.das_ref_.clear_task_map();
}
//3. if buffer is full, frozen node, create a new das op to add row
if (OB_SUCC(ret) && buffer_full) {
need_retry = true;
if (REACH_COUNT_INTERVAL(10)) { // print log per 10 times.
LOG_INFO("DAS write buffer full, ", K(dml_op->get_row_cnt()), K(dml_rtctx.get_row_buffer_size()));
}
dml_rtctx.das_ref_.set_frozen_node();
}
} while (OB_SUCC(ret) && need_retry);
} else {
// serialize submit
}
return ret;
}

2
src/sql/engine/dml/ob_dml_service.h
View File

@ -255,6 +255,8 @@ public:
static int get_exec_ctx_for_duplicate_rowkey_check(ObExecContext *ctx, ObExecContext* &needed_ctx);
private:
static int check_agg_task_state(ObDMLRtCtx &dml_rtctx, ObIDASTaskOp *das_op, int64_t row_size, bool &reach_mem_limit);
static int parallel_submit_das_task(ObDMLRtCtx &dml_rtctx, ObDasAggregatedTask *agg_task);
template <int N>
static int write_row_to_das_op(const ObDASDMLBaseCtDef &ctdef,
ObDASDMLBaseRtDef &rtdef,

2
src/sql/engine/dml/ob_table_delete_op.cpp
View File

@ -289,7 +289,7 @@ OB_INLINE int ObTableDeleteOp::delete_row_to_das()
}
if (OB_SUCC(ret)) {
int64_t delete_rows = is_skipped ? 0 : 1;
if (OB_FAIL(merge_implict_cursor(0, 0, delete_rows, 0))) {
if (OB_FAIL(merge_implict_cursor(delete_rows, 0, 0, 0))) {
LOG_WARN("merge implict cursor failed", K(ret));
}
}

2
src/sql/engine/dml/ob_table_insert_all_op.cpp
View File

@ -136,7 +136,6 @@ int ObTableInsertAllOp::write_row_to_das_buffer()
{
int ret = OB_SUCCESS;
//int64_t savepoint_no = 0;
NG_TRACE(insert_start);
// first get next row from child operator
ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(ctx_);
// erro logging not support, fix it later
@ -213,7 +212,6 @@ int ObTableInsertAllOp::write_row_to_das_buffer()
if (OB_ERR_PRIMARY_KEY_DUPLICATE == ret) {
plan_ctx->set_last_insert_id_cur_stmt(0);
}
NG_TRACE(insert_end);
return ret;
}

2
src/sql/engine/dml/ob_table_insert_op.cpp
View File

@ -233,7 +233,6 @@ OB_INLINE int ObTableInsertOp::insert_row_to_das()
{
int ret = OB_SUCCESS;
transaction::ObTxSEQ savepoint_no;
NG_TRACE(insert_start);
// first get next row from child operator
ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(ctx_);
bool is_skipped = false;
@ -310,7 +309,6 @@ OB_INLINE int ObTableInsertOp::insert_row_to_das()
plan_ctx->record_last_insert_id_cur_stmt();
}
}
NG_TRACE(insert_end);
return ret;
}

253
src/sql/engine/dml/ob_table_insert_up_op.cpp
View File

@ -121,11 +121,17 @@ int ObTableInsertUpOp::check_need_exec_single_row()
return ret;
}
ObDasParallelType ObTableInsertUpOp::check_das_parallel_type()
{
return DAS_SERIALIZATION;
}
int ObTableInsertUpOp::inner_open()
{
int ret = OB_SUCCESS;
ObSQLSessionInfo *my_session = GET_MY_SESSION(ctx_);
NG_TRACE(replace_open);
NG_TRACE(insertup_open);
if (OB_FAIL(check_insert_up_ctdefs_valid())) {
LOG_WARN("check insert up ctdefs valid failed", K(ret));
} else if (OB_FAIL(ObTableModifyOp::inner_open())) {
@ -152,20 +158,33 @@ int ObTableInsertUpOp::inner_open()
return ret;
}
int ObTableInsertUpOp::inner_open_with_das()
{
int ret = OB_SUCCESS;
const ObExprFrameInfo *expr_frame_info = NULL;
ObDASTableLoc *table_loc = nullptr;
bool use_partition_gts_opt = false;
expr_frame_info = nullptr != MY_SPEC.expr_frame_info_
? MY_SPEC.expr_frame_info_
: &MY_SPEC.plan_->get_expr_frame_info();
if (ctx_.get_das_ctx().get_use_gts_opt()) {
gts_state_ = MY_SPEC.has_global_unique_index_ == true ?
WITH_UNIQUE_GLOBAL_INDEX_STATE : USE_PARTITION_SNAPSHOT_STATE;
if (gts_state_ == USE_PARTITION_SNAPSHOT_STATE) {
dml_rtctx_.das_ref_.set_do_gts_opt(true);
upd_rtctx_.das_ref_.set_do_gts_opt(true);
use_partition_gts_opt = true;
}
}
if (OB_FAIL(init_insert_up_rtdef())) {
LOG_WARN("init insert_up rtdef failed", K(ret), K(MY_SPEC.insert_up_ctdefs_.count()));
} else if (OB_ISNULL(table_loc = insert_up_rtdefs_.at(0).ins_rtdef_.das_rtdef_.table_loc_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("table location is nullptr", K(ret));
} else if (OB_FAIL(conflict_checker_.init_conflict_checker(expr_frame_info, table_loc))) {
} else if (OB_FAIL(conflict_checker_.init_conflict_checker(expr_frame_info,
table_loc,
use_partition_gts_opt))) {
LOG_WARN("init conflict_checker fail", K(ret));
} else {
// init update das_ref
@ -176,6 +195,13 @@ int ObTableInsertUpOp::inner_open_with_das()
upd_rtctx_.das_ref_.set_expr_frame_info(expr_frame_info);
upd_rtctx_.das_ref_.set_mem_attr(mem_attr);
upd_rtctx_.das_ref_.set_execute_directly(!MY_SPEC.use_dist_das_);
ObDasParallelType type = ObTableModifyOp::check_das_parallel_type();
if (DAS_SERIALIZATION != type) {
type = DAS_BLOCKING_PARALLEL;
LOG_TRACE("this sql use das parallel submit for insert_up", K(check_das_parallel_type()));
}
upd_rtctx_.das_ref_.get_das_parallel_ctx().set_das_parallel_type(type);
upd_rtctx_.das_ref_.get_das_parallel_ctx().set_das_dop(ctx_.get_das_ctx().get_real_das_dop());
}
return ret;
}
@ -220,7 +246,7 @@ int ObTableInsertUpOp::inner_close()
{
int ret = OB_SUCCESS;
int close_ret = OB_SUCCESS;
NG_TRACE(replace_inner_close);
NG_TRACE(insertup_inner_close);
if (OB_FAIL(conflict_checker_.close())) {
LOG_WARN("fail to close conflict_checker", K(ret));
}
@ -339,11 +365,14 @@ int ObTableInsertUpOp::do_insert_up_cache()
ObInsRtDef &ins_rtdef = insert_up_rtdef.ins_rtdef_;
ObUpdRtDef &upd_rtdef = insert_up_rtdef.upd_rtdef_;
NG_TRACE_TIMES(2, insert_up_start_shuff);
NG_TRACE_TIMES(2, insertup_start_shuff);
if (OB_FAIL(insert_up_row_store_.begin(insert_row_iter))) {
LOG_WARN("fail to get insert_up_row_store begin iter", K(ret));
}
while (OB_SUCC(ret) && OB_SUCC(insert_row_iter.get_next_row(insert_row))) {
int64_t insert_rows = 0;
int64_t update_rows = 0;
int64_t found_rows = 0;
constraint_values.reuse();
if (OB_ISNULL(insert_row)) {
ret = OB_ERR_UNEXPECTED;
@ -374,10 +403,11 @@ int ObTableInsertUpOp::do_insert_up_cache()
"insert_row", ROWEXPR2STR(eval_ctx_, get_primary_table_insert_row()));
} else {
modify_row.new_row_ = insert_new_row;
if (need_after_row_process(ins_ctdef) && OB_FAIL(dml_modify_rows_.push_back(modify_row))) {
insert_rows++;
if (OB_FAIL(replace_implict_cursor(insert_rows, 0, 0, 0))) {
LOG_WARN("merge implict cursor failed", K(ret));
} else if (need_after_row_process(ins_ctdef) && OB_FAIL(dml_modify_rows_.push_back(modify_row))) {
LOG_WARN("failed to push dml modify row to modified row list", K(ret));
} else {
insert_rows_++;
}
}
} else {
@ -389,6 +419,7 @@ int ObTableInsertUpOp::do_insert_up_cache()
const ObChunkDatumStore::StoredRow *upd_old_row = constraint_values.at(0).current_datum_row_;
ObDMLModifyRowNode modify_row(this, &upd_ctdef, &upd_rtdef, ObDmlEventType::DE_UPDATING);
clear_evaluated_flag();
found_rows++;
if (OB_FAIL(insert_row->to_expr(MY_SPEC.all_saved_exprs_, eval_ctx_))) {
LOG_WARN("insert_row to expr failed", K(ret), KPC(insert_row),
"exprs", get_primary_table_insert_row());
@ -417,28 +448,23 @@ int ObTableInsertUpOp::do_insert_up_cache()
LOG_WARN("convert expr to stored row failed", K(ret), "exprs", get_primary_table_upd_old_row());
} else if (OB_FAIL(calc_auto_increment(upd_ctdef))) {
LOG_WARN("calc auto_inc failed", K(ret), K(upd_ctdef));
} else if (FALSE_IT(upd_rtdef.found_rows_++)) {
// do nothing
} else if (is_ignore_) {
if (OB_FAIL(do_update_with_ignore())) {
LOG_WARN("do update with ignore failed", K(ret));
} else if (upd_rtdef.is_row_changed_) {
insert_rows_++;
upd_changed_rows_++;
insert_rows++;
update_rows++;
}
} else if (upd_rtdef.is_row_changed_) {
insert_rows++;
update_rows++;
modify_row.old_row_ = const_cast<ObChunkDatumStore::StoredRow *>(upd_old_row);
modify_row.new_row_ = upd_new_row;
if (OB_FAIL(conflict_checker_.update_row(upd_new_row, upd_old_row))) {
LOG_WARN("fail to update row in conflict_checker", K(ret),
KPC(upd_new_row), KPC(upd_old_row));
} else {
modify_row.old_row_ = const_cast<ObChunkDatumStore::StoredRow *>(upd_old_row);
modify_row.new_row_ = upd_new_row;
if (need_after_row_process(upd_ctdef) && OB_FAIL(dml_modify_rows_.push_back(modify_row))) {
LOG_WARN("failed to push dml modify row to modified row list", K(ret));
} else {
insert_rows_++;
upd_changed_rows_++;
}
} else if (need_after_row_process(upd_ctdef) && OB_FAIL(dml_modify_rows_.push_back(modify_row))) {
LOG_WARN("failed to push dml modify row to modified row list", K(ret));
}
} else {
// create table t1(c1 int primary key, c2 timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP);
@ -451,7 +477,21 @@ int ObTableInsertUpOp::do_insert_up_cache()
LOG_TRACE("curr update row is not changed", KPC(upd_new_row), KPC(upd_old_row));
}
}
}
// for insertup batch_dml_optimization
if (OB_SUCC(ret)) {
ObSQLSessionInfo *my_session = GET_MY_SESSION(ctx_);
int64_t affected_rows = my_session->get_capability().cap_flags_.OB_CLIENT_FOUND_ROWS ?
insert_rows + found_rows : insert_rows + update_rows;
if (OB_FAIL(replace_implict_cursor(affected_rows, found_rows, 0, update_rows))) {
LOG_WARN("merge implict cursor failed", K(ret));
}
}
} // end update
upd_rtdef.found_rows_ += found_rows;
insert_rows_ += insert_rows;
upd_changed_rows_ += update_rows;
} // while row store end
ret = OB_ITER_END == ret ? OB_SUCCESS : ret;
return ret;
@ -463,7 +503,7 @@ int ObTableInsertUpOp::insert_row_to_das(const ObInsCtDef &ins_ctdef,
{
int ret = OB_SUCCESS;
ObChunkDatumStore::StoredRow* stored_row = nullptr;
if (OB_FAIL(ObDMLService::insert_row(ins_ctdef, ins_rtdef, tablet_loc, dml_rtctx_, stored_row))) {
if (OB_FAIL(ObDMLService::insert_row(ins_ctdef, ins_rtdef, tablet_loc, upd_rtctx_, stored_row))) {
LOG_WARN("insert row with das failed", K(ret));
} else {
LOG_TRACE("insert one row", KPC(tablet_loc),
@ -489,10 +529,10 @@ int ObTableInsertUpOp::insert_row_to_das(const ObInsCtDef &ins_ctdef,
return ret;
}
int ObTableInsertUpOp::try_insert_row()
int ObTableInsertUpOp::try_insert_row(bool &is_skipped)
{
int ret = OB_SUCCESS;
bool is_skipped = false;
is_skipped = false;
for (int64_t i = 0; OB_SUCC(ret) && i < MY_SPEC.insert_up_ctdefs_.count(); ++i) {
// first time: insert each table with fetched row
// second time: after do conflict checker, insert row without duplicate key
@ -539,7 +579,7 @@ int ObTableInsertUpOp::lock_one_row_to_das(const ObUpdCtDef &upd_ctdef,
allocator,
upd_rtdef.dlock_rtdef_))) {
LOG_WARN("create das lock rtdef failed", K(ret));
} else if (OB_FAIL(ObDMLService::init_das_dml_rtdef(dml_rtctx_,
} else if (OB_FAIL(ObDMLService::init_das_dml_rtdef(upd_rtctx_,
*upd_ctdef.dlock_ctdef_,
*upd_rtdef.dlock_rtdef_,
nullptr))) {
@ -551,7 +591,7 @@ int ObTableInsertUpOp::lock_one_row_to_das(const ObUpdCtDef &upd_ctdef,
} else if (OB_ISNULL(upd_rtdef.dlock_rtdef_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("dlock_rtdef_ is null", K(ret));
} else if (OB_FAIL(ObDMLService::check_dml_tablet_validity(dml_rtctx_,
} else if (OB_FAIL(ObDMLService::check_dml_tablet_validity(upd_rtctx_,
*tablet_loc,
upd_ctdef.old_row_,
upd_ctdef,
@ -634,7 +674,7 @@ int ObTableInsertUpOp::delete_one_upd_old_row_das(const ObUpdCtDef &upd_ctdef,
ObSQLSessionInfo *my_session = GET_MY_SESSION(ctx_);
ObIAllocator &allocator = ctx_.get_allocator();
uint64_t tenant_id = my_session->get_effective_tenant_id();
if (OB_FAIL(ObDMLService::init_das_del_rtdef_for_update(dml_rtctx_, upd_ctdef, upd_rtdef))) {
if (OB_FAIL(ObDMLService::init_das_del_rtdef_for_update(upd_rtctx_, upd_ctdef, upd_rtdef))) {
LOG_WARN("init das dml rtdef failed", K(ret), K(upd_ctdef), K(upd_rtdef));
}
}
@ -644,7 +684,7 @@ int ObTableInsertUpOp::delete_one_upd_old_row_das(const ObUpdCtDef &upd_ctdef,
} else if (OB_ISNULL(upd_rtdef.ddel_rtdef_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ddel_rtdef is null", K(ret));
} else if (OB_FAIL(ObDMLService::check_dml_tablet_validity(dml_rtctx_,
} else if (OB_FAIL(ObDMLService::check_dml_tablet_validity(upd_rtctx_,
*tablet_loc,
upd_ctdef.old_row_,
upd_ctdef,
@ -676,7 +716,7 @@ int ObTableInsertUpOp::insert_one_upd_new_row_das(const ObUpdCtDef &upd_ctdef,
ObSQLSessionInfo *my_session = GET_MY_SESSION(ctx_);
ObIAllocator &allocator = ctx_.get_allocator();
uint64_t tenant_id = my_session->get_effective_tenant_id();
if (OB_FAIL(ObDMLService::init_das_ins_rtdef_for_update(dml_rtctx_, upd_ctdef, upd_rtdef))) {
if (OB_FAIL(ObDMLService::init_das_ins_rtdef_for_update(upd_rtctx_, upd_ctdef, upd_rtdef))) {
LOG_WARN("init das dml rtdef failed", K(ret), K(upd_ctdef), K(upd_rtdef));
}
}
@ -686,7 +726,7 @@ int ObTableInsertUpOp::insert_one_upd_new_row_das(const ObUpdCtDef &upd_ctdef,
} else if (OB_ISNULL(upd_rtdef.dins_rtdef_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("dins_rtdef is null", K(ret));
} else if (OB_FAIL(ObDMLService::check_dml_tablet_validity(dml_rtctx_,
} else if (OB_FAIL(ObDMLService::check_dml_tablet_validity(upd_rtctx_,
*tablet_loc,
upd_ctdef.new_row_,
upd_ctdef,
@ -845,6 +885,16 @@ int ObTableInsertUpOp::get_next_row_from_child()
return ret;
}
int ObTableInsertUpOp::rollback_savepoint(const transaction::ObTxSEQ &savepoint_no)
{
int ret = OB_SUCCESS;
NG_TRACE_TIMES(2, insertup_start_rollback);
if (OB_FAIL(ObSqlTransControl::rollback_savepoint(ctx_, savepoint_no))) {
LOG_WARN("fail to rollback to save_point", K(ret), K(savepoint_no));
}
return ret;
}
int ObTableInsertUpOp::do_insert_up()
{
int ret = OB_SUCCESS;
@ -854,36 +904,32 @@ int ObTableInsertUpOp::do_insert_up()
transaction::ObTxSEQ savepoint_no;
// must set conflict_row fetch flag
add_need_conflict_result_flag();
NG_TRACE_TIMES(2, insert_up_load_all_row);
if (OB_FAIL(ObSqlTransControl::create_anonymous_savepoint(ctx_, savepoint_no))) {
LOG_WARN("fail to create save_point", K(ret));
} else if (OB_FAIL(load_batch_insert_up_rows(is_iter_end, insert_rows))) {
LOG_WARN("fail to load all row", K(ret));
} else if (OB_FAIL(post_all_dml_das_task(dml_rtctx_, false))) {
} else if (OB_FAIL(post_all_try_insert_das_task(dml_rtctx_))) {
LOG_WARN("fail to post all das task", K(ret));
} else if (!check_is_duplicated() && OB_FAIL(ObDMLService::handle_after_row_processing(this, &dml_modify_rows_))) {
LOG_WARN("try insert is not duplicated, failed to process foreign key handle", K(ret));
} else if (!check_is_duplicated()) {
insert_rows_ += insert_rows;
LOG_TRACE("try insert is not duplicated", K(ret), K(insert_rows_));
} else if (OB_FAIL(fetch_conflict_rowkey())) {
} else if (OB_FAIL(fetch_conflict_rowkey(insert_up_row_store_.get_row_cnt()))) {
LOG_WARN("fail to fetch conflict row", K(ret));
} else if (OB_FAIL(reset_das_env())) {
// 这里需要reuse das 相关信息
LOG_WARN("fail to reset das env", K(ret));
} else if (OB_FAIL(ObSqlTransControl::rollback_savepoint(ctx_, savepoint_no))) {
} else if (OB_FAIL(rollback_savepoint(savepoint_no))) {
// 本次插入存在冲突, 回滚到save_point
LOG_WARN("fail to rollback to save_point", K(ret));
} else if (OB_FAIL(conflict_checker_.do_lookup_and_build_base_map(
insert_up_row_store_.get_row_cnt()))) {
} else if (OB_FAIL(conflict_checker_.do_lookup_and_build_base_map(insert_up_row_store_.get_row_cnt()))) {
LOG_WARN("fail to build conflict map", K(ret));
} else if (OB_FAIL(do_insert_up_cache())) {
LOG_WARN("fail to do insert_up in cache", K(ret));
} else if (!is_ignore_ && OB_FAIL(prepare_final_insert_up_task())) {
LOG_WARN("fail to prepare final das task", K(ret));
} else if (OB_FAIL(post_all_dml_das_task(upd_rtctx_, true))) {
LOG_WARN("do insert rows post process failed", K(ret));
} else if (OB_FAIL(post_all_dml_das_task(dml_rtctx_, false))) {
} else if (OB_FAIL(post_all_dml_das_task(upd_rtctx_))) {
LOG_WARN("do insert rows post process failed", K(ret));
} else if (OB_FAIL(ObDMLService::handle_after_row_processing(this, &dml_modify_rows_))) {
LOG_WARN("try insert is duplicated, failed to process foreign key handle", K(ret));
@ -904,33 +950,47 @@ int ObTableInsertUpOp::load_batch_insert_up_rows(bool &is_iter_end, int64_t &ins
{
int ret = OB_SUCCESS;
is_iter_end = false;
int64_t row_cnt = 0;
bool reach_mem_limit = false;
int row_count = 0;
NG_TRACE_TIMES(2, insert_up_start_load_row);
ObPhysicalPlanCtx *plan_ctx = ctx_.get_physical_plan_ctx();
int64_t simulate_batch_row_cnt = - EVENT_CALL(EventTable::EN_TABLE_INSERT_UP_BATCH_ROW_COUNT);
int64_t default_row_batch_cnt = simulate_batch_row_cnt > 0 ?
simulate_batch_row_cnt : INT64_MAX;
if (is_ignore_ || execute_single_row_) {
// If it is ignore mode or need excute row by row, degenerate into single line execution
default_row_batch_cnt = 1;
int64_t default_row_store_mem_limit = OB_DEFAULT_INSERT_UP_MEMORY_LIMIT;
int64_t simulate_batch_count = - EVENT_CALL(EventTable::EN_TABLE_INSERT_UP_BATCH_ROW_COUNT);
if (upd_rtctx_.das_ref_.get_parallel_type() != DAS_SERIALIZATION) {
default_row_store_mem_limit = OB_DEFAULT_INSERT_UP_MEMORY_LIMIT * 5; // 10M
}
LOG_DEBUG("simulate lookup row batch count", K(simulate_batch_row_cnt), K(default_row_batch_cnt));
while (OB_SUCC(ret) && ++row_cnt <= default_row_batch_cnt) {
while (OB_SUCC(ret) && !reach_mem_limit) {
bool is_skipped = false;
if (OB_FAIL(get_next_row_from_child())) {
if (OB_ITER_END != ret) {
LOG_WARN("fail to load next row from child", K(ret));
} else {
iter_end_ = true;
}
} else if (OB_FAIL(try_insert_row())) {
} else if (OB_FAIL(try_insert_row(is_skipped))) {
LOG_WARN("try insert row to das", K(ret));
} else if (OB_FAIL(insert_up_row_store_.add_row(MY_SPEC.all_saved_exprs_, &eval_ctx_))) {
} else if (!is_skipped &&
OB_FAIL(insert_up_row_store_.add_row(MY_SPEC.all_saved_exprs_, &eval_ctx_))) {
LOG_WARN("add insert_up row to row store failed", K(ret));
} else {
plan_ctx->record_last_insert_id_cur_stmt();
insert_rows++;
if (insert_up_row_store_.get_mem_used() >= OB_DEFAULT_INSERT_UP_MEMORY_LIMIT) {
LOG_INFO("insert up rows used memory over limit", K(ret), K(row_cnt), K(insert_rows));
break;
row_count++;
if (insert_up_row_store_.get_mem_used() >= default_row_store_mem_limit ||
is_ignore_ || execute_single_row_) {
reach_mem_limit = true;
LOG_TRACE("insert up rows used memory over limit", K(default_row_store_mem_limit), K(insert_rows),
K(insert_up_row_store_.get_mem_used()), K(is_ignore_), K(execute_single_row_));
} else if (simulate_batch_count != 0 && row_count > simulate_batch_count) {
reach_mem_limit = true;
LOG_TRACE("insert up rows reach simulate_batch_count", K(row_count), K(default_row_store_mem_limit));
}
// record for insertup batch_dml_optimization
int64_t insert_row = is_skipped ? 0 : 1;
if (OB_FAIL(merge_implict_cursor(insert_row, 0, 0, 0))) {
LOG_WARN("merge implict cursor failed", K(ret));
}
}
}
@ -942,17 +1002,59 @@ int ObTableInsertUpOp::load_batch_insert_up_rows(bool &is_iter_end, int64_t &ins
return ret;
}
int ObTableInsertUpOp::post_all_dml_das_task(ObDMLRtCtx &dml_rtctx, bool del_task_ahead)
int ObTableInsertUpOp::post_all_try_insert_das_task(ObDMLRtCtx &dml_rtctx)
{
int ret = OB_SUCCESS;
NG_TRACE_TIMES(2, insert_up_try_insert);
if (dml_rtctx.das_ref_.has_task()) {
if (del_task_ahead) {
if (OB_FAIL(dml_rtctx.das_ref_.pick_del_task_to_first())) {
LOG_WARN("remove delete das task first failed", K(ret));
if (gts_state_ == WITH_UNIQUE_GLOBAL_INDEX_STATE) {
share::ObLSID ls_id;
ObSQLSessionInfo *my_session = GET_MY_SESSION(ctx_);
ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(ctx_);
if (dml_rtctx.das_ref_.check_tasks_same_ls_and_is_local(ls_id)) {
if (OB_FAIL(ObSqlTransControl::get_ls_read_snapshot(my_session,
plan_ctx,
ls_id,
ctx_.get_das_ctx().get_snapshot()))) {
LOG_WARN("fail to get ls read snapshot", K(ret));
} else {
LOG_TRACE("all task with same ls_id get ls snapshot", K(ls_id), K(ctx_.get_das_ctx().get_snapshot()));
}
} else {
if (OB_FAIL(ObSqlTransControl::get_read_snapshot(my_session,
plan_ctx,
ctx_.get_das_ctx().get_snapshot()))) {
LOG_WARN("fail to get global read snapshot", K(ret));
} else {
gts_state_ = GTE_GTS_STATE;
LOG_TRACE("tablets in different ls_id, we get_gts", K(ctx_.get_das_ctx().get_snapshot()));
}
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(dml_rtctx.das_ref_.execute_all_task())) {
LOG_WARN("execute all delete das task failed", K(ret));
}
}
}
return ret;
}
int ObTableInsertUpOp::post_all_dml_das_task(ObDMLRtCtx &dml_rtctx)
{
int ret = OB_SUCCESS;
NG_TRACE_TIMES(2, insertup_final_write);
if (dml_rtctx.das_ref_.has_task()) {
if (gts_state_ == USE_PARTITION_SNAPSHOT_STATE) {
if (OB_FAIL(conflict_checker_.set_partition_snapshot_for_das_task(dml_rtctx.das_ref_))) {
LOG_WARN("fail to set partition snapshot", K(ret));
}
}
if (OB_FAIL(ret)) {
// do nothing
} else if (OB_SUCC(ret)) {
if (OB_FAIL(dml_rtctx.das_ref_.execute_all_task())) {
LOG_WARN("execute all das task failed", K(ret));
}
@ -1022,12 +1124,37 @@ int ObTableInsertUpOp::get_next_conflict_rowkey(DASTaskIter &task_iter)
return ret;
}
int ObTableInsertUpOp::fetch_conflict_rowkey()
int ObTableInsertUpOp::fetch_conflict_rowkey(int64_t row_cnt)
{
int ret = OB_SUCCESS;
bool got_row = false;
NG_TRACE_TIMES(2, insert_up_start_lookup);
NG_TRACE_TIMES(2, insertup_build_fetch_rowkey);
DASTaskIter task_iter = dml_rtctx_.das_ref_.begin_task_iter();
if (row_cnt > ObConflictCheckerCtdef::MIN_ROW_COUNT_USE_HASHSET_DO_DISTICT) {
if (OB_FAIL(conflict_checker_.create_rowkey_check_hashset(row_cnt))) {
LOG_WARN("fail to create conflict_checker hash_set", K(ret), K(row_cnt));
}
}
if (gts_state_ == USE_PARTITION_SNAPSHOT_STATE) {
DASTaskIter task_iter = dml_rtctx_.das_ref_.begin_task_iter();
while (OB_SUCC(ret) && !task_iter.is_end()) {
ObDASInsertOp *ins_op = static_cast<ObDASInsertOp*>(*task_iter);
const ObDASInsCtDef *ins_ctdef = static_cast<const ObDASInsCtDef*>(ins_op->get_ctdef());
transaction::ObTxReadSnapshot *snapshot = ins_op->get_das_gts_opt_info().get_response_snapshot();
if (OB_ISNULL(snapshot)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null", K(ret));
} else if (!snapshot->is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected invalid snaopshot", K(ret), KPC(snapshot));
} else if (OB_FAIL(conflict_checker_.collect_all_snapshot(*snapshot, ins_op->get_tablet_loc()))) {
LOG_WARN("fail to collect snapshot", K(ret), KPC(snapshot), KPC(ins_op));
}
++task_iter;
}
}
while (OB_SUCC(ret) && !task_iter.is_end()) {
// 不需要clear rowkey表达式的eval_flag，因为主键使用的是column_ref表达式，不存在eval_fun
if (OB_FAIL(get_next_conflict_rowkey(task_iter))) {
@ -1046,7 +1173,6 @@ int ObTableInsertUpOp::prepare_final_insert_up_task()
{
int ret = OB_SUCCESS;
ObConflictRowMap *primary_map = NULL;
NG_TRACE_TIMES(2, insert_up_final_shuff);
OZ(conflict_checker_.get_primary_table_map(primary_map));
CK(OB_NOT_NULL(primary_map));
ObConflictRowMap::iterator start_row_iter = primary_map->begin();
@ -1147,7 +1273,7 @@ int ObTableInsertUpOp::do_update_with_ignore()
ret = OB_NOT_SUPPORTED;
LOG_WARN("ignore not supported", K(ret), KPC(old_tablet_loc), KPC(new_tablet_loc));
LOG_USER_ERROR(OB_NOT_SUPPORTED, "Do update with ignore under inconsistent tablet loc");
} else if (OB_FAIL(ObDMLService::update_row(*upd_ctdef, upd_rtdef, old_tablet_loc, new_tablet_loc, dml_rtctx_,
} else if (OB_FAIL(ObDMLService::update_row(*upd_ctdef, upd_rtdef, old_tablet_loc, new_tablet_loc, upd_rtctx_,
modify_row.old_row_, modify_row.new_row_, modify_row.full_row_))) {
LOG_WARN("fail to insert update_row to das", K(ret));
} else if (need_after_row_process(*upd_ctdef) && OB_FAIL(dml_modify_rows_.push_back(modify_row))) {
@ -1248,7 +1374,7 @@ int ObTableInsertUpOp::calc_auto_increment(const ObUpdCtDef &upd_ctdef)
{
int ret = OB_SUCCESS;
ObPhysicalPlanCtx *plan_ctx = ctx_.get_physical_plan_ctx();
NG_TRACE_TIMES(2, insertup_start_calc_update_row);
NG_TRACE_TIMES(2, insertup_calc_auto_inc);
if (OB_SUCC(ret)) {
// before calc new row, to be compatible with MySQL
// 1. disable operation to sync user specified value for auto-increment column because duplicate
@ -1381,7 +1507,6 @@ int ObTableInsertUpOp::update_auto_increment(const ObExpr &expr,
}
}
}
NG_TRACE_TIMES(2, insertup_end_auto_increment);
return ret;
}

17
src/sql/engine/dml/ob_table_insert_up_op.h
View File

@ -85,8 +85,8 @@ public:
eval_ctx_,
MY_SPEC.conflict_checker_ctdef_),
insert_up_row_store_("InsertUpRow"),
is_ignore_(false)
is_ignore_(false),
gts_state_(WITHOUT_GTS_OPT_STATE)
{
}
@ -103,7 +103,7 @@ public:
const uint64_t cid);
// 执行所有尝试插入的 das task， fetch冲突行的主表主键
int fetch_conflict_rowkey();
int fetch_conflict_rowkey(int64_t row_cnt);
int get_next_conflict_rowkey(DASTaskIter &task_iter);
virtual void destroy()
@ -125,6 +125,8 @@ protected:
int do_insert_up();
int rollback_savepoint(const transaction::ObTxSEQ &savepoint_no);
// 检查是否有duplicated key 错误发生
bool check_is_duplicated();
@ -197,10 +199,11 @@ protected:
int do_update_with_ignore();
// 提交当前所有的 das task;
int post_all_dml_das_task(ObDMLRtCtx &das_ctx, bool del_task_ahead);
int post_all_dml_das_task(ObDMLRtCtx &das_ctx);
int post_all_try_insert_das_task(ObDMLRtCtx &dml_rtctx);
// batch的执行插入 process_row and then write to das,
int try_insert_row();
int try_insert_row(bool &is_skipped);
// batch的执行插入 process_row and then write to das,
int update_row_to_das(bool need_do_trigger);
@ -211,7 +214,7 @@ protected:
int deal_hint_part_selection(ObObjectID partition_id);
virtual int check_need_exec_single_row() override;
virtual ObDasParallelType check_das_parallel_type() override;
private:
int check_insert_up_ctdefs_valid() const;
@ -231,6 +234,7 @@ private:
protected:
const static int64_t OB_DEFAULT_INSERT_UP_MEMORY_LIMIT = 2 * 1024 * 1024L; // 2M in default
const static int64_t OB_DEFAULT_INSERT_UP_BATCH_ROW_COUNT = 1000L; // 1000 in default
int64_t insert_rows_;
int64_t upd_changed_rows_;
int64_t found_rows_;
@ -239,6 +243,7 @@ protected:
common::ObArrayWrap<ObInsertUpRtDef> insert_up_rtdefs_;
ObChunkDatumStore insert_up_row_store_; //所有的insert_up的行的集合
bool is_ignore_; // 暂时记录一下是否是ignore的insert_up SQL语句
ObDmlGTSOptState gts_state_;
};
} // end namespace sql
} // end namespace oceanbase

140
src/sql/engine/dml/ob_table_modify_op.cpp
View File

@ -737,6 +737,33 @@ void ObTableModifyOp::clear_dml_evaluated_flag()
}
}
ObDasParallelType ObTableModifyOp::check_das_parallel_type()
{
ObDasParallelType type = DAS_SERIALIZATION;
ObSQLSessionInfo *session = GET_MY_SESSION(ctx_);
if (MY_SPEC.is_pdml_) {
type = DAS_SERIALIZATION;
} else if (!is_user_tenant(MTL_ID())) {
type = DAS_SERIALIZATION;
LOG_TRACE("not user tenant, can't submit task parallel", K(MTL_ID()));
} else if (execute_single_row_) {
type = DAS_SERIALIZATION;
LOG_TRACE("execute_single_row is true, can't submit task parallel", K(execute_single_row_));
} else if (session->is_inner()) {
type = DAS_SERIALIZATION;
LOG_TRACE("session is inner, can't submit task parallel", K(session->is_inner()));
} else if (MY_SPEC.plan_->has_nested_sql()) {
type = DAS_SERIALIZATION;
LOG_TRACE("has nested sql, can't submit task parallel", K(MY_SPEC.plan_->has_nested_sql()));
} else if (MY_SPEC.plan_->get_das_dop() > 1 && MY_SPEC.das_dop_ > 1) {
type = DAS_STREAMING_PARALLEL;
} else {
type = DAS_SERIALIZATION;
LOG_TRACE("das dop not larger than 1", K(MY_SPEC.plan_->get_das_dop()), K(MY_SPEC.das_dop_));
}
return type;
}
OB_INLINE int ObTableModifyOp::init_das_dml_ctx()
{
ObSQLSessionInfo *session = GET_MY_SESSION(ctx_);
@ -764,44 +791,83 @@ OB_INLINE int ObTableModifyOp::init_das_dml_ctx()
: &MY_SPEC.plan_->get_expr_frame_info());
dml_rtctx_.das_ref_.set_mem_attr(memattr);
dml_rtctx_.das_ref_.set_execute_directly(!MY_SPEC.use_dist_das_);
dml_rtctx_.das_ref_.get_das_parallel_ctx().set_das_parallel_type(check_das_parallel_type());
dml_rtctx_.das_ref_.get_das_parallel_ctx().set_das_dop(ctx_.get_das_ctx().get_real_das_dop());
if (check_das_parallel_type() != DAS_SERIALIZATION) {
LOG_TRACE("this sql use das parallel submit", K(check_das_parallel_type()));
}
return OB_SUCCESS;
}
int ObTableModifyOp::merge_implict_cursor(int64_t insert_rows,
int64_t update_rows,
int64_t delete_rows,
int64_t found_rows)
int ObTableModifyOp::replace_implict_cursor(int64_t affected_rows,
int64_t found_rows,
int64_t matched_rows,
int64_t duplicated_rows)
{
int ret = OB_SUCCESS;
bool is_ins_val_opt = ctx_.get_sql_ctx()->is_do_insert_batch_opt();
ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(ctx_);
if (MY_SPEC.ab_stmt_id_ != nullptr && !is_ins_val_opt) {
ObDatum *stmt_id_datum = nullptr;
if (OB_FAIL(MY_SPEC.ab_stmt_id_->eval(eval_ctx_, stmt_id_datum))) {
LOG_WARN("eval ab stmt id failed", K(ret));
} else {
ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(ctx_);
int64_t stmt_id = stmt_id_datum->get_int();
ObImplicitCursorInfo implicit_cursor;
implicit_cursor.stmt_id_ = stmt_id;
implicit_cursor.found_rows_ += found_rows;
implicit_cursor.matched_rows_ += found_rows;
implicit_cursor.last_insert_id_ = plan_ctx->get_autoinc_col_value();
if (insert_rows > 0) {
implicit_cursor.affected_rows_ += insert_rows;
}
if (update_rows > 0) {
ObSQLSessionInfo *session = GET_MY_SESSION(ctx_);
bool client_found_rows = session->get_capability().cap_flags_.OB_CLIENT_FOUND_ROWS;
implicit_cursor.duplicated_rows_ += update_rows;
implicit_cursor.affected_rows_ += client_found_rows ? found_rows : update_rows;
}
if (delete_rows > 0) {
implicit_cursor.affected_rows_ += delete_rows;
}
if (OB_FAIL(plan_ctx->merge_implicit_cursor_info(implicit_cursor))) {
LOG_WARN("merge implicit cursor info to plan ctx failed", K(ret), K(implicit_cursor));
}
LOG_DEBUG("merge implicit cursor", K(ret), K(implicit_cursor));
ObImplicitCursorInfo implicit_cursor;
if (OB_FAIL(prepare_implict_cursor(affected_rows,
found_rows,
matched_rows,
duplicated_rows,
implicit_cursor))) {
LOG_WARN("prepare implict cursor failed", K(ret),
K(affected_rows), K(found_rows), K(matched_rows), K(duplicated_rows));
} else if (OB_FAIL(plan_ctx->replace_implicit_cursor_info(implicit_cursor))) {
LOG_WARN("merge implicit cursor info to plan ctx failed", K(ret), K(implicit_cursor));
}
}
return ret;
}
int ObTableModifyOp::prepare_implict_cursor(int64_t affected_rows,
int64_t found_rows,
int64_t matched_rows,
int64_t duplicated_rows,
ObImplicitCursorInfo &implicit_cursor)
{
int ret = OB_SUCCESS;
ObDatum *stmt_id_datum = nullptr;
if (OB_FAIL(MY_SPEC.ab_stmt_id_->eval(eval_ctx_, stmt_id_datum))) {
LOG_WARN("eval ab stmt id failed", K(ret));
} else {
ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(ctx_);
int64_t stmt_id = stmt_id_datum->get_int();
implicit_cursor.stmt_id_ = stmt_id;
implicit_cursor.found_rows_ += found_rows;
implicit_cursor.matched_rows_ += matched_rows;
implicit_cursor.affected_rows_ += affected_rows;
implicit_cursor.duplicated_rows_ += duplicated_rows;
implicit_cursor.last_insert_id_ = plan_ctx->get_autoinc_col_value();
LOG_DEBUG("merge implicit cursor", K(ret), K(implicit_cursor));
}
return ret;
}
int ObTableModifyOp::merge_implict_cursor(int64_t affected_rows,
int64_t found_rows,
int64_t matched_rows,
int64_t duplicated_rows)
{
int ret = OB_SUCCESS;
bool is_ins_val_opt = ctx_.get_sql_ctx()->is_do_insert_batch_opt();
ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(ctx_);
if (MY_SPEC.ab_stmt_id_ != nullptr && !is_ins_val_opt) {
ObDatum *stmt_id_datum = nullptr;
ObImplicitCursorInfo implicit_cursor;
if (OB_FAIL(prepare_implict_cursor(affected_rows,
found_rows,
matched_rows,
duplicated_rows,
implicit_cursor))) {
LOG_WARN("prepare implict cursor failed", K(ret),
K(affected_rows), K(found_rows), K(matched_rows), K(duplicated_rows));
} else if (OB_FAIL(plan_ctx->merge_implicit_cursor_info(implicit_cursor))) {
LOG_WARN("merge implicit cursor info to plan ctx failed", K(ret), K(implicit_cursor));
}
}
return ret;
@ -1130,16 +1196,10 @@ int ObTableModifyOp::submit_all_dml_task()
int ObTableModifyOp::discharge_das_write_buffer()
{
int ret = OB_SUCCESS;
int64_t simulate_buffer_size = - EVENT_CALL(EventTable::EN_DAS_DML_BUFFER_OVERFLOW);
int64_t buffer_size_limit = is_meta_tenant(tenant_id_) ? das::OB_DAS_MAX_META_TENANT_PACKET_SIZE : das::OB_DAS_MAX_TOTAL_PACKET_SIZE;
if (OB_UNLIKELY(simulate_buffer_size > 0)) {
buffer_size_limit = simulate_buffer_size;
}
if (dml_rtctx_.get_row_buffer_size() >= buffer_size_limit) {
if (REACH_COUNT_INTERVAL(100)) {
LOG_INFO("DASWriteBuffer full, now to write storage",
"buffer memory", dml_rtctx_.das_ref_.get_das_alloc().used(), K(dml_rtctx_.get_row_buffer_size()));
}
if (dml_rtctx_.need_submit_all_tasks()) {
LOG_INFO("DASWriteBuffer full, now to write storage",
"buffer memory", dml_rtctx_.das_ref_.get_das_alloc().used(),
K(dml_rtctx_.get_das_task_memory_size()), K(dml_rtctx_.get_das_parallel_task_size()));
ret = submit_all_dml_task();
} else if (execute_single_row_) {
if (REACH_COUNT_INTERVAL(100)) { // print log per 100 times.

31
src/sql/engine/dml/ob_table_modify_op.h
View File

@ -24,6 +24,14 @@ namespace oceanbase
namespace sql
{
enum ObDmlGTSOptState
{
WITHOUT_GTS_OPT_STATE = 0,
USE_PARTITION_SNAPSHOT_STATE = 1,
WITH_UNIQUE_GLOBAL_INDEX_STATE = 2,
GTE_GTS_STATE = 3,
};
class ForeignKeyHandle
{
public:
@ -134,7 +142,8 @@ public:
uint64_t has_instead_of_trigger_ : 1; // abandoned, don't use again
uint64_t is_pdml_update_split_ : 1; // 标记delete, insert op是否由update拆分而来
uint64_t check_fk_batch_ : 1; // mark if the foreign key constraint can be checked in batch
uint64_t reserved_ : 55;
uint64_t is_pdml_ : 1;
uint64_t reserved_ : 54;
};
};
int64_t das_dop_; // default is 0
@ -264,13 +273,25 @@ protected:
//such as: set affected_rows to query context, rewrite some error code
virtual int write_rows_post_proc(int last_errno)
{ UNUSED(last_errno); return common::OB_NOT_IMPLEMENT; }
virtual ObDasParallelType check_das_parallel_type();
int init_das_dml_ctx();
//to merge array binding cusor info when array binding is executed in batch mode
int merge_implict_cursor(int64_t insert_rows,
int64_t update_rows,
int64_t delete_rows,
int64_t found_rows);
int merge_implict_cursor(int64_t affected_rows,
int64_t found_rows,
int64_t matched_rows,
int64_t duplicated_rows);
int prepare_implict_cursor(int64_t affected_rows,
int64_t found_rows,
int64_t matched_rows_,
int64_t duplicated_rows_,
ObImplicitCursorInfo &implicit_cursor);
int replace_implict_cursor(int64_t insert_rows,
int64_t update_rows,
int64_t delete_rows,
int64_t found_rows);
int discharge_das_write_buffer();
virtual void record_err_for_load_data(int err_ret, int row_num) { UNUSED(err_ret); UNUSED(row_num); }
public:

323
src/sql/engine/dml/ob_table_replace_op.cpp
View File

@ -24,6 +24,7 @@
#include "sql/das/ob_data_access_service.h"
#include "sql/engine/dml/ob_trigger_handler.h"
#include "sql/engine/dml/ob_fk_checker.h"
#include "share/ob_ls_id.h"
namespace oceanbase
{
@ -104,6 +105,12 @@ OB_DEF_SERIALIZE_SIZE(ObTableReplaceSpec)
return len;
}
ObDasParallelType ObTableReplaceOp::check_das_parallel_type()
{
return DAS_SERIALIZATION;
}
int ObTableReplaceOp::check_need_exec_single_row()
{
int ret = OB_SUCCESS;
@ -168,16 +175,44 @@ OB_INLINE int ObTableReplaceOp::inner_open_with_das()
int ret = OB_SUCCESS;
const ObExprFrameInfo *expr_frame_info = NULL;
ObDASTableLoc *table_loc = nullptr;
bool use_partition_gts_opt = false;
expr_frame_info = nullptr != MY_SPEC.expr_frame_info_
? MY_SPEC.expr_frame_info_
: &MY_SPEC.plan_->get_expr_frame_info();
if (ctx_.get_das_ctx().get_use_gts_opt()) {
gts_state_ = MY_SPEC.has_global_unique_index_ == true ?
WITH_UNIQUE_GLOBAL_INDEX_STATE : USE_PARTITION_SNAPSHOT_STATE;
if (gts_state_ == USE_PARTITION_SNAPSHOT_STATE) {
dml_rtctx_.das_ref_.set_do_gts_opt(true);
replace_rtctx_.das_ref_.set_do_gts_opt(true);
use_partition_gts_opt = true;
}
}
if (OB_FAIL(init_replace_rtdef())) {
LOG_WARN("init replace rtdef failed", K(ret), K(MY_SPEC.replace_ctdefs_.count()));
} else if (OB_ISNULL(table_loc = replace_rtdefs_.at(0).ins_rtdef_.das_rtdef_.table_loc_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("table location is invalid", K(ret));
} else if (OB_FAIL(conflict_checker_.init_conflict_checker(expr_frame_info, table_loc))) {
LOG_WARN("init conflict_checker fail", K(ret));
} else if (OB_FAIL(conflict_checker_.init_conflict_checker(expr_frame_info,
table_loc,
use_partition_gts_opt))) {
LOG_WARN("init conflict_checker fail", K(ret), K(use_partition_gts_opt));
} else {
// init update das_ref
ObSQLSessionInfo *session = GET_MY_SESSION(ctx_);
ObMemAttr mem_attr;
mem_attr.tenant_id_ = session->get_effective_tenant_id();
mem_attr.label_ = "SqlReplaceInto";
replace_rtctx_.das_ref_.set_expr_frame_info(expr_frame_info);
replace_rtctx_.das_ref_.set_mem_attr(mem_attr);
replace_rtctx_.das_ref_.set_execute_directly(!MY_SPEC.use_dist_das_);
ObDasParallelType type = ObTableModifyOp::check_das_parallel_type();
if (DAS_SERIALIZATION != type) {
type = DAS_BLOCKING_PARALLEL;
LOG_TRACE("this sql use das parallel submit for replace", K(check_das_parallel_type()));
}
replace_rtctx_.das_ref_.get_das_parallel_ctx().set_das_parallel_type(type);
replace_rtctx_.das_ref_.get_das_parallel_ctx().set_das_dop(ctx_.get_das_ctx().get_real_das_dop());
}
return ret;
}
@ -212,6 +247,16 @@ int ObTableReplaceOp::inner_rescan()
LOG_WARN("rescan child operator failed", K(ret));
} else {
conflict_checker_.set_local_tablet_loc(MY_INPUT.get_tablet_loc());
if (replace_rtctx_.das_ref_.has_task()) {
if (OB_FAIL(replace_rtctx_.das_ref_.close_all_task())) {
LOG_WARN("close all update das task failed", K(ret));
} else {
replace_rtctx_.reuse();
}
}
}
if (OB_SUCC(ret)) {
replace_rtdefs_.release_array();
if (OB_UNLIKELY(iter_end_)) {
//do nothing
@ -227,12 +272,20 @@ int ObTableReplaceOp::inner_rescan()
int ObTableReplaceOp::inner_close()
{
int ret = OB_SUCCESS;
int close_ret = OB_SUCCESS;
NG_TRACE(replace_inner_close);
if (OB_FAIL(conflict_checker_.close())) {
LOG_WARN("fail to close conflict_checker", K(ret));
}
if (replace_rtctx_.das_ref_.has_task()) {
close_ret = (replace_rtctx_.das_ref_.close_all_task());
if (OB_SUCCESS == close_ret) {
replace_rtctx_.das_ref_.reset();
}
}
ret = OB_SUCCESS == ret ? close_ret : ret;
// close dml das tasks
int close_ret = ObTableModifyOp::inner_close();
close_ret = ObTableModifyOp::inner_close();
return (OB_SUCCESS == ret) ? close_ret : ret;
}
@ -287,29 +340,50 @@ OB_INLINE int ObTableReplaceOp::load_all_replace_row(bool &is_iter_end)
{
int ret = OB_SUCCESS;
is_iter_end = false;
bool reach_mem_limit = false;
ObInsCtDef *primary_ins_ctdef = NULL;
int64_t row_cnt = 0;
ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(ctx_);
int64_t simulate_batch_row_cnt = - EVENT_CALL(EventTable::EN_TABLE_REPLACE_BATCH_ROW_COUNT);
int64_t default_row_batch_cnt = simulate_batch_row_cnt > 0 ?
simulate_batch_row_cnt : DEFAULT_REPLACE_BATCH_ROW_COUNT;
LOG_DEBUG("simulate lookup row batch count", K(simulate_batch_row_cnt), K(default_row_batch_cnt));
if (execute_single_row_) {
default_row_batch_cnt = 1;
ObPhysicalPlanCtx *plan_ctx = ctx_.get_physical_plan_ctx();
int64_t simulate_batch_count = - EVENT_CALL(EventTable::EN_TABLE_REPLACE_BATCH_ROW_COUNT);
int64_t default_row_store_mem_limit = OB_DEFAULT_REPLACE_MEMORY_LIMIT;
if (replace_rtctx_.das_ref_.get_parallel_type() != DAS_SERIALIZATION) {
default_row_store_mem_limit = OB_DEFAULT_REPLACE_MEMORY_LIMIT * 5; // 10M
}
while (OB_SUCC(ret) && ++row_cnt <= default_row_batch_cnt) {
NG_TRACE_TIMES(2, replace_start_load_row);
while (OB_SUCC(ret) && !reach_mem_limit) {
// todo @kaizhan.dkz @wangbo.wb 增加行前trigger逻辑在这里
// 新行的外键检查也在这里做
bool is_skipped = false;
if (OB_FAIL(get_next_row_from_child())) {
if (OB_ITER_END != ret) {
LOG_WARN("fail to load next row from child", K(ret));
}
} else if (OB_FAIL(insert_row_to_das(true))) {
} else if (OB_FAIL(insert_row_to_das(is_skipped))) {
LOG_WARN("insert row to das", K(ret));
} else if (OB_FAIL(replace_row_store_.add_row(get_primary_table_new_row(), &eval_ctx_))) {
} else if (get_all_saved_exprs().empty()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected empty", K(ret));
} else if (OB_FAIL(!is_skipped && replace_row_store_.add_row(get_all_saved_exprs(), &eval_ctx_))) {
LOG_WARN("add replace row to row store failed", K(ret));
} else {
row_cnt++;
plan_ctx->record_last_insert_id_cur_stmt();
if (replace_row_store_.get_mem_used() >= default_row_store_mem_limit ||
execute_single_row_) {
reach_mem_limit = true;
LOG_TRACE("replace into rows used memory over limit", K(ret), K(row_cnt),
K(default_row_store_mem_limit), K(replace_row_store_.get_mem_used()), K(execute_single_row_));
} else if (simulate_batch_count != 0 && row_cnt > simulate_batch_count) {
reach_mem_limit = true;
LOG_TRACE("replace into rows count reach simulate_batch_count", K(ret),
K(row_cnt), K(simulate_batch_count), K(replace_row_store_.get_mem_used()));
}
// record for insertup batch_dml_optimization
int64_t insert_row = is_skipped ? 0 : 1;
if (OB_FAIL(merge_implict_cursor(insert_row, 0, 0, 0))) {
LOG_WARN("merge implict cursor failed", K(ret));
}
}
}
if (OB_ITER_END == ret) {
@ -319,10 +393,33 @@ OB_INLINE int ObTableReplaceOp::load_all_replace_row(bool &is_iter_end)
return ret;
}
int ObTableReplaceOp::insert_row_to_das(bool need_do_trigger)
int ObTableReplaceOp::final_insert_row_to_das()
{
int ret = OB_SUCCESS;
bool is_skipped = false;
for (int64_t i = 0; OB_SUCC(ret) && i < MY_SPEC.replace_ctdefs_.count(); ++i) {
// insert each table with fetched row
const ObReplaceCtDef &replace_ctdef = *(MY_SPEC.replace_ctdefs_.at(i));
const ObInsCtDef &ins_ctdef = *(replace_ctdef.ins_ctdef_);
ObReplaceRtDef &replace_rtdef = replace_rtdefs_.at(i);
ObInsRtDef &ins_rtdef = replace_rtdef.ins_rtdef_;
ObDASTabletLoc *tablet_loc = nullptr;
ObChunkDatumStore::StoredRow *stored_row_ = nullptr;
++ins_rtdef.cur_row_num_;
if (OB_FAIL(calc_insert_tablet_loc(ins_ctdef, ins_rtdef, tablet_loc))) {
LOG_WARN("calc partition key failed", K(ret));
} else if (OB_FAIL(ObDMLService::insert_row(ins_ctdef, ins_rtdef, tablet_loc, replace_rtctx_, stored_row_))) {
LOG_WARN("insert row with das failed", K(ret));
} else {
LOG_TRACE("final insert one row", KPC(tablet_loc), "ins row", ROWEXPR2STR(eval_ctx_, ins_ctdef.new_row_));
}
}
return ret;
}
int ObTableReplaceOp::insert_row_to_das(bool &is_skipped)
{
int ret = OB_SUCCESS;
is_skipped = false;
// 尝试写入数据到所有的表
for (int64_t i = 0; OB_SUCC(ret) && i < MY_SPEC.replace_ctdefs_.count(); ++i) {
// insert each table with fetched row
@ -333,29 +430,23 @@ int ObTableReplaceOp::insert_row_to_das(bool need_do_trigger)
ObDASTabletLoc *tablet_loc = nullptr;
ObDMLModifyRowNode modify_row(this, &ins_ctdef, &ins_rtdef, ObDmlEventType::DE_INSERTING);
++ins_rtdef.cur_row_num_;
if (need_do_trigger &&
OB_FAIL(ObDMLService::init_heap_table_pk_for_ins(ins_ctdef, eval_ctx_))) {
if (OB_FAIL(ObDMLService::init_heap_table_pk_for_ins(ins_ctdef, eval_ctx_))) {
LOG_WARN("fail to init heap table pk to null", K(ret));
} else if (need_do_trigger &&
OB_FAIL(ObDMLService::process_insert_row(
ins_ctdef, ins_rtdef, *this, is_skipped))) {
} else if (OB_FAIL(ObDMLService::process_insert_row(ins_ctdef, ins_rtdef, *this, is_skipped))) {
LOG_WARN("process insert row failed", K(ret));
} else if (OB_UNLIKELY(is_skipped)) {
break;
} else if (OB_FAIL(calc_insert_tablet_loc(ins_ctdef, ins_rtdef, tablet_loc))) {
LOG_WARN("calc partition key failed", K(ret));
} else if (need_do_trigger &&
OB_FAIL(ObDMLService::set_heap_table_hidden_pk(ins_ctdef, tablet_loc->tablet_id_, eval_ctx_))) {
} else if (OB_FAIL(ObDMLService::set_heap_table_hidden_pk(ins_ctdef, tablet_loc->tablet_id_, eval_ctx_))) {
LOG_WARN("set_heap_table_hidden_pk failed", K(ret), KPC(tablet_loc), K(ins_ctdef));
} else if (OB_FAIL(ObDMLService::insert_row(ins_ctdef, ins_rtdef, tablet_loc, dml_rtctx_, modify_row.new_row_))) {
LOG_WARN("insert row with das failed", K(ret));
// TODO(yikang): fix trigger related for heap table
} else if (need_do_trigger && need_after_row_process(ins_ctdef) && OB_FAIL(dml_modify_rows_.push_back(modify_row))) {
} else if (need_after_row_process(ins_ctdef) && OB_FAIL(dml_modify_rows_.push_back(modify_row))) {
LOG_WARN("failed to push dml modify row to modified row list", K(ret));
} else {
LOG_DEBUG("insert one row", KPC(tablet_loc), "ins row",
ROWEXPR2STR(eval_ctx_, ins_ctdef.new_row_));
LOG_TRACE("insert one row", KPC(tablet_loc), "ins row", ROWEXPR2STR(eval_ctx_, ins_ctdef.new_row_));
}
}
return ret;
@ -381,22 +472,46 @@ int ObTableReplaceOp::delete_row_to_das(bool need_do_trigger)
break;
} else if (OB_FAIL(calc_delete_tablet_loc(del_ctdef, del_rtdef, tablet_loc))) {
LOG_WARN("calc partition key failed", K(ret));
} else if (OB_FAIL(ObDMLService::delete_row(del_ctdef, del_rtdef, tablet_loc, dml_rtctx_, stored_row))) {
} else if (OB_FAIL(ObDMLService::delete_row(del_ctdef, del_rtdef, tablet_loc, replace_rtctx_, stored_row))) {
LOG_WARN("insert row with das failed", K(ret));
} else {
LOG_DEBUG("delete one row", KPC(tablet_loc), "del row",
ROWEXPR2STR(eval_ctx_, del_ctdef.old_row_));
LOG_DEBUG("delete one row", KPC(tablet_loc), "del row", ROWEXPR2STR(eval_ctx_, del_ctdef.old_row_));
}
}
return ret;
}
int ObTableReplaceOp::fetch_conflict_rowkey()
int ObTableReplaceOp::fetch_conflict_rowkey(int64_t replace_row_cnt)
{
int ret = OB_SUCCESS;
bool got_row = false;
NG_TRACE_TIMES(2, replace_start_lookup);
NG_TRACE_TIMES(2, replace_build_fetch_rowkey);
DASTaskIter task_iter = dml_rtctx_.das_ref_.begin_task_iter();
// 暂时拍脑袋定个100行
if (replace_row_cnt > ObConflictCheckerCtdef::MIN_ROW_COUNT_USE_HASHSET_DO_DISTICT) {
if (OB_FAIL(conflict_checker_.create_rowkey_check_hashset(replace_row_cnt))) {
LOG_WARN("fail to create conflict_checker hash_set", K(ret), K(replace_row_cnt));
}
}
if (gts_state_ == USE_PARTITION_SNAPSHOT_STATE) {
DASTaskIter task_iter = dml_rtctx_.das_ref_.begin_task_iter();
while (OB_SUCC(ret) && !task_iter.is_end()) {
ObDASInsertOp *ins_op = static_cast<ObDASInsertOp*>(*task_iter);
const ObDASInsCtDef *ins_ctdef = static_cast<const ObDASInsCtDef*>(ins_op->get_ctdef());
transaction::ObTxReadSnapshot *snapshot = ins_op->get_das_gts_opt_info().get_response_snapshot();
if (OB_ISNULL(snapshot)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null", K(ret));
} else if (!snapshot->is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected invalid snaopshot", K(ret), KPC(snapshot));
} else if (OB_FAIL(conflict_checker_.collect_all_snapshot(*snapshot, ins_op->get_tablet_loc()))) {
LOG_WARN("fail to collect snapshot", K(ret), KPC(snapshot), KPC(ins_op));
}
++task_iter;
}
}
while (OB_SUCC(ret) && !task_iter.is_end()) {
// 不需要clear rowkey表达式的eval_flag，因为主键使用的是column_ref表达式，不存在eval_fun
if (OB_FAIL(get_next_conflict_rowkey(task_iter))) {
@ -444,11 +559,9 @@ int ObTableReplaceOp::get_next_conflict_rowkey(DASTaskIter &task_iter)
} else if (OB_ISNULL(stored_row = ssr.get_store_row())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("stored row is null", K(ret));
} else if (OB_FAIL(stored_row->to_expr(
conflict_checker_.checker_ctdef_.data_table_rowkey_expr_,conflict_checker_.eval_ctx_))) {
if (OB_ITER_END != ret) {
LOG_WARN("get next row from result iterator failed", K(ret));
}
} else if (OB_FAIL(stored_row->to_expr(conflict_checker_.checker_ctdef_.data_table_rowkey_expr_,
conflict_checker_.eval_ctx_))) {
LOG_WARN("get next row from result iterator failed", K(ret));
} else {
got_row = true;
}
@ -456,53 +569,111 @@ int ObTableReplaceOp::get_next_conflict_rowkey(DASTaskIter &task_iter)
return ret;
}
int ObTableReplaceOp::post_all_dml_das_task()
int ObTableReplaceOp::post_all_try_insert_das_task(ObDMLRtCtx &dml_rtctx)
{
int ret = OB_SUCCESS;
NG_TRACE_TIMES(2, replace_try_insert);
if (dml_rtctx_.das_ref_.has_task()) {
if (OB_FAIL(dml_rtctx_.das_ref_.execute_all_task())) {
if (dml_rtctx.das_ref_.has_task()) {
if (gts_state_ == WITH_UNIQUE_GLOBAL_INDEX_STATE) {
share::ObLSID ls_id;
ObSQLSessionInfo *my_session = GET_MY_SESSION(ctx_);
ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(ctx_);
if (dml_rtctx.das_ref_.check_tasks_same_ls_and_is_local(ls_id)) {
if (OB_FAIL(ObSqlTransControl::get_ls_read_snapshot(my_session,
plan_ctx,
ls_id,
ctx_.get_das_ctx().get_snapshot()))) {
LOG_WARN("fail to get ls read snapshot", K(ret));
} else {
LOG_TRACE("all tasks with same ls_id, get ls snapshot", K(ls_id), K(ctx_.get_das_ctx().get_snapshot()));
}
} else {
if (OB_FAIL(ObSqlTransControl::get_read_snapshot(my_session,
plan_ctx,
ctx_.get_das_ctx().get_snapshot()))) {
LOG_WARN("fail to get global read snapshot", K(ret));
} else {
gts_state_ = GTE_GTS_STATE;
LOG_TRACE("task in different ls_id, get gts", K(ctx_.get_das_ctx().get_snapshot()));
}
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(dml_rtctx.das_ref_.execute_all_task())) {
LOG_WARN("execute all delete das task failed", K(ret));
}
}
}
return ret;
}
int ObTableReplaceOp::post_all_dml_das_task(ObDMLRtCtx &dml_rtctx)
{
int ret = OB_SUCCESS;
NG_TRACE_TIMES(2, replace_final_write);
if (dml_rtctx.das_ref_.has_task()) {
if (gts_state_ == USE_PARTITION_SNAPSHOT_STATE) {
if (OB_FAIL(conflict_checker_.set_partition_snapshot_for_das_task(dml_rtctx.das_ref_))) {
LOG_WARN("fail to set partition snapshot", K(ret));
}
}
if (OB_FAIL(ret)) {
// do nothing
} else if (OB_FAIL(dml_rtctx.das_ref_.execute_all_task())) {
LOG_WARN("execute all delete das task failed", K(ret));
}
}
return ret;
}
int ObTableReplaceOp::rollback_savepoint(const transaction::ObTxSEQ &savepoint_no)
{
int ret = OB_SUCCESS;
NG_TRACE_TIMES(2, replace_start_rollback);
if (OB_FAIL(ObSqlTransControl::rollback_savepoint(ctx_, savepoint_no))) {
LOG_WARN("fail to rollback to save_point", K(ret), K(savepoint_no));
}
return ret;
}
int ObTableReplaceOp::do_replace_into()
{
int ret = OB_SUCCESS;
bool is_iter_end = false;
while (OB_SUCC(ret) && !is_iter_end) {
transaction::ObTxSEQ savepoint_no;
int64_t start_time = 0;
int64_t end_time = 0;
// must set conflict_row fetch flag
add_need_conflict_result_flag();
NG_TRACE_TIMES(2, replace_load_all_row);
if (OB_FAIL(ObSqlTransControl::create_anonymous_savepoint(ctx_, savepoint_no))) {
LOG_WARN("fail to create save_point", K(ret));
} else if (OB_FAIL(load_all_replace_row(is_iter_end))) {
LOG_WARN("fail to load all row", K(ret));
} else if (OB_FAIL(post_all_dml_das_task())) {
} else if (OB_FAIL(post_all_try_insert_das_task(dml_rtctx_))) {
LOG_WARN("fail to post all das task", K(ret));
} else if (!check_is_duplicated() && OB_FAIL(ObDMLService::handle_after_row_processing(this, &dml_modify_rows_))) {
LOG_WARN("try insert is not duplicated, failed to process foreign key handle", K(ret));
} else if (!check_is_duplicated()) {
LOG_DEBUG("try insert is not duplicated", K(ret));
} else if (OB_FAIL(fetch_conflict_rowkey())) {
} else if (OB_FAIL(fetch_conflict_rowkey(replace_row_store_.get_row_cnt()))) {
LOG_WARN("fail to fetch conflict row", K(ret));
} else if (OB_FAIL(reset_das_env())) {
// 这里需要reuse das 相关信息
LOG_WARN("fail to reset das env", K(ret));
} else if (OB_FAIL(ObSqlTransControl::rollback_savepoint(ctx_, savepoint_no))) {
} else if (OB_FAIL(rollback_savepoint(savepoint_no))) {
// 本次插入存在冲突, 回滚到save_point
LOG_WARN("fail to rollback to save_point", K(ret));
} else if (OB_FAIL(conflict_checker_.do_lookup_and_build_base_map(
replace_row_store_.get_row_cnt()))) {
LOG_WARN("fail to rollback to save_point", K(ret), K(savepoint_no));
} else if (OB_FAIL(conflict_checker_.do_lookup_and_build_base_map(replace_row_store_.get_row_cnt()))) {
LOG_WARN("fail to do table lookup", K(ret));
} else if (OB_FAIL(replace_conflict_row_cache())) {
LOG_WARN("fail to shuff all replace row", K(ret));
} else if (OB_FAIL(prepare_final_replace_task())) {
LOG_WARN("fail to prepare final das task", K(ret));
} else if (OB_FAIL(post_all_dml_das_task())) {
} else if (OB_FAIL(post_all_dml_das_task(replace_rtctx_))) {
LOG_WARN("do insert rows post process failed", K(ret));
} else if (OB_FAIL(ObDMLService::handle_after_row_processing(this, &dml_modify_rows_))) {
LOG_WARN("try insert is duplicated, failed to process foreign key handle", K(ret));
@ -545,7 +716,7 @@ int ObTableReplaceOp::replace_conflict_row_cache()
ObReplaceRtDef &replace_rtdef = replace_rtdefs_.at(0);
ObInsRtDef &ins_rtdef = replace_rtdef.ins_rtdef_;
ObDelRtDef &del_rtdef = replace_rtdef.del_rtdef_;
const ObChunkDatumStore::StoredRow *replace_row = NULL;
const ObChunkDatumStore::StoredRow *stored_row = NULL;
NG_TRACE_TIMES(2, replace_start_shuff);
if (OB_FAIL(replace_row_store_.begin(replace_row_iter))) {
@ -553,16 +724,25 @@ int ObTableReplaceOp::replace_conflict_row_cache()
}
// 构建冲突的hash map的时候也使用的是column_ref， 回表也是使用的column_ref expr来读取scan的结果
// 因为constarain_info中使用的column_ref expr，所以此处需要使用table_column_old_exprs (column_ref exprs)
while (OB_SUCC(ret) && OB_SUCC(replace_row_iter.get_next_row(replace_row))) {
while (OB_SUCC(ret) && OB_SUCC(replace_row_iter.get_next_row(stored_row))) {
int64_t delete_rows = 0;
int64_t insert_rows = 0;
constraint_values.reuse();
ObChunkDatumStore::StoredRow *insert_new_row = NULL;
if (OB_ISNULL(replace_row)) {
if (OB_ISNULL(stored_row)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("replace_row is null", K(ret));
} else if (OB_FAIL(conflict_checker_.check_duplicate_rowkey(replace_row,
} else if (OB_FAIL(stored_row->to_expr(get_all_saved_exprs(), eval_ctx_))) {
LOG_WARN("flush replace_row to exprs failed", K(ret), KPC(stored_row));
} else if (OB_FAIL(conflict_checker_.convert_exprs_to_stored_row(get_primary_table_new_row(),
insert_new_row))) {
LOG_WARN("convert exprs to stored_row failed", K(ret), KPC(insert_new_row));
} else if (OB_FAIL(conflict_checker_.check_duplicate_rowkey(insert_new_row,
constraint_values,
false))) {
LOG_WARN("check rowkey from conflict_checker failed", K(ret), KPC(replace_row));
LOG_WARN("check rowkey from conflict_checker failed", K(ret), KPC(insert_new_row));
} else {
insert_rows++;
}
for (int64_t i = 0; OB_SUCC(ret) && i < constraint_values.count(); ++i) {
@ -581,8 +761,8 @@ int ObTableReplaceOp::replace_conflict_row_cache()
} else if (OB_FAIL(conflict_checker_.delete_old_row(delete_row, ObNewRowSource::FROM_INSERT))) {
LOG_WARN("delete old_row from conflict checker failed", K(ret), KPC(delete_row));
} else if (MY_SPEC.only_one_unique_key_) {
if (OB_FAIL(check_values(same_row, replace_row, delete_row))) {
LOG_WARN("check value failed", K(ret), KPC(replace_row), KPC(delete_row));
if (OB_FAIL(check_values(same_row, insert_new_row, delete_row))) {
LOG_WARN("check value failed", K(ret), KPC(insert_new_row), KPC(delete_row));
}
}
if (OB_SUCC(ret)) {
@ -592,21 +772,17 @@ int ObTableReplaceOp::replace_conflict_row_cache()
}
}
if (OB_SUCC(ret) && !same_row) {
delete_rows_++;
delete_rows++;
}
}
if (OB_FAIL(ret)) {
// do nothing
} else if (OB_FAIL(replace_row->to_expr(get_primary_table_new_row(), eval_ctx_))) {
LOG_WARN("flush replace_row to exprs failed", K(ret), KPC(replace_row));
} else if (OB_FAIL(ObDMLService::process_insert_row(ins_ctdef, ins_rtdef, *this, is_skipped))) {
LOG_WARN("convert exprs to stored_row failed", K(ret), KPC(insert_new_row));
} else if (OB_UNLIKELY(is_skipped)) {
continue;
} else if (OB_FAIL(conflict_checker_.convert_exprs_to_stored_row(get_primary_table_new_row(),
insert_new_row))) {
LOG_WARN("convert exprs to stored_row failed", K(ret), KPC(insert_new_row));
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected skipped", K(ret), KPC(insert_new_row));
} else if (OB_FAIL(conflict_checker_.insert_new_row(insert_new_row, ObNewRowSource::FROM_INSERT))) {
LOG_WARN("insert new to conflict_checker failed", K(ret), KPC(insert_new_row));
}
@ -615,8 +791,11 @@ int ObTableReplaceOp::replace_conflict_row_cache()
modify_row.new_row_ = insert_new_row;
if (need_after_row_process(ins_ctdef) && OB_FAIL(dml_modify_rows_.push_back(modify_row))) {
LOG_WARN("failed to push dml modify row to modified row list", K(ret));
} else if (OB_FAIL(replace_implict_cursor(insert_rows + delete_rows, 0, insert_rows, delete_rows))) {
LOG_WARN("merge implict cursor failed", K(ret));
}
}
delete_rows_ += delete_rows;
} // while row store end
ret = OB_ITER_END == ret ? OB_SUCCESS : ret;
return ret;
@ -626,7 +805,6 @@ int ObTableReplaceOp::prepare_final_replace_task()
{
int ret = OB_SUCCESS;
ObConflictRowMap *primary_map = NULL;
NG_TRACE_TIMES(2, replace_final_shuff);
OZ(conflict_checker_.get_primary_table_map(primary_map));
CK(OB_NOT_NULL(primary_map));
OZ(do_delete(primary_map));
@ -642,7 +820,7 @@ int ObTableReplaceOp::do_delete(ObConflictRowMap *primary_map)
for (; OB_SUCC(ret) && start_row_iter != end_row_iter; ++start_row_iter) {
clear_datum_eval_flag();
ObConflictValue &constraint_value = start_row_iter->second;
LOG_DEBUG("get one constraint_value from primary hash map", K(constraint_value));
LOG_TRACE("get one constraint_value from primary hash map", K(constraint_value));
if (NULL != constraint_value.baseline_datum_row_) {
//baseline row is not empty, delete it
if (OB_FAIL(constraint_value.baseline_datum_row_->to_expr(
@ -672,7 +850,7 @@ int ObTableReplaceOp::do_insert(ObConflictRowMap *primary_map)
if (OB_FAIL(constraint_value.current_datum_row_->to_expr(
get_primary_table_new_row(), eval_ctx_))) {
LOG_WARN("stored row to expr faild", K(ret));
} else if (OB_FAIL(insert_row_to_das(false))) {
} else if (OB_FAIL(final_insert_row_to_das())) {
LOG_WARN("shuffle insert row failed", K(ret), K(constraint_value));
} else {
LOG_TRACE("insert one row from primary hash map",
@ -752,6 +930,16 @@ int ObTableReplaceOp::check_replace_ctdefs_valid() const
return ret;
}
const ObIArray<ObExpr *> &ObTableReplaceOp::get_all_saved_exprs()
{
// During the upgrade, all_saved_exprs_ in the previous version is empty.
// When this node is used as a remote execution node of the plan,
// all_saved_exprs_ is also empty after the partition_wise plan is deserialized.
return MY_SPEC.all_saved_exprs_.empty() ?
MY_SPEC.replace_ctdefs_.at(0)->ins_ctdef_->new_row_ :
MY_SPEC.all_saved_exprs_;
}
const ObIArray<ObExpr *> &ObTableReplaceOp::get_primary_table_new_row()
{
return MY_SPEC.replace_ctdefs_.at(0)->ins_ctdef_->new_row_;
@ -794,6 +982,13 @@ int ObTableReplaceOp::reuse()
dml_rtctx_.das_ref_.reuse();
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(replace_rtctx_.das_ref_.close_all_task())) {
LOG_WARN("close all update das task failed", K(ret));
} else {
replace_rtctx_.das_ref_.reuse();
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(conflict_checker_.reuse())) {
LOG_WARN("fail to reuse conflict checker", K(ret));

24
src/sql/engine/dml/ob_table_replace_op.h
View File

@ -75,7 +75,9 @@ public:
conflict_checker_(ctx.get_allocator(),
eval_ctx_,
MY_SPEC.conflict_checker_ctdef_),
replace_row_store_("ReplaceRow")
replace_row_store_("ReplaceRow"),
replace_rtctx_(eval_ctx_, ctx, *this),
gts_state_(WITHOUT_GTS_OPT_STATE)
{}
virtual ~ObTableReplaceOp() {}
@ -89,11 +91,12 @@ public:
int get_next_row_from_child();
// 执行所有尝试插入的 das task， fetch冲突行的主表主键
int fetch_conflict_rowkey();
int fetch_conflict_rowkey(int64_t replace_row_cnt);
int get_next_conflict_rowkey(DASTaskIter &task_iter);
virtual void destroy()
{
replace_rtctx_.cleanup();
conflict_checker_.destroy();
replace_rtdefs_.release_array();
replace_row_store_.~ObChunkDatumStore();
@ -104,6 +107,8 @@ protected:
int do_replace_into();
int rollback_savepoint(const transaction::ObTxSEQ &savepoint_no);
// 检查是否有duplicated key 错误发生
bool check_is_duplicated();
@ -118,11 +123,16 @@ protected:
int do_insert(ObConflictRowMap *primary_map);
// 提交所有的 try_insert 的 das task;
int post_all_try_insert_das_task(ObDMLRtCtx &dml_rtctx);
// 提交所有的 insert 和delete das task;
int post_all_dml_das_task();
int post_all_dml_das_task(ObDMLRtCtx &dml_rtctx);
// batch的执行插入 process_row and then write to das,
int insert_row_to_das(bool need_do_trigger);
int insert_row_to_das(bool &is_skipped);
int final_insert_row_to_das();
// batch的执行插入 process_row and then write to das,
int delete_row_to_das(bool need_do_trigger);
@ -142,9 +152,12 @@ protected:
const ObChunkDatumStore::StoredRow *replace_row,
const ObChunkDatumStore::StoredRow *delete_row);
virtual int check_need_exec_single_row() override;
virtual ObDasParallelType check_das_parallel_type() override;
private:
int check_replace_ctdefs_valid() const;
const ObIArray<ObExpr *> &get_all_saved_exprs();
const ObIArray<ObExpr *> &get_primary_table_new_row();
const ObIArray<ObExpr *> &get_primary_table_old_row();
@ -157,12 +170,15 @@ private:
protected:
const static int64_t DEFAULT_REPLACE_BATCH_ROW_COUNT = 1000L;
const static int64_t OB_DEFAULT_REPLACE_MEMORY_LIMIT = 2 * 1024 * 1024L;
int64_t insert_rows_;
int64_t delete_rows_;
ObConflictChecker conflict_checker_;
common::ObArrayWrap<ObReplaceRtDef> replace_rtdefs_;
ObChunkDatumStore replace_row_store_; //所有的replace的行的集合
ObDMLRtCtx replace_rtctx_;
ObDmlGTSOptState gts_state_;
};
class ObTableReplaceOpInput : public ObTableModifyOpInput

5
src/sql/engine/dml/ob_table_update_op.cpp
View File

@ -381,7 +381,10 @@ OB_INLINE int ObTableUpdateOp::update_row_to_das()
} //end for global index ctdef loop
if (OB_SUCC(ret)) {
int64_t update_rows = rtdefs.at(0).is_row_changed_ ? 1 : 0;
if (OB_FAIL(merge_implict_cursor(0, update_rows, 0, 1))) {
ObSQLSessionInfo *session = GET_MY_SESSION(ctx_);
bool client_found_rows = session->get_capability().cap_flags_.OB_CLIENT_FOUND_ROWS;
int64_t affected_rows = client_found_rows ? 1 /*found_rows*/ : update_rows;
if (OB_FAIL(merge_implict_cursor(affected_rows, 1 /*found_rows*/, 1 /*match_rows*/, update_rows /*duplicated_rows*/))) {
LOG_WARN("merge implict cursor failed", K(ret));
}
}

11
src/sql/engine/ob_exec_context.cpp
View File

@ -755,6 +755,8 @@ int ObExecContext::init_physical_plan_ctx(const ObPhysicalPlan &plan)
{
int ret = OB_SUCCESS;
int64_t foreign_key_checks = 0;
uint64_t tenant_data_version = 0;
bool supprt_check_pdml_affected_row = false;
if (OB_ISNULL(phy_plan_ctx_) || OB_ISNULL(my_session_) || OB_ISNULL(sql_ctx_)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K_(phy_plan_ctx), K_(my_session), K(ret));
@ -805,6 +807,15 @@ int ObExecContext::init_physical_plan_ctx(const ObPhysicalPlan &plan)
phy_plan_ctx_->set_ignore_stmt(plan.is_ignore());
phy_plan_ctx_->set_foreign_key_checks(0 != foreign_key_checks);
phy_plan_ctx_->set_table_row_count_list_capacity(plan.get_access_table_num());
if (plan.is_use_pdml() && GCONF.enable_defensive_check()) {
if (OB_FAIL(GET_MIN_DATA_VERSION(my_session_->get_effective_tenant_id(), tenant_data_version))) {
LOG_WARN("get tenant data version failed", K(ret));
} else if ((DATA_VERSION_4_3_5_0 <= tenant_data_version)) {
// ([4.3.5, ...)) support check pdml affected_rows
supprt_check_pdml_affected_row = true;
}
}
phy_plan_ctx_->set_check_pdml_affected_rows(supprt_check_pdml_affected_row);
THIS_WORKER.set_timeout_ts(phy_plan_ctx_->get_timeout_timestamp());
#ifdef OB_BUILD_SPM
if (sql_ctx_ != NULL && sql_ctx_->spm_ctx_.need_spm_timeout_) {

1
src/sql/engine/ob_exec_feedback_info.cpp
View File

@ -57,6 +57,7 @@ int ObExecFeedbackInfo::merge_feedback_info(const ObExecFeedbackInfo &feedback_i
max(fb_nodes.at(right).op_close_time_, nodes_.at(left).op_close_time_);
nodes_.at(left).db_time_ = max(fb_nodes.at(right).db_time_, nodes_.at(left).db_time_);
nodes_.at(left).output_row_count_ += fb_nodes.at(right).output_row_count_;
nodes_.at(left).pdml_op_write_rows_ += fb_nodes.at(right).pdml_op_write_rows_;
nodes_.at(left).worker_count_ += fb_nodes.at(right).worker_count_;
left++;
right++;

2
src/sql/engine/ob_exec_feedback_info.h
View File

@ -36,7 +36,7 @@ public:
~ObExecFeedbackNode() {}
TO_STRING_KV(K_(op_id), K_(output_row_count), K_(op_open_time),
K_(op_close_time), K_(op_first_row_time), K_(op_last_row_time),
K_(db_time), K_(block_time));
K_(db_time), K_(block_time), K_(pdml_op_write_rows));
public:
int64_t op_id_;
int64_t output_row_count_;

15
src/sql/engine/ob_operator.cpp
View File

@ -461,10 +461,11 @@ int ObOpSpec::create_exec_feedback_node_recursive(ObExecContext &exec_ctx) const
{
int ret = OB_SUCCESS;
ObOperatorKit *kit = exec_ctx.get_operator_kit(id_);
if (OB_ISNULL(plan_)) {
ObPhysicalPlanCtx *physical_ctx = exec_ctx.get_physical_plan_ctx();
if (OB_ISNULL(plan_) || OB_ISNULL(physical_ctx)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("phy plan is null", K(ret));
} else if (!plan_->need_record_plan_info()) {
LOG_WARN("phy plan or ctx is null", K(ret), K(plan_), K(physical_ctx));
} else if (!physical_ctx->get_check_pdml_affected_rows() && !plan_->need_record_plan_info()) {
} else if (OB_ISNULL(kit)) {
LOG_TRACE("operator kit is NULL", K(ret));
} else {
@ -1155,10 +1156,11 @@ int ObOperator::close()
int ObOperator::setup_op_feedback_info()
{
int ret = OB_SUCCESS;
if (OB_ISNULL(spec_.plan_)) {
ObPhysicalPlanCtx *phy_ctx = ctx_.get_physical_plan_ctx();
if (OB_ISNULL(spec_.plan_) || OB_ISNULL(phy_ctx)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("phy plan is null", K(ret));
} else if (!spec_.plan_->need_record_plan_info() ||
} else if ((!spec_.plan_->need_record_plan_info() && !phy_ctx->get_check_pdml_affected_rows()) ||
OB_INVALID_INDEX == fb_node_idx_) {
} else {
ObExecFeedbackInfo &fb_info = ctx_.get_feedback_info();
@ -1178,6 +1180,9 @@ int ObOperator::setup_op_feedback_info()
node.op_open_time_ = op_monitor_info_.open_time_;
node.output_row_count_ = op_monitor_info_.output_row_count_;
node.worker_count_ = 1;
if (spec_.is_pdml_operator()) {
node.pdml_op_write_rows_ = op_monitor_info_.otherstat_6_value_;
}
}
}
return ret;

104
src/sql/engine/ob_physical_plan.cpp
View File

@ -35,6 +35,7 @@
#include "sql/engine/ob_exec_feedback_info.h"
#include "sql/engine/expr/ob_expr_sql_udt_utils.h"
#include "sql/code_generator/ob_static_engine_cg.h"
#include "sql/monitor/ob_sql_plan.h"
namespace oceanbase
{
@ -137,6 +138,7 @@ ObPhysicalPlan::ObPhysicalPlan(MemoryContext &mem_context /* = CURRENT_CONTEXT *
is_enable_px_fast_reclaim_(false),
use_rich_format_(false),
subschema_ctx_(allocator_),
das_dop_(0),
disable_auto_memory_mgr_(false),
is_inner_sql_(false),
is_batch_params_execute_(false),
@ -151,7 +153,8 @@ ObPhysicalPlan::ObPhysicalPlan(MemoryContext &mem_context /* = CURRENT_CONTEXT *
need_switch_to_table_lock_worker_(false),
data_complement_gen_doc_id_(false),
dml_table_ids_(&allocator_),
direct_load_need_sort_(false)
direct_load_need_sort_(false),
insertup_can_do_gts_opt_(false)
{
}
@ -246,6 +249,7 @@ void ObPhysicalPlan::reset()
logical_plan_.reset();
is_enable_px_fast_reclaim_ = false;
subschema_ctx_.reset();
das_dop_ = 0;
all_local_session_vars_.reset();
sql_stat_record_value_.reset();
udf_has_dml_stmt_ = false;
@ -261,6 +265,7 @@ void ObPhysicalPlan::reset()
data_complement_gen_doc_id_ = false;
dml_table_ids_.reset();
direct_load_need_sort_ = false;
insertup_can_do_gts_opt_ = false;
}
void ObPhysicalPlan::destroy()
{
@ -1443,6 +1448,103 @@ int ObPhysicalPlan::set_logical_plan(ObLogicalPlanRawData &logical_plan)
return ret;
}
int ObPhysicalPlan::get_all_spec_op(ObIArray<const ObOpSpec *> &simple_op_infos, const ObOpSpec &op)
{
int ret = OB_SUCCESS;
if (OB_FAIL(simple_op_infos.push_back(&op))) {
LOG_WARN("fail to push back spec_op", K(ret), K(op.get_id()));
}
for (int32_t i = 0; OB_SUCC(ret) && i < op.get_child_num(); ++i) {
const ObOpSpec *child_op = op.get_child(i);
if (OB_ISNULL(child_op)) {
ret = OB_ERR_UNEXPECTED;
} else if (OB_FAIL(SMART_CALL(get_all_spec_op(simple_op_infos, *child_op)))) {
LOG_WARN("fail to find child ops",
K(ret), K(i), "op_id", op.get_id(), "child_id", child_op->get_id());
}
}
return ret;
}
int ObPhysicalPlan::print_this_plan_info(ObExecContext &ctx)
{
int ret = OB_SUCCESS;
PlanText out_plan_text;
ObExplainDisplayOpt option;
ObSEArray<common::ObString, 32> plan_strs;
option.with_tree_line_ = false;
ObSqlPlan sql_plan(ctx.get_allocator());
ExplainType type = EXPLAIN_EXTENDED;
ObSEArray<ObSqlPlanItem*, 4> plan_items;
if (OB_FAIL(logical_plan_.uncompress_logical_plan(ctx.get_allocator(), plan_items))) {
LOG_WARN("failed to uncompress logical plan", K(ret));
} else if (OB_FAIL(sql_plan.format_sql_plan(plan_items,
type,
option,
out_plan_text))) {
LOG_WARN("failed to format sql plan", K(ret));
} else if (OB_FAIL(sql_plan.plan_text_to_strings(out_plan_text, plan_strs))) {
LOG_WARN("failed to convert plan text to strings", K(ret));
} else {
LOG_INFO("print plan info:");
}
for (int64_t i = 0; OB_SUCC(ret) && i < plan_strs.count(); i++) {
_OB_LOG(INFO, "%*s", plan_strs.at(i).length(), plan_strs.at(i).ptr());
}
return ret;
}
int ObPhysicalPlan::check_pdml_affected_rows(ObExecContext &ctx)
{
int ret = OB_SUCCESS;
ObSEArray<const ObOpSpec *, 16> all_ops;
ObExecFeedbackInfo &fb_info = ctx.get_feedback_info();
const common::ObIArray<ObExecFeedbackNode> &feedback_nodes = fb_info.get_feedback_nodes();
if (!is_use_pdml()) {
// do nothing
} else if (OB_FAIL(get_all_spec_op(all_ops, *root_op_spec_))) {
LOG_WARN("fail to get all spec", K(ret));
} else if (!fb_info.is_valid()) {
// do nothing
} else if (all_ops.count() != feedback_nodes.count()) {
// do nothing
} else {
int64_t pdml_write_rows = 0;
bool set_pdml_write_rows = false;
for (int64_t i = 0; OB_SUCC(ret) && i < all_ops.count(); ++i) {
const ObExecFeedbackNode &feedback_node = feedback_nodes.at(i);
const ObOpSpec *op = all_ops.at(i);
if (OB_ISNULL(op)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null ptr", K(ret));
} else if (op->id_ != feedback_node.op_id_) {
// do nothing
} else if (op->is_pdml_operator()) {
if (!set_pdml_write_rows) {
set_pdml_write_rows = true;
pdml_write_rows = feedback_node.pdml_op_write_rows_;
} else if (pdml_write_rows != feedback_node.pdml_op_write_rows_) {
ret = OB_ERR_DEFENSIVE_CHECK;
ObString func_name = ObString::make_string("check_pdml_affected_row");
LOG_USER_ERROR(OB_ERR_DEFENSIVE_CHECK, func_name.length(), func_name.ptr());
LOG_DBA_ERROR(OB_ERR_DEFENSIVE_CHECK, "msg", "Fatal Error!!! pdml write affected row is not match with index table", K(ret),
"curr_pdml_op_write_rows", feedback_node.pdml_op_write_rows_,
"pdml_write_rows", pdml_write_rows,
"op_info", feedback_node,
K(feedback_nodes));
int tmp_ret = OB_SUCCESS;
if (OB_SUCCESS != (tmp_ret = print_this_plan_info(ctx))) {
LOG_WARN("failed to uncompress logical plan", K(tmp_ret));
}
}
}
}
}
return ret;
}
int ObPhysicalPlan::set_feedback_info(ObExecContext &ctx)
{
int ret = OB_SUCCESS;

11
src/sql/engine/ob_physical_plan.h
View File

@ -396,7 +396,8 @@ public:
}
inline double get_online_sample_percent() const { return online_sample_percent_; }
inline void set_online_sample_percent(double v) { online_sample_percent_ = v; }
int64_t get_das_dop() { return das_dop_; }
void set_das_dop(int64_t v) { das_dop_ = v; }
public:
int inc_concurrent_num();
void dec_concurrent_num();
@ -535,7 +536,9 @@ public:
inline ObLogicalPlanRawData& get_logical_plan() { return logical_plan_; }
inline const ObLogicalPlanRawData& get_logical_plan()const { return logical_plan_; }
int set_feedback_info(ObExecContext &ctx);
int check_pdml_affected_rows(ObExecContext &ctx);
int print_this_plan_info(ObExecContext &ctx);
int get_all_spec_op(ObIArray<const ObOpSpec *> &simple_op_infos, const ObOpSpec &root_op_spec);
void set_enable_px_fast_reclaim(bool value) { is_enable_px_fast_reclaim_ = value; }
bool is_enable_px_fast_reclaim() const { return is_enable_px_fast_reclaim_; }
ObSubSchemaCtx &get_subschema_ctx_for_update() { return subschema_ctx_; }
@ -551,6 +554,8 @@ public:
direct_load_need_sort_ = direct_load_need_sort;
}
bool get_direct_load_need_sort() const { return direct_load_need_sort_; }
inline bool get_insertup_can_do_gts_opt() const {return insertup_can_do_gts_opt_; }
inline void set_insertup_can_do_gts_opt(bool v) { insertup_can_do_gts_opt_ = v; }
void set_is_use_auto_dop(bool use_auto_dop) { stat_.is_use_auto_dop_ = use_auto_dop; }
bool get_is_use_auto_dop() const { return stat_.is_use_auto_dop_; }
public:
@ -733,6 +738,7 @@ public:
bool is_enable_px_fast_reclaim_;
bool use_rich_format_;
ObSubSchemaCtx subschema_ctx_;
int64_t das_dop_;
bool disable_auto_memory_mgr_;
bool is_inner_sql_;
bool is_batch_params_execute_;
@ -755,6 +761,7 @@ private:
// to decide whether it read uncommitted data
common::ObFixedArray<uint64_t, common::ObIAllocator> dml_table_ids_;
bool direct_load_need_sort_;
bool insertup_can_do_gts_opt_;
};
inline void ObPhysicalPlan::set_affected_last_insert_id(bool affected_last_insert_id)

18
src/sql/engine/ob_physical_plan_ctx.cpp
View File

@ -381,6 +381,23 @@ int ObPhysicalPlanCtx::merge_implicit_cursor_info(const ObImplicitCursorInfo &im
return ret;
}
int ObPhysicalPlanCtx::replace_implicit_cursor_info(const ObImplicitCursorInfo &implicit_cursor)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(implicit_cursor.stmt_id_ < 0)
|| OB_UNLIKELY(implicit_cursor.stmt_id_ >= implicit_cursor_infos_.count())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("implicit cursor is invalid", K(ret),
K(implicit_cursor), K(implicit_cursor_infos_.count()));
} else if (OB_FAIL(implicit_cursor_infos_.at(implicit_cursor.stmt_id_).
replace_cursor(implicit_cursor))) {
LOG_WARN("merge implicit cursor info failed", K(ret),
K(implicit_cursor), K(implicit_cursor_infos_.count()), K(implicit_cursor));
}
LOG_DEBUG("merge implicit cursor info", K(ret), K(implicit_cursor), K(lbt()));
return ret;
}
const ObIArray<int64_t> *ObPhysicalPlanCtx::get_part_param_idxs(int64_t part_id) const
{
const ObIArray<int64_t> *part_param_idxs = nullptr;
@ -997,7 +1014,6 @@ OB_DEF_DESERIALIZE(ObPhysicalPlanCtx)
OB_UNIS_DECODE(*local_vars);
}
}
// following is not deserialize, please add deserialize ahead.
if (OB_SUCC(ret) && array_group_count > 0 &&
datum_param_store_.count() == 0 &&

10
src/sql/engine/ob_physical_plan_ctx.h
View File

@ -405,6 +405,15 @@ public:
{
return is_or_expand_transformed_;
}
inline void set_check_pdml_affected_rows(const bool check_pdml_affected_rows)
{
check_pdml_affected_rows_ = check_pdml_affected_rows;
}
inline bool get_check_pdml_affected_rows() const
{
return check_pdml_affected_rows_;
}
/*
** 目前OB有语句级重试和语句内部重试，当我们遇到错误码OB_TRANSACTION_SET_VIOLATION,
** 不会重新执行sql，而是重新执行计划，此时plan_ctx有些变量需要重置，目前梳理只有下面几种，
@ -475,6 +484,7 @@ public:
{ return implicit_cursor_infos_; }
int merge_implicit_cursor_info(const ObImplicitCursorInfo &implicit_cursor_info);
int merge_implicit_cursors(const common::ObIArray<ObImplicitCursorInfo> &implicit_cursors);
int replace_implicit_cursor_info(const ObImplicitCursorInfo &implicit_cursor_info);
void reset_cursor_info();
void set_cur_stmt_id(int64_t cur_stmt_id) { cur_stmt_id_ = cur_stmt_id; }
int64_t get_cur_stmt_id() const { return cur_stmt_id_; }

11
src/sql/engine/pdml/static/ob_pdml_op_data_driver.cpp
View File

@ -45,7 +45,14 @@ int ObPDMLOpDataDriver::init(const ObTableModifySpec &spec,
op_monitor_info_.otherstat_1_value_ = 0;
op_monitor_info_.otherstat_1_id_ = ObSqlMonitorStatIds::PDML_PARTITION_FLUSH_TIME;
op_monitor_info_.otherstat_2_value_ = 0;
op_monitor_info_.otherstat_2_id_ = ObSqlMonitorStatIds::PDML_PARTITION_FLUSH_COUNT;
op_monitor_info_.otherstat_2_id_ = ObSqlMonitorStatIds::PDML_GET_ROW_COUNT_FROM_CHILD_OP;
op_monitor_info_.otherstat_3_value_ = 0;
op_monitor_info_.otherstat_3_id_ = ObSqlMonitorStatIds::PDML_WRITE_DAS_BUFF_ROW_COUNT;
op_monitor_info_.otherstat_4_value_ = 0;
op_monitor_info_.otherstat_4_id_ = ObSqlMonitorStatIds::PDML_SKIP_ROW_COUNT;
op_monitor_info_.otherstat_6_value_ = 0;
op_monitor_info_.otherstat_6_id_ = ObSqlMonitorStatIds::PDML_STORAGE_RETURN_ROW_COUNT;
if (OB_ISNULL(reader)
|| OB_ISNULL(writer)) {
ret = OB_ERR_UNEXPECTED;
@ -243,8 +250,6 @@ int ObPDMLOpDataDriver::write_partitions(ObExecContext &ctx)
} else if (OB_FAIL(cache_.get_part_id_array(tablet_id_array))) {
LOG_WARN("fail get part index iterator", K(ret));
} else {
// 调用存储层写接口的次数 (flush 的次数)
op_monitor_info_.otherstat_2_value_++;
// 消耗在存储层的总时间
TimingGuard g(op_monitor_info_.otherstat_1_value_);
// 按照分区逐个写入存储层

13
src/sql/engine/pdml/static/ob_px_multi_part_delete_op.cpp
View File

@ -145,6 +145,7 @@ int ObPxMultiPartDeleteOp::read_row(ObExecContext &ctx,
LOG_WARN("fail get next row from child", K(ret));
}
} else {
op_monitor_info_.otherstat_2_value_++;
// 每一次从child节点获得新的数据都需要进行清除计算标记
clear_evaluated_flag();
if (OB_FAIL(ObDMLService::process_delete_row(MY_SPEC.del_ctdef_, del_rtdef_, is_skipped, *this))) {
@ -169,6 +170,8 @@ int ObPxMultiPartDeleteOp::read_row(ObExecContext &ctx,
tablet_id = expr_datum.get_int();
LOG_DEBUG("get the part id", K(ret), K(expr_datum));
}
} else {
op_monitor_info_.otherstat_4_value_++;
}
}
return ret;
@ -202,12 +205,16 @@ int ObPxMultiPartDeleteOp::write_rows(ObExecContext &ctx,
LOG_WARN("delete row to das failed", K(ret));
} else if (OB_FAIL(discharge_das_write_buffer())) {
LOG_WARN("failed to submit all dml task when the buffer of das op is full", K(ret));
} else {
op_monitor_info_.otherstat_3_value_++;
}
}
if (OB_ITER_END == ret) {
if (OB_FAIL(submit_all_dml_task())) {
LOG_WARN("do delete rows post process failed", K(ret));
} else {
op_monitor_info_.otherstat_6_value_ += del_rtdef_.das_rtdef_.affected_rows_;
}
}
if (!(MY_SPEC.is_pdml_index_maintain_) && !(MY_SPEC.is_pdml_update_split_)) {
@ -219,8 +226,10 @@ int ObPxMultiPartDeleteOp::write_rows(ObExecContext &ctx,
plan_ctx->add_row_duplicated_count(del_rtdef_.das_rtdef_.affected_rows_);
}
LOG_TRACE("pdml delete ok", K(MY_SPEC.is_pdml_index_maintain_),
K(del_rtdef_.das_rtdef_.affected_rows_));
LOG_TRACE("pdml delete ok",K(MY_SPEC.is_pdml_index_maintain_),
K(del_rtdef_.das_rtdef_.affected_rows_), K(op_monitor_info_.otherstat_1_value_),
K(op_monitor_info_.otherstat_2_value_), K(op_monitor_info_.otherstat_3_value_),
K(op_monitor_info_.otherstat_4_value_), K(op_monitor_info_.otherstat_6_value_));
del_rtdef_.das_rtdef_.affected_rows_ = 0;
}
return ret;

11
src/sql/engine/pdml/static/ob_px_multi_part_insert_op.cpp
View File

@ -122,6 +122,7 @@ int ObPxMultiPartInsertOp::read_row(ObExecContext &ctx,
LOG_WARN("fail get next row from child", K(ret));
}
} else {
op_monitor_info_.otherstat_2_value_++;
// 每一次从child节点获得新的数据都需要进行清除计算标记
clear_evaluated_flag();
if (OB_FAIL(ObDMLService::process_insert_row(MY_SPEC.ins_ctdef_, ins_rtdef_, *this, is_skipped))) {
@ -145,6 +146,8 @@ int ObPxMultiPartInsertOp::read_row(ObExecContext &ctx,
tablet_id = expr_datum.get_int();
LOG_DEBUG("get the part id", K(ret), K(expr_datum));
}
} else {
op_monitor_info_.otherstat_4_value_++;
}
}
if (OB_SUCC(ret)) {
@ -180,12 +183,16 @@ int ObPxMultiPartInsertOp::write_rows(ObExecContext &ctx,
LOG_WARN("insert row to das failed", K(ret));
} else if (OB_FAIL(discharge_das_write_buffer())) {
LOG_WARN("failed to submit all dml task when the buffer of das op is full", K(ret));
} else {
op_monitor_info_.otherstat_3_value_++;
}
}
if (OB_ITER_END == ret) {
if (OB_FAIL(submit_all_dml_task())) {
LOG_WARN("do insert rows post process failed", K(ret));
} else {
op_monitor_info_.otherstat_6_value_ += ins_rtdef_.das_rtdef_.affected_rows_;
}
}
if (!(MY_SPEC.is_pdml_index_maintain_)) {
@ -193,7 +200,9 @@ int ObPxMultiPartInsertOp::write_rows(ObExecContext &ctx,
plan_ctx->add_affected_rows(ins_rtdef_.das_rtdef_.affected_rows_);
}
LOG_TRACE("pdml insert ok", K(MY_SPEC.is_pdml_index_maintain_),
K(ins_rtdef_.das_rtdef_.affected_rows_));
K(ins_rtdef_.das_rtdef_.affected_rows_), K(op_monitor_info_.otherstat_1_value_),
K(op_monitor_info_.otherstat_2_value_), K(op_monitor_info_.otherstat_3_value_),
K(op_monitor_info_.otherstat_4_value_), K(op_monitor_info_.otherstat_6_value_));
ins_rtdef_.das_rtdef_.affected_rows_ = 0;
}
return ret;

25
src/sql/engine/pdml/static/ob_px_multi_part_update_op.cpp
View File

@ -135,6 +135,7 @@ int ObPxMultiPartUpdateOp::read_row(ObExecContext &ctx,
LOG_WARN("fail get next row from child", K(ret));
}
} else {
op_monitor_info_.otherstat_2_value_++;
// 每一次从child节点获得新的数据都需要进行清除计算标记
clear_evaluated_flag();
++upd_rtdef_.cur_row_num_;
@ -160,6 +161,8 @@ int ObPxMultiPartUpdateOp::read_row(ObExecContext &ctx,
tablet_id = expr_datum.get_int();
LOG_DEBUG("get the part id", K(ret), K(expr_datum));
}
} else {
op_monitor_info_.otherstat_4_value_++;
}
}
return ret;
@ -192,12 +195,23 @@ int ObPxMultiPartUpdateOp::write_rows(ObExecContext &ctx,
LOG_WARN("update row to das failed", K(ret));
} else if (OB_FAIL(discharge_das_write_buffer())) {
LOG_WARN("failed to submit all dml task when the buffer of das op is full", K(ret));
} else {
op_monitor_info_.otherstat_3_value_++;
}
}
if (OB_ITER_END == ret) {
if (OB_FAIL(submit_all_dml_task())) {
LOG_WARN("do insert rows post process failed", K(ret));
} else {
if (upd_rtdef_.ddel_rtdef_ != nullptr) {
//update rows across partitions, need to add das delete op's affected rows
op_monitor_info_.otherstat_6_value_ += upd_rtdef_.ddel_rtdef_->affected_rows_;
}
if (upd_rtdef_.dlock_rtdef_ != nullptr) {
op_monitor_info_.otherstat_6_value_ += upd_rtdef_.dlock_rtdef_->affected_rows_;
}
op_monitor_info_.otherstat_6_value_ += upd_rtdef_.dupd_rtdef_.affected_rows_;
}
}
if (!(MY_SPEC.is_pdml_index_maintain_)) {
@ -208,8 +222,19 @@ int ObPxMultiPartUpdateOp::write_rows(ObExecContext &ctx,
plan_ctx->add_affected_rows(session->get_capability().cap_flags_.OB_CLIENT_FOUND_ROWS ?
found_rows : changed_rows);
}
LOG_TRACE("pdml update ok", K(MY_SPEC.is_pdml_index_maintain_),
K(op_monitor_info_.otherstat_1_value_), K(op_monitor_info_.otherstat_2_value_),
K(op_monitor_info_.otherstat_3_value_), K(op_monitor_info_.otherstat_4_value_),
K(op_monitor_info_.otherstat_6_value_));
upd_rtdef_.found_rows_ = 0;
upd_rtdef_.dupd_rtdef_.affected_rows_ = 0;
if (upd_rtdef_.ddel_rtdef_ != nullptr) {
//update rows across partitions, need to add das delete op's affected rows
upd_rtdef_.ddel_rtdef_->affected_rows_ = 0;
}
if (upd_rtdef_.dlock_rtdef_ != nullptr) {
upd_rtdef_.dlock_rtdef_->affected_rows_ = 0;
}
}
return ret;
}

12
src/sql/engine/px/ob_px_admission.cpp
View File

@ -180,6 +180,18 @@ int ObPxAdmission::enter_query_admission(ObSQLSessionInfo &session,
}
}
}
if (stmt::T_EXPLAIN != stmt_type && plan.get_das_dop() > 0) {
int64_t minimal_px_worker_count = plan.get_minimal_worker_count();
int64_t parallel_servers_target = INT64_MAX;
if (OB_FAIL(OB_PX_TARGET_MGR.get_parallel_servers_target(session.get_effective_tenant_id(),
parallel_servers_target))) {
LOG_WARN("get parallel_servers_target failed", K(ret));
} else {
int64_t real_das_dop = std::min(parallel_servers_target, plan.get_das_dop());
exec_ctx.get_das_ctx().set_real_das_dop(real_das_dop);
LOG_TRACE("real das dop", K(real_das_dop), K(plan.get_das_dop()), K(parallel_servers_target));
}
}
return ret;
}

25
src/sql/monitor/ob_sql_plan.cpp
View File

@ -282,22 +282,39 @@ int ObSqlPlan::construct_outline_global_hint(ObLogPlan &plan, ObGlobalHint &outl
ObDelUpdLogPlan *del_upd_plan = NULL;
outline_global_hint.pdml_option_ = ObPDMLOption::NOT_SPECIFIED;
outline_global_hint.parallel_ = ObGlobalHint::UNSET_PARALLEL;
outline_global_hint.parallel_das_dml_option_ = ObParallelDASOption::NOT_SPECIFIED;
const ObQueryCtx *query_ctx = NULL;
if (OB_ISNULL(query_ctx = plan.get_optimizer_context().get_query_ctx())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected NULL", K(ret), K(query_ctx));
} else {
bool has_set_parallel = false;
outline_global_hint.opt_features_version_ = query_ctx->optimizer_features_enable_version_;
if (NULL != (del_upd_plan = dynamic_cast<ObDelUpdLogPlan*>(&plan)) && del_upd_plan->use_pdml()) {
outline_global_hint.pdml_option_ = ObPDMLOption::ENABLE;
if (NULL != (del_upd_plan = dynamic_cast<ObDelUpdLogPlan*>(&plan))) {
if (del_upd_plan->use_pdml()) {
outline_global_hint.pdml_option_ = ObPDMLOption::ENABLE;
}
if (del_upd_plan->get_can_use_parallel_das_dml()) {
has_set_parallel = true;
outline_global_hint.parallel_das_dml_option_ = ObParallelDASOption::ENABLE;
if (plan.get_optimizer_context().is_use_auto_dop()) {
outline_global_hint.merge_parallel_hint(ObGlobalHint::SET_ENABLE_AUTO_DOP);
} else {
outline_global_hint.merge_parallel_hint(del_upd_plan->get_max_dml_parallel());
}
}
}
if (plan.get_optimizer_context().is_use_auto_dop()) {
if (has_set_parallel) {
// set parallel_ before
} else if (plan.get_optimizer_context().is_use_auto_dop()) {
outline_global_hint.merge_parallel_hint(ObGlobalHint::SET_ENABLE_AUTO_DOP);
} else if (plan.get_optimizer_context().get_max_parallel() > ObGlobalHint::DEFAULT_PARALLEL) {
outline_global_hint.merge_parallel_hint(plan.get_optimizer_context().get_max_parallel());
}
}
LOG_TRACE("after construct_outline_global_hint", K(outline_global_hint.parallel_das_dml_option_),
K(outline_global_hint.pdml_option_), K(outline_global_hint.parallel_));
return ret;
}

32
src/sql/ob_result_set.cpp
View File

@ -827,6 +827,28 @@ bool ObResultSet::need_rollback(int ret, int errcode, bool is_error_ignored) con
return bret;
}
int ObResultSet::deal_feedback_info(ObPhysicalPlan *physical_plan, bool is_rollback, ObExecContext &ctx)
{
int ret = OB_SUCCESS;
ObPhysicalPlanCtx *physical_ctx = get_exec_context().get_physical_plan_ctx();
if (ctx.get_feedback_info().is_valid() && physical_plan->get_logical_plan().is_valid()) {
if (physical_ctx != nullptr && !is_rollback && physical_ctx->get_check_pdml_affected_rows()) {
if (OB_FAIL(physical_plan->check_pdml_affected_rows(ctx))) {
LOG_WARN("fail to check pdml affected_rows", K(ret));
}
}
if (physical_plan->try_record_plan_info()) {
if (OB_FAIL(physical_plan->set_feedback_info(ctx))) {
LOG_WARN("fail to set feed_back info", K(ret));
} else {
physical_plan->set_record_plan_info(false);
}
}
}
return ret;
}
OB_INLINE int ObResultSet::do_close_plan(int errcode, ObExecContext &ctx)
{
int ret = common::OB_SUCCESS;
@ -908,14 +930,8 @@ OB_INLINE int ObResultSet::do_close_plan(int errcode, ObExecContext &ctx)
ret = sret;
}
if (OB_SUCC(ret)) {
if (physical_plan_->try_record_plan_info()) {
if (ctx.get_feedback_info().is_valid() &&
physical_plan_->get_logical_plan().is_valid() &&
OB_FAIL(physical_plan_->set_feedback_info(ctx))) {
LOG_WARN("failed to set feedback info", K(ret));
} else {
physical_plan_->set_record_plan_info(false);
}
if (OB_FAIL(deal_feedback_info(physical_plan_, rollback, ctx))) {
LOG_WARN("fail to deal feedback info", K(ret));
}
}
} else {

1
src/sql/ob_result_set.h
View File

@ -353,6 +353,7 @@ private:
int open_result();
int do_open_plan(ObExecContext &ctx);
int do_close_plan(int errcode, ObExecContext &ctx);
int deal_feedback_info(ObPhysicalPlan *physical_plan, bool is_rollback, ObExecContext &ctx);
bool transaction_set_violation_and_retry(int &err, int64_t &retry);
int init_cmd_exec_context(ObExecContext &exec_ctx);
int on_cmd_execute();

8
src/sql/ob_sql.cpp
View File

@ -3897,14 +3897,6 @@ int ObSql::code_generate(
phy_plan->stat_.is_use_jit_ = use_jit;
phy_plan->set_returning(stmt->is_returning());
phy_plan->set_has_link_table(has_dblink || phy_plan->has_link_udf());
// set plan insert flag : insert into values(..); // value num is n (n >= 1);
if (stmt->is_insert_stmt()) {
ObInsertStmt *insert_stmt = static_cast<ObInsertStmt *>(stmt);
phy_plan->set_is_plain_insert(!insert_stmt->value_from_select()
&& !insert_stmt->is_insert_up()
&& insert_stmt->get_subquery_exprs().empty()
&& !insert_stmt->is_replace());
}
last_mem_usage = phy_plan->get_mem_size();
}
}

7
src/sql/ob_sql_define.h
View File

@ -433,6 +433,13 @@ enum class ObPDMLOption {
MAX_VALUE
};
enum class ObParallelDASOption {
NOT_SPECIFIED = -1,
ENABLE,
DISABLE,
MAX_VALUE
};
enum OrderingFlag
{
NOT_MATCH = 0,

96
src/sql/ob_sql_trans_control.cpp
View File

@ -631,7 +631,7 @@ int ObSqlTransControl::start_stmt(ObExecContext &exec_ctx)
OZ (stmt_setup_savepoint_(session, das_ctx, plan_ctx, txs, nested_level), session_id, *tx_desc);
}
OZ (stmt_setup_snapshot_(session, das_ctx, plan, plan_ctx, txs), session_id, *tx_desc);
OZ (stmt_setup_snapshot_(session, das_ctx, plan, plan_ctx, txs, exec_ctx), session_id, *tx_desc);
// add tx id to AuditRecord
OX (session->get_raw_audit_record().trans_id_ = session->get_tx_id());
@ -854,11 +854,13 @@ int ObSqlTransControl::stmt_setup_snapshot_(ObSQLSessionInfo *session,
ObDASCtx &das_ctx,
const ObPhysicalPlan *plan,
const ObPhysicalPlanCtx *plan_ctx,
ObTransService *txs)
ObTransService *txs,
ObExecContext &exec_ctx)
{
int ret = OB_SUCCESS;
ObConsistencyLevel cl = plan_ctx->get_consistency_level();
ObTxReadSnapshot &snapshot = das_ctx.get_snapshot();
bool can_plain_insert = false;
if (cl == ObConsistencyLevel::WEAK || cl == ObConsistencyLevel::FROZEN) {
SCN snapshot_version = SCN::min_scn();
const bool local_single_ls = plan->is_local_plan() &&
@ -878,10 +880,11 @@ int ObSqlTransControl::stmt_setup_snapshot_(ObSQLSessionInfo *session,
}
// 1) acquire snapshot version when insert operator is executed
// 2) don't resolve RR and SERIALIZABLE isolation scenario temporarily, because of remote stmt plan
} else if (plan->is_plain_insert()
&& session->get_tx_isolation() != ObTxIsolationLevel::SERIAL
&& session->get_tx_isolation() != ObTxIsolationLevel::RR) {
} else if (OB_FAIL(can_do_plain_insert(session, plan, exec_ctx, can_plain_insert))) {
TRANS_LOG(WARN, "check can do plain insert failed", KPC(txs));
} else if (can_plain_insert) {
ObTxDesc &tx_desc = *session->get_tx_desc();
das_ctx.set_use_gts_opt(true);
snapshot.init_none_read();
snapshot.core_.tx_id_ = tx_desc.get_tx_id();
snapshot.core_.scn_ = tx_desc.get_tx_seq();
@ -935,7 +938,7 @@ int ObSqlTransControl::stmt_refresh_snapshot(ObExecContext &exec_ctx) {
//NOTE: oracle insert and insert all stmt can't see the evaluated results of before stmt trigger, no need to refresh snapshot
} else if (OB_FAIL(get_tx_service(session, txs))) {
LOG_WARN("failed to get transaction service", K(ret));
} else if (OB_FAIL(stmt_setup_snapshot_(session, das_ctx, plan, plan_ctx, txs))) {
} else if (OB_FAIL(stmt_setup_snapshot_(session, das_ctx, plan, plan_ctx, txs, exec_ctx))) {
LOG_WARN("failed to set snapshot", K(ret));
}
return ret;
@ -981,6 +984,87 @@ int ObSqlTransControl::set_fk_check_snapshot(ObExecContext &exec_ctx)
return ret;
}
int ObSqlTransControl::can_do_plain_insert(ObSQLSessionInfo *session,
const ObPhysicalPlan *plan,
ObExecContext &exec_ctx,
bool &can_plain_insert)
{
int ret = OB_SUCCESS;
can_plain_insert = false;
int last_query_retry_err = session->get_retry_info().get_last_query_retry_err();
if (OB_ISNULL(session) || OB_ISNULL(plan)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null ptr", K(ret), KPC(session), KPC(plan));
} else if (plan->get_need_serial_exec()
|| ObSQLUtils::is_nested_sql(&exec_ctx)
|| last_query_retry_err == OB_TRANSACTION_SET_VIOLATION
|| session->get_tx_isolation() != ObTxIsolationLevel::RC) {
LOG_TRACE("can't support plain insert", K(plan->get_need_serial_exec()), K(last_query_retry_err),
K(ObSQLUtils::is_nested_sql(&exec_ctx)), K(session->get_tx_isolation()));
} else if (plan->is_plain_insert()) {
can_plain_insert = true;
} else if (session->enable_insertup_replace_gts_opt() && plan->get_insertup_can_do_gts_opt()) {
if ((GET_MIN_CLUSTER_VERSION() >= CLUSTER_VERSION_4_3_5_0)
&& session->is_user_session()) {
can_plain_insert = true;
}
}
if (OB_SUCC(ret)) {
if (plan->get_insertup_can_do_gts_opt()) {
LOG_TRACE("whether can do_batch_insert", K(can_plain_insert), K(plan->is_plain_insert()),
K(last_query_retry_err), K(plan->get_insertup_can_do_gts_opt()),
K(plan->get_need_serial_exec()), K(session->is_user_session()),
K(session->get_tx_isolation()), K(ObSQLUtils::is_nested_sql(&exec_ctx)),
K(session->enable_insertup_replace_gts_opt()));
}
}
return ret;
}
int ObSqlTransControl::get_read_snapshot(ObSQLSessionInfo *session,
ObPhysicalPlanCtx *plan_ctx,
transaction::ObTxReadSnapshot &snapshot)
{
int ret = OB_SUCCESS;
ObTxIsolationLevel isolation = session->get_tx_isolation();
const ObPhysicalPlan *plan = plan_ctx->get_phy_plan();
int64_t expire_ts = get_stmt_expire_ts(plan_ctx, *session);
transaction::ObTransService *txs = NULL;
transaction::ObTxDesc &tx_desc = *session->get_tx_desc();
if (OB_FAIL(get_tx_service(session, txs))) {
LOG_WARN("failed to get transaction service", K(ret));
} else if (OB_FAIL(txs->get_read_snapshot(tx_desc, isolation, expire_ts, snapshot))) {
LOG_WARN("failed to set snapshot", K(ret));
} else if (!snapshot.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected invalid snapshot", K(ret), K(tx_desc), K(isolation));
}
return ret;
}
int ObSqlTransControl::get_ls_read_snapshot(ObSQLSessionInfo *session,
ObPhysicalPlanCtx *plan_ctx,
const share::ObLSID &local_ls_id,
transaction::ObTxReadSnapshot &snapshot)
{
int ret = OB_SUCCESS;
ObTxIsolationLevel isolation = session->get_tx_isolation();
const ObPhysicalPlan *plan = plan_ctx->get_phy_plan();
int64_t expire_ts = get_stmt_expire_ts(plan_ctx, *session);
transaction::ObTransService *txs = NULL;
transaction::ObTxDesc &tx_desc = *session->get_tx_desc();
if (OB_FAIL(get_tx_service(session, txs))) {
LOG_WARN("failed to get transaction service", K(ret));
} else if (OB_FAIL(txs->get_ls_read_snapshot(tx_desc, isolation, local_ls_id, expire_ts, snapshot))) {
LOG_WARN("failed to set snapshot", K(ret));
} else if (!snapshot.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected invalid snapshot", K(ret), K(tx_desc), K(isolation), K(local_ls_id));
}
return ret;
}
int ObSqlTransControl::stmt_setup_savepoint_(ObSQLSessionInfo *session,
ObDASCtx &das_ctx,
ObPhysicalPlanCtx *plan_ctx,

14
src/sql/ob_sql_trans_control.h
View File

@ -193,6 +193,13 @@ public:
const int64_t expire_ts,
ObEndTransAsyncCallback *callback);
static int start_stmt(ObExecContext &ctx);
static int get_ls_read_snapshot(ObSQLSessionInfo *session,
ObPhysicalPlanCtx *plan_ctx,
const share::ObLSID &local_ls_id,
transaction::ObTxReadSnapshot &snapshot);
static int get_read_snapshot(ObSQLSessionInfo *session,
ObPhysicalPlanCtx *plan_ctx,
transaction::ObTxReadSnapshot &snapshot);
static int dblink_xa_prepare(ObExecContext &exec_ctx);
static int stmt_sanity_check_(ObSQLSessionInfo *session,
const ObPhysicalPlan *plan,
@ -201,7 +208,12 @@ public:
ObDASCtx &das_ctx,
const ObPhysicalPlan *plan,
const ObPhysicalPlanCtx *plan_ctx,
transaction::ObTransService *txs);
transaction::ObTransService *txs,
ObExecContext &exec_ctx);
static int can_do_plain_insert(ObSQLSessionInfo *session,
const ObPhysicalPlan *plan,
ObExecContext &exec_ctx,
bool &can_plain_insert);
static int stmt_refresh_snapshot(ObExecContext &ctx);
static int set_fk_check_snapshot(ObExecContext &exec_ctx);
static int stmt_setup_savepoint_(ObSQLSessionInfo *session,

19
src/sql/ob_sql_utils.cpp
View File

@ -3949,6 +3949,25 @@ int ObImplicitCursorInfo::merge_cursor(const ObImplicitCursorInfo &other)
return ret;
}
int ObImplicitCursorInfo::replace_cursor(const ObImplicitCursorInfo &other)
{
int ret = OB_SUCCESS;
if (OB_INVALID_INDEX == stmt_id_) {
//not init, init it with first cursor info
*this = other;
} else if (stmt_id_ != other.stmt_id_) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("stmt_id is different", K(stmt_id_), K(other.stmt_id_));
} else {
affected_rows_ = other.affected_rows_;
found_rows_ = other.found_rows_;
matched_rows_ = other.matched_rows_;
duplicated_rows_ = other.duplicated_rows_;
deleted_rows_ = other.deleted_rows_;
}
return ret;
}
bool ObSQLUtils::is_same_type_for_compare(const ObObjMeta &meta1, const ObObjMeta &meta2)
{
bool is_same = false;

1
src/sql/ob_sql_utils.h
View File

@ -1078,6 +1078,7 @@ public:
last_insert_id_(0) {}
int merge_cursor(const ObImplicitCursorInfo &other);
int replace_cursor(const ObImplicitCursorInfo &other);
TO_STRING_KV(K_(stmt_id),
K_(affected_rows),
K_(found_rows),

100
src/sql/optimizer/ob_del_upd_log_plan.cpp
View File

@ -49,6 +49,7 @@ int ObDelUpdLogPlan::compute_dml_parallel()
int ret = OB_SUCCESS;
use_pdml_ = false;
max_dml_parallel_ = ObGlobalHint::UNSET_PARALLEL;
int64_t dml_parallel = ObGlobalHint::UNSET_PARALLEL;
const ObOptimizerContext &opt_ctx = get_optimizer_context();
const ObSQLSessionInfo *session_info = NULL;
const ObDelUpdStmt *del_upd_stmt = NULL;
@ -56,7 +57,7 @@ int ObDelUpdLogPlan::compute_dml_parallel()
OB_ISNULL(del_upd_stmt = get_stmt())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected null", K(ret), K(get_optimizer_context().get_session_info()));
} else if (!opt_ctx.can_use_pdml()) {
} else if (!opt_ctx.can_use_pdml() && !opt_ctx.get_can_use_parallel_das_dml()) {
max_dml_parallel_ = ObGlobalHint::DEFAULT_PARALLEL;
use_pdml_ = false;
if (opt_ctx.is_online_ddl()) {
@ -64,52 +65,54 @@ int ObDelUpdLogPlan::compute_dml_parallel()
LOG_WARN("a online ddl expect PDML enabled. but it does not!", K(ret));
LOG_USER_ERROR(OB_NOT_SUPPORTED, "online ddl without pdml");
}
} else {
int64_t dml_parallel = ObGlobalHint::UNSET_PARALLEL;
if (OB_FAIL(get_parallel_info_from_candidate_plans(dml_parallel))) {
LOG_WARN("failed to get parallel info from candidate plans", K(ret));
} else if (OB_UNLIKELY(ObGlobalHint::DEFAULT_PARALLEL > dml_parallel)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected parallel", K(ret), K(dml_parallel), K(opt_ctx.get_parallel_rule()));
} else {
if (del_upd_stmt->is_insert_stmt()) {
const ObInsertStmt *insert_stmt = static_cast<const ObInsertStmt*>(del_upd_stmt);
if (insert_stmt->is_normal_table_overwrite()) {
const int64_t default_insert_overwrite_parallel = 2;
if (dml_parallel <= ObGlobalHint::DEFAULT_PARALLEL) {
dml_parallel = default_insert_overwrite_parallel;
}
}
if (dml_parallel <= ObGlobalHint::DEFAULT_PARALLEL && opt_ctx.get_parallel_rule() == PXParallelRule::MANUAL_HINT) {
// do nothing
} else if (OB_FAIL(check_is_direct_load(*insert_stmt, dml_parallel))) {
LOG_WARN("failed to check is direct load", K(ret));
} else if (opt_ctx.get_direct_load_optimizer_ctx().can_use_direct_load()) {
const int64_t default_direct_insert_parallel = 2;
if (dml_parallel <= ObGlobalHint::DEFAULT_PARALLEL) {
dml_parallel = default_direct_insert_parallel;
}
} else if (OB_FAIL(get_parallel_info_from_candidate_plans(dml_parallel))) {
LOG_WARN("failed to get parallel info from candidate plans", K(ret));
} else if (OB_UNLIKELY(ObGlobalHint::DEFAULT_PARALLEL > dml_parallel)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected parallel", K(ret), K(dml_parallel), K(opt_ctx.get_parallel_rule()));
} else if (opt_ctx.can_use_pdml()) {
if (del_upd_stmt->is_insert_stmt()) {
const ObInsertStmt *insert_stmt = static_cast<const ObInsertStmt*>(del_upd_stmt);
if (insert_stmt->is_normal_table_overwrite()) {
const int64_t default_insert_overwrite_parallel = 2;
if (dml_parallel <= ObGlobalHint::DEFAULT_PARALLEL) {
dml_parallel = default_insert_overwrite_parallel;
}
}
if (OB_SUCC(ret)) {
max_dml_parallel_ = dml_parallel;
use_pdml_ = (opt_ctx.is_online_ddl() ||
(ObGlobalHint::DEFAULT_PARALLEL < dml_parallel &&
is_strict_mode(session_info->get_sql_mode())));
if (opt_ctx.get_direct_load_optimizer_ctx().can_use_direct_load() && use_pdml_) {
get_optimizer_context().get_direct_load_optimizer_ctx().set_use_direct_load();
ObExecContext *exec_ctx = get_optimizer_context().get_exec_ctx();
if (OB_ISNULL(exec_ctx)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("exec_ctx is null", K(ret));
} else {
exec_ctx->get_table_direct_insert_ctx().set_is_direct(true);
}
if (dml_parallel <= ObGlobalHint::DEFAULT_PARALLEL && opt_ctx.get_parallel_rule() == PXParallelRule::MANUAL_HINT) {
// do nothing
} else if (OB_FAIL(check_is_direct_load(*insert_stmt, dml_parallel))) {
LOG_WARN("failed to check is direct load", K(ret));
} else if (opt_ctx.get_direct_load_optimizer_ctx().can_use_direct_load()) {
const int64_t default_direct_insert_parallel = 2;
if (dml_parallel <= ObGlobalHint::DEFAULT_PARALLEL) {
dml_parallel = default_direct_insert_parallel;
}
}
}
if (OB_SUCC(ret)) {
max_dml_parallel_ = dml_parallel;
use_pdml_ = (opt_ctx.is_online_ddl() ||
(ObGlobalHint::DEFAULT_PARALLEL < dml_parallel &&
is_strict_mode(session_info->get_sql_mode())));
if (opt_ctx.get_direct_load_optimizer_ctx().can_use_direct_load() && use_pdml_) {
get_optimizer_context().get_direct_load_optimizer_ctx().set_use_direct_load();
ObExecContext *exec_ctx = get_optimizer_context().get_exec_ctx();
if (OB_ISNULL(exec_ctx)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("exec_ctx is null", K(ret));
} else {
exec_ctx->get_table_direct_insert_ctx().set_is_direct(true);
}
}
}
} else if (opt_ctx.get_can_use_parallel_das_dml()) {
max_dml_parallel_ = dml_parallel;
use_parallel_das_dml_ = (!opt_ctx.is_online_ddl() &&
(ObGlobalHint::DEFAULT_PARALLEL < dml_parallel &&
is_strict_mode(session_info->get_sql_mode())));
}
LOG_TRACE("finish compute dml parallel", K(use_pdml_), K(max_dml_parallel_),
LOG_TRACE("finish compute dml parallel", K(use_pdml_), K(max_dml_parallel_), K(use_parallel_das_dml_),
K(opt_ctx.can_use_pdml()), K(opt_ctx.is_online_ddl()),
K(opt_ctx.get_parallel_rule()), K(opt_ctx.get_parallel()));
return ret;
@ -572,7 +575,11 @@ int ObDelUpdLogPlan::compute_hash_dist_exprs_for_pdml_del_upd(ObExchangeInfo &ex
{
int ret = OB_SUCCESS;
ObSEArray<ObRawExpr*, 8> rowkey_exprs;
if (OB_UNLIKELY(dml_info.get_real_uk_cnt() > dml_info.column_exprs_.count())) {
const ObDelUpdStmt *stmt = get_stmt();
if (OB_ISNULL(stmt)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null", K(ret));
} else if (OB_UNLIKELY(dml_info.get_real_uk_cnt() > dml_info.column_exprs_.count())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected count", K(dml_info.get_real_uk_cnt()), K(dml_info.column_exprs_), K(ret));
} else {
@ -582,7 +589,8 @@ int ObDelUpdLogPlan::compute_hash_dist_exprs_for_pdml_del_upd(ObExchangeInfo &ex
// 那么就选择更新后的值，否则选择更新前的值。
// 对于 delete，因为 assignment 为空，所以会直接 push rowkey
ObRawExpr *target_expr = dml_info.column_exprs_.at(i);
if (OB_FAIL(replace_assignment_expr_from_dml_info(dml_info, target_expr))) {
if (!(stmt->is_merge_stmt()) &&
OB_FAIL(replace_assignment_expr_from_dml_info(dml_info, target_expr))) {
LOG_WARN("failed to replace assignment expr", K(ret));
} else if (OB_FAIL(rowkey_exprs.push_back(target_expr))) {
LOG_WARN("failed to push back expr", K(ret));
@ -921,7 +929,8 @@ int ObDelUpdLogPlan::build_merge_stmt_hash_dist_exprs(const IndexDMLInfo &dml_in
} else if (OB_ISNULL(when_expr)) {
for (int64_t i = 0; OB_SUCC(ret) && i < dml_info.get_real_uk_cnt(); i++) {
ObRawExpr *target_expr = dml_info.column_exprs_.at(i);
if (OB_FAIL(replace_assignment_expr_from_dml_info(dml_info, target_expr))) {
if (!(stmt->is_merge_stmt()) &&
OB_FAIL(replace_assignment_expr_from_dml_info(dml_info, target_expr))) {
LOG_WARN("failed to replace assignment expr", K(ret));
} else if (OB_FAIL(rowkey_exprs.push_back(target_expr))) {
LOG_WARN("failed to push back expr", K(ret));
@ -938,7 +947,8 @@ int ObDelUpdLogPlan::build_merge_stmt_hash_dist_exprs(const IndexDMLInfo &dml_in
for (int64_t i = 0; OB_SUCC(ret) && i < dml_info.get_real_uk_cnt(); i++) {
ObRawExpr *raw_expr = NULL;
ObRawExpr *target_expr = dml_info.column_exprs_.at(i);
if (OB_FAIL(replace_assignment_expr_from_dml_info(dml_info, target_expr))) {
if (!(stmt->is_merge_stmt()) &&
OB_FAIL(replace_assignment_expr_from_dml_info(dml_info, target_expr))) {
LOG_WARN("failed to replace assignment expr", K(ret));
} else if (OB_FAIL(ObRawExprUtils::build_case_when_expr(get_optimizer_context().get_expr_factory(),
when_expr,

7
src/sql/optimizer/ob_del_upd_log_plan.h
View File

@ -26,7 +26,8 @@ public:
ObDelUpdLogPlan(ObOptimizerContext &ctx, const ObDelUpdStmt *del_upd_stmt)
: ObLogPlan(ctx, del_upd_stmt),
max_dml_parallel_(ObGlobalHint::UNSET_PARALLEL),
use_pdml_(false)
use_pdml_(false),
use_parallel_das_dml_(false)
{}
virtual ~ObDelUpdLogPlan() {}
@ -241,8 +242,9 @@ public:
int check_is_direct_load(const ObInsertStmt &insert_stmt, const int64_t dml_parallel);
int get_parallel_info_from_candidate_plans(int64_t &dop) const;
int get_pdml_parallel_degree(const int64_t target_part_cnt, int64_t &dop) const;
bool get_can_use_parallel_das_dml() const { return use_parallel_das_dml_; }
int64_t get_max_dml_parallel() { return max_dml_parallel_; }
virtual int perform_vector_assign_expr_replacement(ObDelUpdStmt *stmt);
protected:
virtual int generate_normal_raw_plan() override;
virtual int generate_dblink_raw_plan() override;
@ -263,6 +265,7 @@ protected:
ObSEArray<share::schema::ObSchemaObjVersion, 4, common::ModulePageAllocator, true> extra_dependency_tables_;
int64_t max_dml_parallel_;
int64_t use_pdml_;
bool use_parallel_das_dml_;
};
} /* namespace sql */

4
src/sql/optimizer/ob_delete_log_plan.cpp
View File

@ -235,6 +235,7 @@ int ObDeleteLogPlan::create_delete_plans(ObIArray<CandidatePlan> &candi_plans,
OB_FAIL(check_need_multi_partition_dml(*get_stmt(),
*candi_plan.plan_tree_,
index_dml_infos_,
use_parallel_das_dml_,
is_multi_part_dml,
is_result_local))) {
LOG_WARN("failed to check need multi-partition dml", K(ret));
@ -270,6 +271,9 @@ int ObDeleteLogPlan::allocate_delete_as_top(ObLogicalOperator *&top,
delete_op->set_is_returning(delete_stmt->is_returning());
delete_op->set_is_multi_part_dml(is_multi_part_dml);
delete_op->set_has_instead_of_trigger(delete_stmt->has_instead_of_trigger());
if (get_can_use_parallel_das_dml()) {
delete_op->set_das_dop(max_dml_parallel_);
}
if (OB_FAIL(delete_op->assign_dml_infos(index_dml_infos_))) {
LOG_WARN("failed to assign dml infos", K(ret));
} else if (delete_stmt->is_error_logging() && OB_FAIL(delete_op->extract_err_log_info())) {

7
src/sql/optimizer/ob_insert_log_plan.cpp
View File

@ -703,6 +703,10 @@ int ObInsertLogPlan::allocate_insert_as_top(ObLogicalOperator *&top,
insert_op->set_table_partition_info(table_partition_info);
insert_op->set_lock_row_flag_expr(lock_row_flag_expr);
insert_op->set_has_instead_of_trigger(insert_stmt->has_instead_of_trigger());
if (get_can_use_parallel_das_dml()) {
insert_op->set_das_dop(max_dml_parallel_);
LOG_TRACE("insert das dop", K(max_dml_parallel_));
}
insert_op->set_is_partition_wise(is_partition_wise);
if (OB_NOT_NULL(insert_stmt->get_table_item(0))) {
insert_op->set_append_table_id(insert_stmt->get_table_item(0)->ref_id_);
@ -853,6 +857,9 @@ int ObInsertLogPlan::check_insert_plan_need_multi_partition_dml(ObTablePartition
} else if (0 == insert_table_part->get_phy_tbl_location_info().get_partition_cnt()) {
is_multi_part_dml = true;
OPT_TRACE("insert table has no partition, force use multi part dml");
} else if (use_parallel_das_dml_) {
is_multi_part_dml = true;
OPT_TRACE("insert table use parallel das dml, force use multi part dml");
} else if (insert_stmt->has_instead_of_trigger() ||
index_dml_infos_.count() > 1 ||
get_optimizer_context().is_batched_multi_stmt() ||

3
src/sql/optimizer/ob_log_del_upd.cpp
View File

@ -308,6 +308,7 @@ ObLogDelUpd::ObLogDelUpd(ObDelUpdLogPlan &plan)
is_first_dml_op_(false),
table_location_uncertain_(false),
is_pdml_update_split_(false),
das_dop_(0),
pdml_partition_id_expr_(NULL),
ddl_slice_id_expr_(NULL),
pdml_is_returning_(false),
@ -867,7 +868,7 @@ int ObLogDelUpd::compute_sharding_info()
} else if (OB_ISNULL(get_sharding())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected null", K(ret));
} else if (!is_pdml()) {
} else if (!(is_pdml() || get_das_dop() > 0)) {
is_partition_wise_ = !is_multi_part_dml_ && !child->is_exchange_allocated() &&
get_sharding()->is_distributed() &&
NULL != get_sharding()->get_phy_table_location_info();

3
src/sql/optimizer/ob_log_del_upd.h
View File

@ -337,6 +337,8 @@ public:
const ObIArray<IndexDMLInfo *> &index_dml_infos);
virtual int is_my_fixed_expr(const ObRawExpr *expr, bool &is_fixed) override = 0;
virtual int check_use_child_ordering(bool &used, int64_t &inherit_child_ordering_index)override;
void set_das_dop(int64_t dop) { das_dop_ = dop; }
int64_t get_das_dop() { return das_dop_; }
protected:
virtual int generate_rowid_expr_for_trigger() = 0;
virtual int generate_part_id_expr_for_foreign_key(ObIArray<ObRawExpr*> &all_exprs) = 0;
@ -430,6 +432,7 @@ protected:
//
bool table_location_uncertain_;
bool is_pdml_update_split_; // 标记delete, insert op是否由update拆分而来
int64_t das_dop_; // zero marks not use parallel_das_dml
private:
// 如果是PDML，那么对应的DML算子（insert，update，delete）需要一个partition id expr
ObRawExpr *pdml_partition_id_expr_;

3
src/sql/optimizer/ob_log_delete.cpp
View File

@ -185,6 +185,9 @@ int ObLogDelete::get_plan_item_info(PlanText &plan_text,
ret = BUF_PRINTF(", ");
ret = BUF_PRINTF("with_barrier");
}
if (OB_SUCC(ret) && get_das_dop() > 0) {
ret = BUF_PRINTF(", das_dop=%ld", this->get_das_dop());
}
END_BUF_PRINT(plan_item.special_predicates_,
plan_item. special_predicates_len_);
}

41
src/sql/optimizer/ob_log_expr_values.cpp
View File

@ -45,7 +45,7 @@ namespace sql
} \
} else if (OB_UNLIKELY(0 != N % M)) { \
ret = OB_ERR_UNEXPECTED; \
LOG_WARN("invalid value count", K(ret), "value_count", N, "row_count", M); \
LOG_WARN("invalid value count", K(ret), "value_count", N, "row_count", M, K(values)); \
} else { \
for (int64_t i = 0; OB_SUCC(ret) && i < N / M; i++) { \
if (OB_FAIL(BUF_PRINTF("{"))) { \
@ -97,8 +97,12 @@ int ObLogExprValues::add_values_expr(const common::ObIArray<ObRawExpr *> &value_
if (OB_ISNULL(stmt_id_expr = insert_stmt->get_ab_stmt_id_expr())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("stmt_id_expr is null", K(ret));
} else if (OB_FAIL(append(value_exprs_, get_stmt()->get_query_ctx()->ab_param_exprs_))) {
LOG_WARN("assign ab param exprs to value exprs failed", K(ret));
} else if (OB_FAIL(value_exprs_.push_back(stmt_id_expr))) {
LOG_WARN("fail to push stmt_id_expr", K(ret));
} else {
LOG_TRACE("print after add_values_expr", K(get_stmt()->get_query_ctx()->ab_param_exprs_), K(stmt_id_expr));
}
}
return ret;
@ -296,6 +300,35 @@ int ObLogExprValues::get_op_exprs(ObIArray<ObRawExpr*> &all_exprs)
return ret;
}
int ObLogExprValues::append_batch_insert_used_exprs(ObAllocExprContext &ctx)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(get_stmt())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected null", K(get_stmt()), K(ret));
} else if (get_stmt()->is_insert_stmt()) {
const ObInsertStmt *insert_stmt = static_cast<const ObInsertStmt*>(get_stmt());
const common::ObIArray<ObRawExpr*> &group_param_exprs = insert_stmt->get_group_param_exprs();
for (int64_t i = 0; OB_SUCC(ret) && i < group_param_exprs.count(); ++i) {
ObRawExpr *group_param_expr = group_param_exprs.at(i);
if (OB_FAIL(mark_expr_produced(group_param_expr, branch_id_, id_, ctx))) {
LOG_WARN("makr expr produced failed", K(ret));
} else if (!is_plan_root() && OB_FAIL(output_exprs_.push_back(group_param_expr))) {
LOG_WARN("failed to push back exprs", K(ret));
} else { /*do nothing*/ }
}
ObRawExpr *stmt_id_expr = NULL;
if (OB_FAIL(ret)) {
} else if (OB_ISNULL(stmt_id_expr = insert_stmt->get_ab_stmt_id_expr())) {
// is not batch_optimization, do nothing
} else if (OB_FAIL(output_exprs_.push_back(stmt_id_expr))) {
LOG_WARN("fail to push stmt_id_expr", K(ret));
}
}
return ret;
}
int ObLogExprValues::allocate_expr_post(ObAllocExprContext &ctx)
{
int ret = OB_SUCCESS;
@ -319,6 +352,12 @@ int ObLogExprValues::allocate_expr_post(ObAllocExprContext &ctx)
LOG_WARN("failed to push back exprs", K(ret));
} else { /*do nothing*/ }
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(append_batch_insert_used_exprs(ctx))) {
LOG_WARN("failed to append batch insert used exprs", K(ret));
}
}
if (OB_FAIL(ret)) {
/*do nothing*/

1
src/sql/optimizer/ob_log_expr_values.h
View File

@ -69,6 +69,7 @@ class ObLogExprValues : public ObLogicalOperator
virtual int get_op_exprs(ObIArray<ObRawExpr*> &all_exprs) override;
virtual int is_my_fixed_expr(const ObRawExpr *expr, bool &is_fixed) override;
virtual int allocate_expr_post(ObAllocExprContext &ctx) override;
int append_batch_insert_used_exprs(ObAllocExprContext &ctx);
int extract_var_init_exprs(const ObDMLStmt *stmt, ObIArray<ObRawExpr*> &assign_exprs);
int extract_err_log_info();
int mark_probably_local_exprs();

44
src/sql/optimizer/ob_log_insert.cpp
View File

@ -143,6 +143,10 @@ int ObLogInsert::get_plan_item_info(PlanText &plan_text,
} else { /* Do nothing */ }
BUF_PRINTF(")");
} else { /* Do nothing */ }
if (OB_SUCC(ret) && get_das_dop() > 0) {
ret = BUF_PRINTF(", das_dop=%ld", this->get_das_dop());
}
END_BUF_PRINT(plan_item.special_predicates_,
plan_item.special_predicates_len_);
}
@ -577,4 +581,42 @@ int ObLogInsert::inner_replace_op_exprs(ObRawExprReplacer &replacer)
}
}
return ret;
}
}
// set plain insert flag : insert into values(..); // value num is n (n >= 1);
int ObLogInsert::is_plain_insert(bool &is_plain_insert)
{
int ret = OB_SUCCESS;
const ObInsertStmt *insert_stmt = nullptr;
is_plain_insert = false;
if (OB_ISNULL(get_stmt()) || OB_UNLIKELY(!get_stmt()->is_insert_stmt())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected null", K(ret));
} else if (FALSE_IT(insert_stmt = static_cast<const ObInsertStmt *>(get_stmt()))) {
} else {
is_plain_insert = (!insert_stmt->value_from_select()
&& !insert_stmt->is_insert_up()
&& insert_stmt->get_subquery_exprs().empty()
&& !insert_stmt->is_replace());
}
return ret;
}
int ObLogInsert::is_insertup_or_replace_values(bool &is)
{
int ret = OB_SUCCESS;
const ObInsertStmt *insert_stmt = nullptr;
is = false;
if (OB_ISNULL(get_stmt()) || OB_UNLIKELY(!get_stmt()->is_insert_stmt())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected null", K(ret));
} else if (FALSE_IT(insert_stmt = static_cast<const ObInsertStmt *>(get_stmt()))) {
} else if (insert_stmt->is_insert_up() || insert_stmt->is_replace()) {
is = (!insert_stmt->value_from_select() && insert_stmt->get_subquery_exprs().empty());
}
return ret;
}

2
src/sql/optimizer/ob_log_insert.h
View File

@ -121,6 +121,8 @@ public:
virtual int inner_replace_op_exprs(ObRawExprReplacer &replacer) override;
virtual int get_plan_item_info(PlanText &plan_text,
ObSqlPlanItem &plan_item) override;
int is_plain_insert(bool &is_plain_insert);
int is_insertup_or_replace_values(bool &is);
protected:
int get_constraint_info_exprs(ObIArray<ObRawExpr*> &all_exprs);
virtual int generate_rowid_expr_for_trigger() override;

3
src/sql/optimizer/ob_log_merge.cpp
View File

@ -95,6 +95,9 @@ int ObLogMerge::get_plan_item_info(PlanText &plan_text,
}
EXPLAIN_PRINT_EXPRS(delete_conds, type);
}
if (OB_SUCC(ret) && get_das_dop() > 0) {
ret = BUF_PRINTF(", das_dop=%ld", this->get_das_dop());
}
END_BUF_PRINT(plan_item.special_predicates_,
plan_item.special_predicates_len_);
}

6
src/sql/optimizer/ob_log_plan.cpp
View File

@ -114,6 +114,7 @@ ObLogPlan::ObLogPlan(ObOptimizerContext &ctx, const ObDMLStmt *stmt)
candidates_(),
group_replaced_exprs_(),
stat_partition_id_expr_(nullptr),
stat_table_scan_(nullptr),
query_ref_(NULL),
root_(NULL),
sql_text_(),
@ -10384,6 +10385,7 @@ bool ObLogPlan::need_consistent_read() const
int ObLogPlan::check_need_multi_partition_dml(const ObDMLStmt &stmt,
ObLogicalOperator &top,
const ObIArray<IndexDMLInfo *> &index_dml_infos,
bool use_parallel_das,
bool &is_multi_part_dml,
bool &is_result_local)
{
@ -10397,6 +10399,9 @@ int ObLogPlan::check_need_multi_partition_dml(const ObDMLStmt &stmt,
} else if (stmt.has_instead_of_trigger()) {
is_multi_part_dml = true;
is_result_local = true;
} else if (use_parallel_das) {
is_multi_part_dml = true;
is_result_local = true;
} else if (OB_FAIL(check_stmt_need_multi_partition_dml(stmt,
index_dml_infos,
is_multi_part_dml))) {
@ -12875,6 +12880,7 @@ int ObLogPlan::create_for_update_plan(ObLogicalOperator *&top,
} else if (OB_FAIL(check_need_multi_partition_dml(*get_stmt(),
*top,
index_dml_infos,
false,
is_multi_part_dml,
is_result_local))) {
LOG_WARN("failed to check need multi-partition dml", K(ret));

1
src/sql/optimizer/ob_log_plan.h
View File

@ -390,6 +390,7 @@ public:
int check_need_multi_partition_dml(const ObDMLStmt &stmt,
ObLogicalOperator &top,
const ObIArray<IndexDMLInfo *> &index_dml_infos,
bool use_parallel_das,
bool &is_multi_part_dml,
bool &is_result_local);

3
src/sql/optimizer/ob_log_update.cpp
View File

@ -75,6 +75,9 @@ int ObLogUpdate::get_plan_item_info(PlanText &plan_text,
pos = pos - 2;
}
OZ(BUF_PRINTF(")"));
if (OB_SUCC(ret) && get_das_dop() > 0) {
ret = BUF_PRINTF(", das_dop=%ld", this->get_das_dop());
}
END_BUF_PRINT(plan_item.special_predicates_,
plan_item. special_predicates_len_);
}

5
src/sql/optimizer/ob_merge_log_plan.cpp
View File

@ -387,6 +387,9 @@ int ObMergeLogPlan::allocate_merge_as_top(ObLogicalOperator *&top,
merge_op->set_child(ObLogicalOperator::first_child, top);
merge_op->set_is_multi_part_dml(is_multi_part_dml);
merge_op->set_table_partition_info(table_partition_info);
if (get_can_use_parallel_das_dml()) {
merge_op->set_das_dop(max_dml_parallel_);
}
if (NULL != equal_pairs && OB_FAIL(merge_op->set_equal_pairs(*equal_pairs))) {
LOG_WARN("failed to set equal pairs", K(ret));
} else if (OB_FAIL(merge_op->compute_property())) {
@ -427,6 +430,8 @@ int ObMergeLogPlan::check_merge_need_multi_partition_dml(ObLogicalOperator &top,
LOG_WARN("failed to check stmt need multi partition dml", K(ret));
} else if (is_multi_part_dml) {
/*do nothing*/
} else if (use_parallel_das_dml_) {
is_multi_part_dml = true;
} else if (OB_FAIL(check_merge_stmt_need_multi_partition_dml(is_multi_part_dml, is_one_part_table))) {
LOG_WARN("failed to check need multi-partition dml", K(ret));
} else if (is_multi_part_dml) {

189
src/sql/optimizer/ob_optimizer.cpp
View File

@ -319,6 +319,118 @@ int ObOptimizer::get_session_parallel_info(int64_t &force_parallel_dop,
return ret;
}
int ObOptimizer::check_dml_parallel_mode()
{
int ret = OB_SUCCESS;
if (ctx_.can_use_pdml() && ctx_.get_can_use_parallel_das_dml()) {
if (ctx_.get_global_hint().get_parallel_das_dml_option() == ObParallelDASOption::ENABLE) {
ctx_.set_can_use_pdml(false);
LOG_TRACE("parallel_das_dml hint cause force use das parallel");
} else {
ctx_.set_can_use_parallel_das_dml(false);
LOG_TRACE("pdml and parallel das dml all supported, choose pdml");
}
}
return ret;
}
int ObOptimizer::check_parallel_das_dml_supported_feature(const ObDelUpdStmt &pdml_stmt,
const ObSQLSessionInfo &session,
bool &use_parallel_das_dml)
{
int ret = OB_SUCCESS;
if (pdml_stmt.is_ignore()) {
use_parallel_das_dml = false;
} else if (ctx_.has_dblink()) {
use_parallel_das_dml = false;
LOG_TRACE("has dblink not use parallel das dml");
} else if (!ctx_.has_fk() && ctx_.contain_user_nested_sql()) {
use_parallel_das_dml = false;
LOG_TRACE("ctx has user nexted sql not use parallel das dml");
} else if (ctx_.is_online_ddl()) {
use_parallel_das_dml = false;
LOG_TRACE("is online ddl not use parallel das dml");
}
return ret;
}
int ObOptimizer::check_parallel_das_dml_enabled(const ObDMLStmt &stmt,
ObSQLSessionInfo &session)
{
int ret = OB_SUCCESS;
bool can_use_parallel_das_dml = true;
ObSqlCtx *sql_ctx = NULL;
ObQueryCtx *query_ctx = NULL;
bool session_enable_pdml = false;
bool session_enable_auto_dop = false;
uint64_t session_pdml_dop = ObGlobalHint::UNSET_PARALLEL;
if (OB_ISNULL(ctx_.get_exec_ctx()) || OB_ISNULL(query_ctx = ctx_.get_query_ctx())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null", K(ret), K(ctx_.get_exec_ctx()), K(query_ctx));
} else if (OB_ISNULL(sql_ctx = ctx_.get_exec_ctx()->get_sql_ctx())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null", K(ret), K(ctx_.get_exec_ctx()));
} else if (!session.enable_parallel_das_dml()) {
can_use_parallel_das_dml = false;
LOG_TRACE("enable_das_parallel_execution is false, can't submit task parallel");
} else if (!stmt.is_px_dml_supported_stmt()) {
can_use_parallel_das_dml = false;
LOG_TRACE("stmt can't support parallel_das_dml");
} else if (ctx_.has_var_assign() && !ctx_.is_var_assign_only_in_root_stmt()) {
// It can be supported, but we won’t let it go for now and wait for the follow-up.
can_use_parallel_das_dml = false;
LOG_TRACE("has var_assign, can't support parallel_das_dml now");
} else if (!session.is_user_session()) {
// not user request
can_use_parallel_das_dml = false;
LOG_TRACE("not user request, can't support parallel_das_dml");
} else if (OB_FAIL(check_parallel_das_dml_supported_feature(static_cast<const ObDelUpdStmt&>(stmt),
session,
can_use_parallel_das_dml))) {
} else if (!can_use_parallel_das_dml) {
LOG_TRACE("not support parallel das dml");
} else if (ctx_.get_global_hint().get_parallel_das_dml_option() == ObParallelDASOption::DISABLE) {
can_use_parallel_das_dml = false;
LOG_TRACE("hint force close das parallel, can't support parallel_das_dml");
} else if (query_ctx->get_query_hint().has_outline_data() &&
ctx_.get_global_hint().get_parallel_das_dml_option() != ObParallelDASOption::ENABLE) {
can_use_parallel_das_dml = false;
LOG_TRACE("has outline data and hint disable, can't support parallel_das_dml",
K(ctx_.get_global_hint().get_parallel_das_dml_option()),
K(query_ctx->get_query_hint().has_outline_data()));
} else if (ctx_.can_use_pdml()) {
// can use pdml must can use parallel_das_dml
LOG_TRACE("pdml is enabled, can use parallel das");
} else if (ctx_.get_global_hint().get_pdml_option() == ObPDMLOption::ENABLE ||
ctx_.get_global_hint().get_parallel_das_dml_option() == ObParallelDASOption::ENABLE) {
LOG_TRACE("can use parallel_das_dml by pdml hint",
K(ctx_.get_global_hint().get_pdml_option()),
K(ctx_.get_global_hint().get_parallel_das_dml_option()));
} else if (ctx_.get_global_hint().enable_auto_dop()) {
// enable parallel das dml by auto dop hint
} else if (OB_FAIL(session.get_parallel_degree_policy_enable_auto_dop(session_enable_auto_dop))) {
LOG_WARN("failed to get sys variable for parallel degree policy", K(ret));
} else if (session_enable_auto_dop && !ctx_.get_global_hint().has_parallel_hint()) {
// session enable parallel dml by auto dop
} else if (OB_FAIL(session.get_enable_parallel_dml(session_enable_pdml))
|| OB_FAIL(session.get_force_parallel_dml_dop(session_pdml_dop))) {
LOG_WARN("failed to get sys variable for parallel dml", K(ret));
} else if (session_enable_pdml || ObGlobalHint::DEFAULT_PARALLEL < session_pdml_dop) {
// enable parallel das dml by session
} else {
can_use_parallel_das_dml = false;
LOG_TRACE("other scense, can't support parallel_das_dml");
}
if (OB_SUCC(ret)) {
ctx_.set_can_use_parallel_das_dml(can_use_parallel_das_dml);
}
return ret;
}
int ObOptimizer::check_pdml_enabled(const ObDMLStmt &stmt,
const ObSQLSessionInfo &session)
{
@ -472,24 +584,6 @@ int ObOptimizer::check_pdml_supported_feature(const ObDelUpdStmt &pdml_stmt,
ctx_.add_plan_note(PDML_DISABLED_BY_UPDATE_NOW);
}
}
} else if (stmt::T_MERGE == pdml_stmt.get_stmt_type()) {
bool update_rowkey = false;
ObSEArray<uint64_t, 4> index_ids;
if (OB_FAIL(schema_guard->get_all_unique_index(session.get_effective_tenant_id(),
main_table_tid,
index_ids))) {
LOG_WARN("failed to get all local unique index", K(ret));
} else if (OB_FAIL(index_ids.push_back(main_table_tid))) {
LOG_WARN("failed to push back index ids", K(ret));
} else if (OB_FAIL(check_merge_stmt_is_update_index_rowkey(session,
pdml_stmt,
index_ids,
update_rowkey))) {
LOG_WARN("failed to check merge stmt update rowkey", K(ret));
} else if (update_rowkey) {
is_use_pdml = false;
ctx_.add_plan_note(PDML_DISABLE_BY_MERGE_UPDATE_PK);
}
}
}
return ret;
@ -541,6 +635,10 @@ int ObOptimizer::init_env_info(ObDMLStmt &stmt)
LOG_WARN("fail to extract opt ctx basic flags", K(ret));
} else if (OB_FAIL(check_pdml_enabled(stmt, *session_info))) {
LOG_WARN("fail to check enable pdml", K(ret));
} else if (OB_FAIL(check_parallel_das_dml_enabled(stmt, *session_info))) {
LOG_WARN("fail to check enable parallel das dml", K(ret));
} else if (OB_FAIL(check_dml_parallel_mode())) {
LOG_WARN("fail to check force use parallel das dml", K(ret));
} else if (OB_FAIL(init_parallel_policy(stmt, *session_info))) { // call after check pdml enabled
LOG_WARN("fail to check enable pdml", K(ret));
} else if (OB_FAIL(init_replica_policy(stmt, *session_info))) {
@ -1254,58 +1352,3 @@ int ObOptimizer::init_system_stat()
}
return ret;
}
int ObOptimizer::check_merge_stmt_is_update_index_rowkey(const ObSQLSessionInfo &session,
const ObDMLStmt &stmt,
const ObIArray<uint64_t> &index_ids,
bool &is_update)
{
int ret = OB_SUCCESS;
share::schema::ObSchemaGetterGuard *schema_guard = ctx_.get_schema_guard();
const ObTableSchema *table_schema = NULL;
const ObMergeStmt &merge_stmt = static_cast<const ObMergeStmt&>(stmt);
const ObMergeTableInfo& merge_table_info = merge_stmt.get_merge_table_info();
ObSEArray<uint64_t, 4> rowkey_ids;
ObSEArray<uint64_t, 4> tmp_rowkey_ids;
const ObColumnRefRawExpr* column_expr = nullptr;
const ColumnItem* column_item = nullptr;
is_update = false;
if (OB_ISNULL(schema_guard) ||
OB_UNLIKELY(stmt::T_MERGE != stmt.get_stmt_type())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("the schema guard is null", K(ret));
}
for (int64_t i = 0; OB_SUCC(ret) && i < index_ids.count(); ++i) {
tmp_rowkey_ids.reuse();
if (OB_FAIL(schema_guard->get_table_schema(session.get_effective_tenant_id(),
index_ids.at(i),
table_schema))) {
LOG_WARN("failed to get table schema", K(ret));
} else if (OB_ISNULL(table_schema)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpect null", K(ret), K(table_schema));
} else if (OB_FAIL(table_schema->get_rowkey_info().get_column_ids(tmp_rowkey_ids))) {
LOG_WARN("failed to get column ids", K(ret));
} else if (OB_FAIL(append_array_no_dup(rowkey_ids, tmp_rowkey_ids))) {
LOG_WARN("failed to append array no dup", K(ret));
}
}
for (int64_t i = 0; OB_SUCC(ret) && !is_update && i < merge_table_info.assignments_.count(); ++i) {
if (OB_ISNULL(column_expr = merge_table_info.assignments_.at(i).column_expr_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get null column expr", K(ret));
} else if (merge_table_info.table_id_ != merge_table_info.loc_table_id_) {
if (OB_ISNULL(column_item = stmt.get_column_item_by_id(column_expr->get_table_id(),
column_expr->get_column_id()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get null column item", K(ret));
} else {
is_update = ObOptimizerUtil::find_item(rowkey_ids, column_item->base_cid_);
}
} else {
is_update = ObOptimizerUtil::find_item(rowkey_ids, column_expr->get_column_id());
}
}
return ret;
}

11
src/sql/optimizer/ob_optimizer.h
View File

@ -210,11 +210,14 @@ namespace sql
int check_pdml_supported_feature(const ObDelUpdStmt &pdml_stmt,
const ObSQLSessionInfo &session,
bool &is_use_pdml);
int check_parallel_das_dml_enabled(const ObDMLStmt &stmt,
ObSQLSessionInfo &session);
int check_parallel_das_dml_supported_feature(const ObDelUpdStmt &pdml_stmt,
const ObSQLSessionInfo &session,
bool &use_parallel_das_dml);
int check_dml_parallel_mode();
int check_is_heap_table(const ObDMLStmt &stmt);
int check_merge_stmt_is_update_index_rowkey(const ObSQLSessionInfo &session,
const ObDMLStmt &stmt,
const ObIArray<uint64_t> &index_ids,
bool &is_update);
int extract_column_usage_info(const ObDMLStmt &stmt);
int analyze_one_expr(const ObDMLStmt &stmt, const ObRawExpr *expr);
int add_column_usage_arg(const ObDMLStmt &stmt,

Some files were not shown because too many files have changed in this diff Show More