hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[CP] fix concurrency of direct load and ddl

Browse Source
This commit is contained in:
suz-yang
2024-06-17 14:14:59 +00:00
committed by ob-robot
parent 6964369909
commit b859aea8d6
31 changed files with 687 additions and 368 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
src/sql/engine/cmd/ob_load_data_direct_impl.cpp
View File

@ -69,7 +69,7 @@ ObLoadDataDirectImpl::LoadExecuteParam::LoadExecuteParam()
method_(ObDirectLoadMethod::INVALID_METHOD),
insert_mode_(ObDirectLoadInsertMode::INVALID_INSERT_MODE)
{
store_column_idxs_.set_tenant_id(MTL_ID());
column_ids_.set_tenant_id(MTL_ID());
}
bool ObLoadDataDirectImpl::LoadExecuteParam::is_valid() const
@ -90,7 +90,7 @@ bool ObLoadDataDirectImpl::LoadExecuteParam::is_valid() const
(storage::ObDirectLoadInsertMode::INC_REPLACE == insert_mode_
? sql::ObLoadDupActionType::LOAD_REPLACE == dup_action_
: true) &&
data_access_param_.is_valid() && !store_column_idxs_.empty();
data_access_param_.is_valid() && !column_ids_.empty();
}
/**
@ -1738,6 +1738,7 @@ int ObLoadDataDirectImpl::execute(ObExecContext &ctx, ObLoadDataStmt &load_stmt)
load_stmt_ = &load_stmt;
const ObLoadArgument &load_args = load_stmt_->get_load_arguments();
ObSQLSessionInfo *session = nullptr;
ObSchemaGetterGuard *schema_guard = nullptr;
int64_t total_line_count = 0;
if (OB_UNLIKELY(load_args.file_iter_.count() > ObTableLoadSequenceNo::MAX_DATA_ID)) {
@ -1745,7 +1746,8 @@ int ObLoadDataDirectImpl::execute(ObExecContext &ctx, ObLoadDataStmt &load_stmt)
LOG_WARN("not support file counts more than 65535", KR(ret), K(load_args.file_iter_.count()));
FORWARD_USER_ERROR_MSG(ret, "not support file counts %ld more than 65535", load_args.file_iter_.count());
} else if (OB_ISNULL(session = ctx.get_my_session()) || OB_ISNULL(ctx.get_stmt_factory()) ||
OB_ISNULL(ctx.get_stmt_factory()->get_query_ctx())) {
OB_ISNULL(ctx.get_stmt_factory()->get_query_ctx()) || OB_ISNULL(ctx_->get_sql_ctx()) ||
OB_ISNULL(schema_guard = ctx_->get_sql_ctx()->schema_guard_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ctx is unexpected", KR(ret), K(ctx));
} else if (OB_FAIL(plan_.set_vars(ctx.get_stmt_factory()->get_query_ctx()->variables_))) {
@ -1782,6 +1784,11 @@ int ObLoadDataDirectImpl::execute(ObExecContext &ctx, ObLoadDataStmt &load_stmt)
if (OB_SUCC(ret)) {
if (OB_FAIL(init_execute_param())) {
LOG_WARN("fail to init execute param", KR(ret), K(ctx), K(load_stmt));
} else if (OB_FAIL(ObTableLoadService::check_support_direct_load(*schema_guard,
execute_param_.table_id_,
execute_param_.method_,
execute_param_.insert_mode_))) {
LOG_WARN("fail to check support direct load", KR(ret));
} else if (OB_FAIL(init_execute_context())) {
LOG_WARN("fail to init execute context", KR(ret), K(ctx), K(load_stmt));
} else {
@ -1960,66 +1967,29 @@ int ObLoadDataDirectImpl::init_execute_param()
data_access_param.file_cs_type_ = load_args.file_cs_type_;
data_access_param.access_info_ = load_args.access_info_;
}
// store_column_idxs_
// column_ids_
if (OB_SUCC(ret)) {
if (OB_FAIL(init_store_column_idxs(execute_param_.store_column_idxs_))) {
LOG_WARN("fail to init store column idxs", KR(ret));
}
}
return ret;
}
int ObLoadDataDirectImpl::init_store_column_idxs(ObIArray<int64_t> &store_column_idxs)
{
int ret = OB_SUCCESS;
const ObLoadArgument &load_args = load_stmt_->get_load_arguments();
const ObIArray<ObLoadDataStmt::FieldOrVarStruct> &field_or_var_list =
load_stmt_->get_field_or_var_list();
const uint64_t tenant_id = load_args.tenant_id_;
const uint64_t table_id = load_args.table_id_;
ObSchemaGetterGuard schema_guard;
const ObTableSchema *table_schema = nullptr;
ObArray<ObColDesc> column_descs;
column_descs.set_tenant_id(MTL_ID());
if (OB_FAIL(ObMultiVersionSchemaService::get_instance().get_tenant_schema_guard(tenant_id,
schema_guard))) {
LOG_WARN("fail to get tenant schema guard", KR(ret), K(tenant_id));
} else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, table_id, table_schema))) {
LOG_WARN("fail to get table schema", KR(ret), K(tenant_id), K(table_id));
} else if (OB_ISNULL(table_schema)) {
ret = OB_TABLE_NOT_EXIST;
LOG_WARN("table not exist", KR(ret), K(tenant_id), K(table_id));
} else if (OB_FAIL(table_schema->get_column_ids(column_descs))) {
STORAGE_LOG(WARN, "fail to get column descs", KR(ret), KPC(table_schema));
} else {
bool found_column = true;
for (int64_t i = 0; OB_SUCC(ret) && OB_LIKELY(found_column) && i < column_descs.count(); ++i) {
const ObColDesc &col_desc = column_descs.at(i);
const ObColumnSchemaV2 *col_schema = table_schema->get_column_schema(col_desc.col_id_);
if (OB_ISNULL(col_schema)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null column schema", KR(ret), K(col_desc));
} else {
found_column = col_schema->is_hidden();
}
// 在源数据的列数组中找到对应的列
for (int64_t j = 0; OB_SUCC(ret) && OB_LIKELY(!found_column) && j < field_or_var_list.count();
++j) {
const ObLoadDataStmt::FieldOrVarStruct &field_or_var_struct = field_or_var_list.at(j);
if (col_desc.col_id_ == field_or_var_struct.column_id_) {
found_column = true;
if (OB_FAIL(store_column_idxs.push_back(j))) {
LOG_WARN("fail to push back column desc", KR(ret), K(store_column_idxs), K(i),
K(col_desc), K(j), K(field_or_var_struct));
}
}
}
}
if (OB_SUCC(ret) && OB_UNLIKELY(!found_column)) {
ObSchemaGetterGuard *schema_guard = ctx_->get_sql_ctx()->schema_guard_;
int64_t column_count = 0;
execute_param_.column_ids_.reset();
if (OB_FAIL(ObTableLoadSchema::get_user_column_count(*schema_guard,
execute_param_.tenant_id_,
execute_param_.table_id_,
column_count))) {
LOG_WARN("fail to get user column count", KR(ret));
} else if (OB_UNLIKELY(column_count != field_or_var_list.count())) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("not supported incomplete column data", KR(ret), K(store_column_idxs),
K(column_descs), K(field_or_var_list));
FORWARD_USER_ERROR_MSG(ret, "not supported incomplete column data");
LOG_WARN("not contain all columns is not supported", KR(ret), K(column_count),
K(field_or_var_list));
}
for (int64_t i = 0; OB_SUCC(ret) && i < field_or_var_list.count(); ++i) {
const ObLoadDataStmt::FieldOrVarStruct &field_or_var_struct = field_or_var_list.at(i);
if (OB_UNLIKELY(!field_or_var_struct.is_table_column_)) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("var is not supported", KR(ret), K(field_or_var_struct), K(i), K(field_or_var_list));
} else if (OB_FAIL(execute_param_.column_ids_.push_back(field_or_var_struct.column_id_))) {
LOG_WARN("fail to push back column id", KR(ret));
}
}
}
return ret;
@ -2037,14 +2007,15 @@ int ObLoadDataDirectImpl::init_execute_context()
load_param.session_count_ = execute_param_.parallel_;
load_param.batch_size_ = execute_param_.batch_row_count_;
load_param.max_error_row_count_ = execute_param_.max_error_rows_;
load_param.column_count_ = execute_param_.store_column_idxs_.count();
load_param.column_count_ = execute_param_.column_ids_.count();
load_param.need_sort_ = execute_param_.need_sort_;
load_param.dup_action_ = execute_param_.dup_action_;
load_param.px_mode_ = false;
load_param.online_opt_stat_gather_ = execute_param_.online_opt_stat_gather_;
load_param.method_ = execute_param_.method_;
load_param.insert_mode_ = execute_param_.insert_mode_;
if (OB_FAIL(direct_loader_.init(load_param, execute_param_.store_column_idxs_,
if (OB_FAIL(direct_loader_.init(load_param,
execute_param_.column_ids_,
&execute_ctx_.exec_ctx_))) {
LOG_WARN("fail to init direct loader", KR(ret));
} else if (OB_FAIL(init_logger())) {