hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Fix table load task concurrency problem

Browse Source
This commit is contained in:
obdev 2023-02-17 08:11:54 +00:00 committed by ob-robot
parent b0b540090f
commit 38ec0851ad
36 changed files with 1258 additions and 751 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/observer/table_load/ob_table_load_abort_processor.cpp
View File

@ -26,16 +26,20 @@ int ObTableLoadAbortP::process()
LOG_WARN("fail to check_user_access", KR(ret));
} else {
ObTableLoadTableCtx *table_ctx = nullptr;
ObTableLoadKey key(credential_.tenant_id_, arg_.table_id_);
ObTableLoadUniqueKey key(arg_.table_id_, arg_.task_id_);
if (OB_FAIL(ObTableLoadService::get_ctx(key, table_ctx))) {
if (OB_UNLIKELY(OB_ENTRY_NOT_EXIST == ret)) {
LOG_WARN("fail to get table ctx", KR(ret), K(key));
} else {
ret = OB_SUCCESS;
}
} else {
if (OB_FAIL(ObTableLoadCoordinator::abort_ctx(table_ctx))) {
LOG_WARN("fail to abort coordinator ctx", KR(ret));
if (OB_FAIL(ObTableLoadUtils::init_session_info(credential_.user_id_, session_info_))) {
LOG_WARN("fail to init session info", KR(ret));
} else if (OB_FAIL(ObTableLoadService::remove_ctx(table_ctx))) {
LOG_WARN("fail to remove table ctx", KR(ret), K(key));
} else {
ObTableLoadCoordinator::abort_ctx(table_ctx, session_info_);
}
}
if (OB_NOT_NULL(table_ctx)) {
@ -62,16 +66,16 @@ int ObTableLoadAbortPeerP::process()
LOG_WARN("fail to check_user_access", KR(ret));
} else {
ObTableLoadTableCtx *table_ctx = nullptr;
ObTableLoadKey key(credential_.tenant_id_, arg_.table_id_);
ObTableLoadUniqueKey key(arg_.table_id_, arg_.task_id_);
if (OB_FAIL(ObTableLoadService::get_ctx(key, table_ctx))) {
if (OB_UNLIKELY(OB_ENTRY_NOT_EXIST == ret)) {
LOG_WARN("fail to get table ctx", KR(ret), K(key));
}
} else {
if (OB_FAIL(ObTableLoadStore::abort_ctx(table_ctx))) {
LOG_WARN("fail to abort store ctx", KR(ret));
} else if (OB_FAIL(ObTableLoadService::remove_ctx(table_ctx))) {
if (OB_FAIL(ObTableLoadService::remove_ctx(table_ctx))) {
LOG_WARN("fail to remove table ctx", KR(ret), K(key));
} else {
ObTableLoadStore::abort_ctx(table_ctx);
}
}
if (OB_NOT_NULL(table_ctx)) {

1
src/observer/table_load/ob_table_load_abort_processor.h
View File

@ -30,6 +30,7 @@ private:
private:
const ObGlobalContext &gctx_;
table::ObTableApiCredential credential_;
sql::ObSQLSessionInfo session_info_;
};
class ObTableLoadAbortPeerP : public obrpc::ObRpcProcessor<obrpc::ObTableRpcProxy::ObRpc<obrpc::OB_TABLE_API_LOAD_ABORT_PEER> >

257
src/observer/table_load/ob_table_load_begin_processor.cpp
View File

@ -4,10 +4,11 @@
#define USING_LOG_PREFIX SERVER
#include "observer/table_load/ob_table_load_begin_processor.h"
#include "observer/omt/ob_multi_tenant.h"
#include "observer/omt/ob_tenant.h"
#include "observer/table_load/ob_table_load_begin_processor.h"
#include "observer/table_load/ob_table_load_coordinator.h"
#include "observer/table_load/ob_table_load_redef_table.h"
#include "observer/table_load/ob_table_load_schema.h"
#include "observer/table_load/ob_table_load_service.h"
#include "observer/table_load/ob_table_load_store.h"
@ -21,12 +22,28 @@ using namespace table;
using namespace sql;
using namespace omt;
/**
* ObTableLoadBeginP
*/
ObTableLoadBeginP::ObTableLoadBeginP(const ObGlobalContext &gctx) : gctx_(gctx), table_ctx_(nullptr)
{
}
ObTableLoadBeginP::~ObTableLoadBeginP()
{
if (OB_NOT_NULL(table_ctx_)) {
ObTableLoadService::put_ctx(table_ctx_);
table_ctx_ = nullptr;
}
}
int ObTableLoadBeginP::process()
{
int ret = OB_SUCCESS;
uint64_t table_id = OB_INVALID_ID;
ObTableLoadArray<ObString> column_names;
int32_t session_count = 0;
bool is_new = false;
if (OB_FAIL(check_user_access(arg_.credential_))) {
LOG_WARN("fail to check_user_access", KR(ret), K_(arg));
@ -53,80 +70,89 @@ int ObTableLoadBeginP::process()
}
if (OB_SUCC(ret)) {
if (OB_FAIL(ObTableLoadUtils::init_session_info(credential_.user_id_, session_info_))) {
LOG_WARN("fail to init session info", KR(ret));
}
}
// get the existing table ctx if it exists
if (OB_SUCC(ret)) {
ObTableLoadKey key(credential_.tenant_id_, table_id);
if (OB_FAIL(ObTableLoadService::get_ctx(key, table_ctx_))) {
if (OB_UNLIKELY(OB_ENTRY_NOT_EXIST != ret)) {
LOG_WARN("fail to get ctx", KR(ret), K(key));
} else {
ret = OB_SUCCESS;
table_ctx_ = nullptr;
}
} else if (OB_UNLIKELY(!ObTableLoadCoordinator::is_ctx_inited(table_ctx_))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected uninited coordinator ctx", KR(ret));
}
}
// create new table ctx if it does not exist
if (OB_SUCC(ret) && nullptr == table_ctx_) {
ObTenant *tenant = nullptr;
if (OB_FAIL(GCTX.omt_->get_tenant(credential_.tenant_id_, tenant))) {
if (arg_.config_.flag_.data_type_ >=
static_cast<uint64_t>(ObTableLoadDataType::MAX_DATA_TYPE)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("data type is error", KR(ret), K(arg_.config_.flag_.data_type_));
} else if (OB_FAIL(GCTX.omt_->get_tenant(credential_.tenant_id_, tenant))) {
LOG_WARN("fail to get tenant handle", KR(ret), K(credential_.tenant_id_));
} else {
session_count = MIN(arg_.config_.session_count_, (int32_t)tenant->unit_max_cpu());
ObTableLoadParam param;
param.tenant_id_ = credential_.tenant_id_;
param.table_id_ = table_id;
param.batch_size_ = arg_.config_.batch_size_;
param.session_count_ = MIN(arg_.config_.session_count_, (int32_t)tenant->unit_max_cpu());
param.max_error_row_count_ = arg_.config_.max_error_row_count_;
param.column_count_ = column_names.count();
param.need_sort_ = arg_.config_.flag_.is_need_sort_;
param.px_mode_ = false;
param.online_opt_stat_gather_ = false;
param.data_type_ = static_cast<ObTableLoadDataType>(arg_.config_.flag_.data_type_);
param.dup_action_ = ObLoadDupActionType::LOAD_STOP_ON_DUP;
if (OB_FAIL(param.normalize())) {
LOG_WARN("fail to normalize param", KR(ret));
}
// check support
else if (OB_FAIL(ObTableLoadService::check_support_direct_load(table_id))) {
LOG_WARN("fail to check support direct load", KR(ret), K(table_id));
}
// create table ctx
else if (OB_FAIL(create_table_ctx(param, idx_array_, session_info_, table_ctx_))) {
LOG_WARN("fail to create table ctx", KR(ret));
} else {
is_new = true;
}
}
}
if (OB_SUCC(ret)) {
ObTableLoadTableCtx *table_ctx = nullptr;
bool is_new = false;
ObTableLoadParam param;
param.tenant_id_ = credential_.tenant_id_;
param.table_id_ = table_id;
param.batch_size_ = arg_.config_.batch_size_;
param.session_count_ = session_count;
param.max_error_row_count_ = arg_.config_.max_error_row_count_;
param.column_count_ = column_names.count();
param.need_sort_ = arg_.config_.flag_.is_need_sort_;
param.px_mode_ = false;
param.online_opt_stat_gather_ = false;
param.dup_action_ = ObLoadDupActionType::LOAD_STOP_ON_DUP;
if (OB_FAIL(param.normalize())) {
LOG_WARN("fail to normalize param", KR(ret));
} else if (arg_.config_.flag_.data_type_ >= static_cast<uint64_t>(ObTableLoadDataType::MAX_DATA_TYPE)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("data type is error", KR(ret), K(arg_.config_.flag_.data_type_));
} else {
param.data_type_ = static_cast<ObTableLoadDataType>(arg_.config_.flag_.data_type_);
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(ObTableLoadService::create_ctx(param, table_ctx, is_new))) {
LOG_WARN("fail to create table ctx", KR(ret), K(param));
} else {
if (is_new) { // 新建的ctx需要初始化
if (OB_FAIL(table_ctx->init_session_info(credential_.user_id_))) {
LOG_WARN("fail to init session info", KR(ret));
} else if (OB_FAIL(ObTableLoadCoordinator::init_ctx(table_ctx, idx_array_, nullptr))) {
LOG_WARN("fail to coordinator init ctx", KR(ret));
}
} else { // 已存在的ctx检查是否已初始化
if (OB_UNLIKELY(!ObTableLoadCoordinator::is_ctx_inited(table_ctx))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected uninited coordinator ctx", KR(ret));
}
}
if (OB_FAIL(ret)) {
int tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(ObTableLoadService::remove_ctx(table_ctx))) {
LOG_WARN("fail to remove table ctx", KR(tmp_ret));
}
}
}
if (OB_SUCC(ret)) {
ObTableLoadCoordinator coordinator(table_ctx);
if (OB_FAIL(coordinator.init())) {
LOG_WARN("fail to init coordinator", KR(ret));
} else if (is_new && OB_FAIL(coordinator.begin())) { // 新建的ctx需要begin
LOG_WARN("fail to coordinator begin", KR(ret));
} else if (OB_FAIL(coordinator.get_status(result_.status_, result_.error_code_))) {
LOG_WARN("fail to coordinator get status", KR(ret));
}
}
if (OB_NOT_NULL(table_ctx)) {
ObTableLoadService::put_ctx(table_ctx);
table_ctx = nullptr;
ObTableLoadCoordinator coordinator(table_ctx_);
if (OB_FAIL(coordinator.init())) {
LOG_WARN("fail to init coordinator", KR(ret));
} else if (is_new && OB_FAIL(coordinator.begin())) { // 新建的ctx需要begin
LOG_WARN("fail to coordinator begin", KR(ret));
} else if (OB_FAIL(coordinator.get_status(result_.status_, result_.error_code_))) {
LOG_WARN("fail to coordinator get status", KR(ret));
}
}
if (OB_SUCC(ret)) {
result_.table_id_ = table_id;
result_.task_id_ = table_ctx_->ddl_param_.task_id_;
result_.column_names_ = column_names;
}
if (OB_FAIL(ret)) {
if (nullptr != table_ctx_) {
ObTableLoadService::remove_ctx(table_ctx_);
ObTableLoadCoordinator::abort_ctx(table_ctx_, session_info_);
}
}
return ret;
}
@ -157,9 +183,65 @@ int ObTableLoadBeginP::init_idx_array(const ObTableSchema *table_schema)
return ret;
}
int ObTableLoadBeginP::create_table_ctx(const ObTableLoadParam &param,
const ObIArray<int64_t> &idx_array,
ObSQLSessionInfo &session_info,
ObTableLoadTableCtx *&table_ctx)
{
int ret = OB_SUCCESS;
table_ctx = nullptr;
ObTableLoadDDLParam ddl_param;
// start redef table
ObTableLoadRedefTableStartArg start_arg;
ObTableLoadRedefTableStartRes start_res;
start_arg.tenant_id_ = param.tenant_id_;
start_arg.table_id_ = param.table_id_;
start_arg.parallelism_ = param.session_count_;
if (OB_FAIL(ObTableLoadRedefTable::start(start_arg, start_res, session_info))) {
LOG_WARN("fail to start redef table", KR(ret), K(start_arg));
} else {
ddl_param.dest_table_id_ = start_res.dest_table_id_;
ddl_param.task_id_ = start_res.task_id_;
ddl_param.schema_version_ = start_res.schema_version_;
}
if (OB_SUCC(ret)) {
const int64_t origin_timeout_ts = THIS_WORKER.get_timeout_ts();
THIS_WORKER.set_timeout_ts(ObTimeUtil::current_time() + arg_.timeout_);
if (OB_ISNULL(table_ctx = ObTableLoadService::alloc_ctx())) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to alloc table ctx", KR(ret), K(param));
} else if (OB_FAIL(table_ctx->init(param, ddl_param))) {
LOG_WARN("fail to init table ctx", KR(ret));
} else if (OB_FAIL(ObTableLoadCoordinator::init_ctx(table_ctx, idx_array_,
session_info.get_priv_user_id()))) {
LOG_WARN("fail to coordinator init ctx", KR(ret));
} else if (OB_FAIL(ObTableLoadService::add_ctx(table_ctx))) {
LOG_WARN("fail to add ctx", KR(ret));
}
THIS_WORKER.set_timeout_ts(origin_timeout_ts);
}
if (OB_FAIL(ret)) {
int tmp_ret = OB_SUCCESS;
if (ddl_param.is_valid()) {
ObTableLoadRedefTableAbortArg abort_arg;
abort_arg.tenant_id_ = param.tenant_id_;
abort_arg.task_id_ = ddl_param.task_id_;
if (OB_TMP_FAIL(ObTableLoadRedefTable::abort(abort_arg, session_info))) {
LOG_WARN("fail to abort redef table", KR(tmp_ret), K(abort_arg));
}
}
if (nullptr != table_ctx) {
ObTableLoadService::free_ctx(table_ctx);
table_ctx = nullptr;
}
}
return ret;
}
/**
* ObTableLoadPreBeginPeerP
*/
int ObTableLoadPreBeginPeerP::deserialize()
{
arg_.partition_id_array_.set_allocator(allocator_);
@ -177,11 +259,9 @@ int ObTableLoadPreBeginPeerP::process()
if (OB_SUCC(ret)) {
ObTableLoadTableCtx *table_ctx = nullptr;
bool is_new = false;
ObTableLoadParam param;
param.tenant_id_ = credential_.tenant_id_;
param.table_id_ = arg_.table_id_;
param.target_table_id_ = arg_.target_table_id_;
param.batch_size_ = arg_.config_.batch_size_;
param.session_count_ = arg_.config_.session_count_;
param.max_error_row_count_ = arg_.config_.max_error_row_count_;
@ -190,22 +270,12 @@ int ObTableLoadPreBeginPeerP::process()
param.px_mode_ = arg_.px_mode_;
param.online_opt_stat_gather_ = arg_.online_opt_stat_gather_;
param.dup_action_ = arg_.dup_action_;
if (OB_FAIL(ObTableLoadService::create_ctx(param, table_ctx, is_new))) {
LOG_WARN("fail to create table ctx", KR(ret), K(param));
} else if (OB_UNLIKELY(!is_new)) { // 数据节点不能重复begin
ret = OB_ENTRY_EXIST;
LOG_WARN("table ctx exists", KR(ret));
} else {
if (OB_FAIL(ObTableLoadStore::init_ctx(table_ctx, arg_.ddl_task_id_, arg_.partition_id_array_,
arg_.target_partition_id_array_))) {
LOG_WARN("fail to store init ctx", KR(ret));
}
if (OB_FAIL(ret)) {
int tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(ObTableLoadService::remove_ctx(table_ctx))) {
LOG_WARN("fail to remove table ctx", KR(tmp_ret));
}
}
ObTableLoadDDLParam ddl_param;
ddl_param.dest_table_id_ = arg_.dest_table_id_;
ddl_param.task_id_ = arg_.task_id_;
ddl_param.schema_version_ = arg_.schema_version_;
if (OB_FAIL(create_table_ctx(param, ddl_param, table_ctx))) {
LOG_WARN("fail to create table ctx", KR(ret));
}
if (OB_SUCC(ret)) {
ObTableLoadStore store(table_ctx);
@ -229,6 +299,33 @@ int ObTableLoadPreBeginPeerP::check_user_access(const ObString &credential_str)
return ObTableLoadUtils::check_user_access(credential_str, gctx_, credential_);
}
int ObTableLoadPreBeginPeerP::create_table_ctx(const ObTableLoadParam &param,
const ObTableLoadDDLParam &ddl_param,
ObTableLoadTableCtx *&table_ctx)
{
int ret = OB_SUCCESS;
table_ctx = nullptr;
if (OB_ISNULL(table_ctx = ObTableLoadService::alloc_ctx())) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to alloc table ctx", KR(ret), K(param));
} else if (OB_FAIL(table_ctx->init(param, ddl_param))) {
LOG_WARN("fail to init table ctx", KR(ret));
} else if (OB_FAIL(ObTableLoadStore::init_ctx(table_ctx,
arg_.partition_id_array_,
arg_.target_partition_id_array_))) {
LOG_WARN("fail to store init ctx", KR(ret));
} else if (OB_FAIL(ObTableLoadService::add_ctx(table_ctx))) {
LOG_WARN("fail to add ctx", KR(ret));
}
if (OB_FAIL(ret)) {
if (nullptr != table_ctx) {
ObTableLoadService::free_ctx(table_ctx);
table_ctx = nullptr;
}
}
return ret;
}
/**
* ObTableLoadConfirmBeginPeerP
*/
@ -243,7 +340,7 @@ int ObTableLoadConfirmBeginPeerP::process()
if (OB_SUCC(ret)) {
ObTableLoadTableCtx *table_ctx = nullptr;
ObTableLoadKey key(credential_.tenant_id_, arg_.table_id_);
ObTableLoadUniqueKey key(arg_.table_id_, arg_.task_id_);
if (OB_FAIL(ObTableLoadService::get_ctx(key, table_ctx))) {
LOG_WARN("fail to get table ctx", KR(ret), K(key));
} else {

13
src/observer/table_load/ob_table_load_begin_processor.h
View File

@ -13,12 +13,15 @@ namespace oceanbase
{
namespace observer
{
class ObTableLoadParam;
class ObTableLoadDDLParam;
class ObTableLoadTableCtx;
class ObTableLoadBeginP : public obrpc::ObRpcProcessor<obrpc::ObTableRpcProxy::ObRpc<obrpc::OB_TABLE_API_LOAD_BEGIN> >
{
public:
explicit ObTableLoadBeginP(const ObGlobalContext &gctx) : gctx_(gctx) {}
virtual ~ObTableLoadBeginP() = default;
explicit ObTableLoadBeginP(const ObGlobalContext &gctx);
virtual ~ObTableLoadBeginP();
protected:
int process() override;
@ -26,13 +29,17 @@ protected:
private:
int check_user_access(const ObString &credential_str);
int init_idx_array(const ObTableSchema *table_schema);
int create_table_ctx(const ObTableLoadParam &param, const common::ObIArray<int64_t> &idx_array,
sql::ObSQLSessionInfo &session_info, ObTableLoadTableCtx *&table_ctx);
DISALLOW_COPY_AND_ASSIGN(ObTableLoadBeginP);
private:
const ObGlobalContext &gctx_;
common::ObArenaAllocator allocator_;
table::ObTableApiCredential credential_;
sql::ObSQLSessionInfo session_info_;
common::ObArray<int64_t> idx_array_;
ObTableLoadTableCtx *table_ctx_;
};
class ObTableLoadPreBeginPeerP : public obrpc::ObRpcProcessor<obrpc::ObTableRpcProxy::ObRpc<obrpc::OB_TABLE_API_LOAD_PRE_BEGIN_PEER> >
@ -48,6 +55,8 @@ protected:
private:
int check_user_access(const ObString &credential_str);
int create_table_ctx(const ObTableLoadParam &param, const ObTableLoadDDLParam &ddl_param,
ObTableLoadTableCtx *&table_ctx);
DISALLOW_COPY_AND_ASSIGN(ObTableLoadPreBeginPeerP);
private:

8
src/observer/table_load/ob_table_load_commit_processor.cpp
View File

@ -26,17 +26,19 @@ int ObTableLoadCommitP::process()
int ret = OB_SUCCESS;
if (OB_FAIL(check_user_access(arg_.credential_))) {
LOG_WARN("fail to check_user_access", KR(ret));
} else if (OB_FAIL(ObTableLoadUtils::init_session_info(credential_.user_id_, session_info_))) {
LOG_WARN("fail to init session info", KR(ret));
} else {
ObTableLoadTableCtx *table_ctx = nullptr;
ObExecContext *exec_ctx = nullptr;
ObTableLoadKey key(credential_.tenant_id_, arg_.table_id_);
ObTableLoadUniqueKey key(arg_.table_id_, arg_.task_id_);
if (OB_FAIL(ObTableLoadService::get_ctx(key, table_ctx))) {
LOG_WARN("fail to get table ctx", KR(ret), K(key));
} else {
ObTableLoadCoordinator coordinator(table_ctx);
if (OB_FAIL(coordinator.init())) {
LOG_WARN("fail to init coordinator", KR(ret));
} else if (OB_FAIL(coordinator.commit(*exec_ctx, result_.result_info_))) {
} else if (OB_FAIL(coordinator.commit(exec_ctx, session_info_, result_.result_info_))) {
LOG_WARN("fail to coordinator commit", KR(ret));
} else if (OB_FAIL(ObTableLoadService::remove_ctx(table_ctx))) {
LOG_WARN("fail to remove table ctx", KR(ret), K(key));
@ -66,7 +68,7 @@ int ObTableLoadCommitPeerP::process()
LOG_WARN("fail to check_user_access", KR(ret));
} else {
ObTableLoadTableCtx *table_ctx = nullptr;
ObTableLoadKey key(credential_.tenant_id_, arg_.table_id_);
ObTableLoadUniqueKey key(arg_.table_id_, arg_.task_id_);
if (OB_FAIL(ObTableLoadService::get_ctx(key, table_ctx))) {
LOG_WARN("fail to get table ctx", KR(ret), K(key));
} else {

1
src/observer/table_load/ob_table_load_commit_processor.h
View File

@ -30,6 +30,7 @@ private:
private:
const ObGlobalContext &gctx_;
table::ObTableApiCredential credential_;
sql::ObSQLSessionInfo session_info_;
};
class ObTableLoadCommitPeerP : public obrpc::ObRpcProcessor<obrpc::ObTableRpcProxy::ObRpc<obrpc::OB_TABLE_API_LOAD_COMMIT_PEER> >

121
src/observer/table_load/ob_table_load_coordinator.cpp
View File

@ -8,6 +8,7 @@
#include "observer/ob_server.h"
#include "observer/table_load/ob_table_load_coordinator_ctx.h"
#include "observer/table_load/ob_table_load_coordinator_trans.h"
#include "observer/table_load/ob_table_load_redef_table.h"
#include "observer/table_load/ob_table_load_service.h"
#include "observer/table_load/ob_table_load_stat.h"
#include "observer/table_load/ob_table_load_store.h"
@ -29,7 +30,7 @@ using namespace table;
using namespace share;
using namespace sql;
#define TABLE_LOAD_RPC_CALL(name, addr, request, result) \
#define TABLE_LOAD_RPC_CALL(name, addr, request, ...) \
if (OB_SUCC(ret)) { \
ObTimeoutCtx ctx; \
if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, DEFAULT_TIMEOUT_US))) { \
@ -39,7 +40,7 @@ using namespace sql;
.timeout(ctx.get_timeout()) \
.by(MTL_ID()) \
.to(addr) \
.name(request, result))) { \
.name(request, ##__VA_ARGS__))) { \
LOG_WARN("fail to rpc call " #name, KR(ret), K(addr), K(request)); \
} \
}
@ -63,36 +64,27 @@ bool ObTableLoadCoordinator::is_ctx_inited(ObTableLoadTableCtx *ctx)
}
int ObTableLoadCoordinator::init_ctx(ObTableLoadTableCtx *ctx, const ObIArray<int64_t> &idx_array,
sql::ObSQLSessionInfo *session_info)
uint64_t user_id)
{
int ret = OB_SUCCESS;
ObSchemaGetterGuard schema_guard;
const ObTableSchema *table_schema = nullptr;
if (OB_ISNULL(ctx)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid agrs", KR(ret));
} else if (OB_FAIL(ObTableLoadSchema::get_table_schema(ctx->param_.tenant_id_,
ctx->param_.table_id_,
schema_guard, table_schema))) {
LOG_WARN("fail to get table schema", KR(ret), K(ctx->param_.tenant_id_),
K(ctx->schema_.table_name_));
} else if (OB_FAIL(ObTableLoadSchema::check_constraints(ctx->param_.tenant_id_,
schema_guard, table_schema))) {
LOG_WARN("fail to check schema constraints", KR(ret), K(ctx->param_.tenant_id_));
} else if (OB_FAIL(ctx->init_coordinator_ctx(idx_array, session_info))) {
} else if (OB_FAIL(ctx->init_coordinator_ctx(idx_array, user_id))) {
LOG_WARN("fail to init coordinator ctx", KR(ret));
}
return ret;
}
int ObTableLoadCoordinator::abort_ctx(ObTableLoadTableCtx *ctx)
void ObTableLoadCoordinator::abort_ctx(ObTableLoadTableCtx *ctx, ObSQLSessionInfo &session_info)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!ctx->is_valid())) {
if (OB_UNLIKELY(nullptr == ctx || !ctx->is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), KPC(ctx));
} else if (OB_UNLIKELY(nullptr == ctx->coordinator_ctx_ || !ctx->coordinator_ctx_->is_valid())) {
// coordinator ctx not init, do nothing
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected invalid coordinator ctx", KR(ret), KP(ctx->coordinator_ctx_));
} else {
LOG_INFO("coordinator abort");
// 1. mark status abort, speed up background task exit
@ -100,19 +92,18 @@ int ObTableLoadCoordinator::abort_ctx(ObTableLoadTableCtx *ctx)
LOG_WARN("fail to set coordinator status abort", KR(ret));
}
// 2. mark all active trans abort
else if (OB_FAIL(abort_active_trans(ctx))) {
if (OB_FAIL(abort_active_trans(ctx))) {
LOG_WARN("fail to abort active trans", KR(ret));
}
// 3. abort peers ctx
else if (OB_FAIL(abort_peers_ctx(ctx))) {
if (OB_FAIL(abort_peers_ctx(ctx))) {
LOG_WARN("fail to abort peers ctx", KR(ret));
}
// 4. abort ddl redef table, release table lock
else if (OB_FAIL(ctx->coordinator_ctx_->abort())) {
LOG_WARN("fail to abort coordinator ctx", KR(ret));
// 4. abort redef table, release table lock
if (OB_FAIL(abort_redef_table(ctx, session_info))) {
LOG_WARN("fail to abort redef table", KR(ret));
}
}
return ret;
}
int ObTableLoadCoordinator::abort_active_trans(ObTableLoadTableCtx *ctx)
@ -154,12 +145,11 @@ int ObTableLoadCoordinator::abort_peers_ctx(ObTableLoadTableCtx *ctx)
ObTableLoadAbortPeerResult result;
request.credential_ = ctx->coordinator_ctx_->credential_;
request.table_id_ = ctx->param_.table_id_;
for (int64_t i = 0; OB_SUCC(ret) && i < all_addr_array.count(); ++i) {
request.task_id_ = ctx->ddl_param_.task_id_;
for (int64_t i = 0; i < all_addr_array.count(); ++i) {
const ObAddr &addr = all_addr_array.at(i);
if (ObTableLoadUtils::is_local_addr(addr)) { // 本机
if (OB_FAIL(ObTableLoadStore::abort_ctx(ctx))) {
LOG_WARN("fail to abort store ctx", KR(ret));
}
ObTableLoadStore::abort_ctx(ctx);
} else { // 远端, 发送rpc
const int64_t origin_timeout_ts = THIS_WORKER.get_timeout_ts();
THIS_WORKER.set_timeout_ts(INT64_MAX); // use default timeout value, avoid timeout now
@ -171,6 +161,18 @@ int ObTableLoadCoordinator::abort_peers_ctx(ObTableLoadTableCtx *ctx)
return ret;
}
int ObTableLoadCoordinator::abort_redef_table(ObTableLoadTableCtx *ctx, ObSQLSessionInfo &session_info)
{
int ret = OB_SUCCESS;
ObTableLoadRedefTableAbortArg arg;
arg.tenant_id_ = ctx->param_.tenant_id_;
arg.task_id_ = ctx->ddl_param_.task_id_;
if (OB_FAIL(ObTableLoadRedefTable::abort(arg, session_info))) {
LOG_WARN("fail to abort redef table", KR(ret), K(arg));
}
return ret;
}
int ObTableLoadCoordinator::init()
{
int ret = OB_SUCCESS;
@ -215,8 +217,6 @@ int ObTableLoadCoordinator::pre_begin_peers()
ObTableLoadPreBeginPeerResult result;
request.credential_ = coordinator_ctx_->credential_;
request.table_id_ = param_.table_id_;
request.target_table_id_ = param_.target_table_id_;
request.ddl_task_id_ = coordinator_ctx_->get_ddl_task_id();
request.config_.session_count_ = param_.session_count_;
request.config_.max_error_row_count_ = param_.max_error_row_count_;
request.config_.batch_size_ = param_.batch_size_;
@ -225,6 +225,9 @@ int ObTableLoadCoordinator::pre_begin_peers()
request.dup_action_ = param_.dup_action_;
request.px_mode_ = param_.px_mode_;
request.online_opt_stat_gather_ = param_.online_opt_stat_gather_;
request.dest_table_id_ = ctx_->ddl_param_.dest_table_id_;
request.task_id_ = ctx_->ddl_param_.task_id_;
request.schema_version_ = ctx_->ddl_param_.schema_version_;
for (int64_t i = 0; OB_SUCC(ret) && i < all_leader_info_array.count(); ++i) {
const ObTableLoadPartitionLocation::LeaderInfo &leader_info = all_leader_info_array.at(i);
const ObTableLoadPartitionLocation::LeaderInfo &target_leader_info = target_all_leader_info_array.at(i);
@ -237,7 +240,7 @@ int ObTableLoadCoordinator::pre_begin_peers()
request.partition_id_array_ = leader_info.partition_id_array_;
request.target_partition_id_array_ = target_leader_info.partition_id_array_;
if (ObTableLoadUtils::is_local_addr(addr)) { // 本机
if (OB_FAIL(ObTableLoadStore::init_ctx(ctx_, request.ddl_task_id_, request.partition_id_array_,
if (OB_FAIL(ObTableLoadStore::init_ctx(ctx_, request.partition_id_array_,
request.target_partition_id_array_))) {
LOG_WARN("fail to store init ctx", KR(ret));
} else {
@ -269,6 +272,7 @@ int ObTableLoadCoordinator::confirm_begin_peers()
ObTableLoadConfirmBeginPeerResult result;
request.credential_ = coordinator_ctx_->credential_;
request.table_id_ = param_.table_id_;
request.task_id_ = ctx_->ddl_param_.task_id_;
for (int64_t i = 0; OB_SUCC(ret) && i < all_addr_array.count(); ++i) {
const ObAddr &addr = all_addr_array.at(i);
if (ObTableLoadUtils::is_local_addr(addr)) { // 本机
@ -323,6 +327,7 @@ int ObTableLoadCoordinator::pre_merge_peers()
allocator.set_tenant_id(MTL_ID());
request.credential_ = coordinator_ctx_->credential_;
request.table_id_ = param_.table_id_;
request.task_id_ = ctx_->ddl_param_.task_id_;
if (!ctx_->param_.px_mode_) {
if (OB_FAIL(coordinator_ctx_->get_committed_trans_ids(request.committed_trans_id_array_,
allocator))) {
@ -360,6 +365,7 @@ int ObTableLoadCoordinator::start_merge_peers()
ObTableLoadStartMergePeerResult result;
request.credential_ = coordinator_ctx_->credential_;
request.table_id_ = param_.table_id_;
request.task_id_ = ctx_->ddl_param_.task_id_;
for (int64_t i = 0; OB_SUCC(ret) && i < all_addr_array.count(); ++i) {
const ObAddr &addr = all_addr_array.at(i);
if (ObTableLoadUtils::is_local_addr(addr)) { // 本机
@ -442,6 +448,7 @@ int ObTableLoadCoordinator::check_peers_merge_result(bool &is_finish)
ObTableLoadGetStatusPeerResult result;
request.credential_ = coordinator_ctx_->credential_;
request.table_id_ = param_.table_id_;
request.task_id_ = ctx_->ddl_param_.task_id_;
is_finish = true;
for (int64_t i = 0; OB_SUCC(ret) && i < all_addr_array.count(); ++i) {
const ObAddr &addr = all_addr_array.at(i);
@ -590,6 +597,7 @@ int ObTableLoadCoordinator::commit_peers(ObTableLoadSqlStatistics &sql_statistic
ObTableLoadCommitPeerResult result;
request.credential_ = coordinator_ctx_->credential_;
request.table_id_ = param_.table_id_;
request.task_id_ = ctx_->ddl_param_.task_id_;
for (int64_t i = 0; OB_SUCC(ret) && i < all_addr_array.count(); ++i) {
const ObAddr &addr = all_addr_array.at(i);
if (ObTableLoadUtils::is_local_addr(addr)) { // 本机
@ -616,9 +624,25 @@ int ObTableLoadCoordinator::commit_peers(ObTableLoadSqlStatistics &sql_statistic
return ret;
}
int ObTableLoadCoordinator::commit_redef_table(ObSQLSessionInfo &session_info)
{
int ret = OB_SUCCESS;
ObTableLoadRedefTableFinishArg arg;
arg.tenant_id_ = param_.tenant_id_;
arg.table_id_ = param_.table_id_;
arg.dest_table_id_ = ctx_->ddl_param_.dest_table_id_;
arg.task_id_ = ctx_->ddl_param_.task_id_;
arg.schema_version_ = ctx_->ddl_param_.schema_version_;
if (OB_FAIL(ObTableLoadRedefTable::finish(arg, session_info))) {
LOG_WARN("fail to finish redef table", KR(ret), K(arg));
}
return ret;
}
// commit() = px_commit_data() + px_commit_ddl()
// used in non px_mode
int ObTableLoadCoordinator::commit(ObExecContext &ctx, ObTableLoadResultInfo &result_info)
int ObTableLoadCoordinator::commit(ObExecContext *exec_ctx, ObSQLSessionInfo &session_info,
ObTableLoadResultInfo &result_info)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
@ -631,10 +655,11 @@ int ObTableLoadCoordinator::commit(ObExecContext &ctx, ObTableLoadResultInfo &re
LOG_WARN("fail to check coordinator status", KR(ret));
} else if (OB_FAIL(commit_peers(sql_statistics))) {
LOG_WARN("fail to commit peers", KR(ret));
} else if (param_.online_opt_stat_gather_ && OB_FAIL(drive_sql_stat(ctx, sql_statistics))) {
} else if (param_.online_opt_stat_gather_ &&
OB_FAIL(drive_sql_stat(*exec_ctx, sql_statistics))) {
LOG_WARN("fail to drive sql stat", KR(ret));
} else if (OB_FAIL(coordinator_ctx_->commit())) {
LOG_WARN("fail to commit coordinator", KR(ret));
} else if (OB_FAIL(commit_redef_table(session_info))) {
LOG_WARN("fail to commit redef table", KR(ret));
} else if (OB_FAIL(coordinator_ctx_->set_status_commit())) {
LOG_WARN("fail to set coordinator status commit", KR(ret));
} else {
@ -646,7 +671,7 @@ int ObTableLoadCoordinator::commit(ObExecContext &ctx, ObTableLoadResultInfo &re
// used in insert /*+ append */ into select clause
// commit data loaded
int ObTableLoadCoordinator::px_commit_data(ObExecContext &ctx)
int ObTableLoadCoordinator::px_commit_data(ObExecContext *exec_ctx)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
@ -659,7 +684,8 @@ int ObTableLoadCoordinator::px_commit_data(ObExecContext &ctx)
LOG_WARN("fail to check coordinator status", KR(ret));
} else if (OB_FAIL(commit_peers(sql_statistics))) {
LOG_WARN("fail to commit peers", KR(ret));
} else if (param_.online_opt_stat_gather_ && OB_FAIL(drive_sql_stat(ctx, sql_statistics))) {
} else if (param_.online_opt_stat_gather_ &&
OB_FAIL(drive_sql_stat(*exec_ctx, sql_statistics))) {
LOG_WARN("fail to drive sql stat", KR(ret));
}
}
@ -667,7 +693,7 @@ int ObTableLoadCoordinator::px_commit_data(ObExecContext &ctx)
}
// commit ddl procedure
int ObTableLoadCoordinator::px_commit_ddl()
int ObTableLoadCoordinator::px_commit_ddl(ObSQLSessionInfo &session_info)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
@ -675,8 +701,8 @@ int ObTableLoadCoordinator::px_commit_ddl()
LOG_WARN("ObTableLoadCoordinator not init", KR(ret), KP(this));
} else {
LOG_INFO("coordinator px_commit_ddl");
if (OB_FAIL(coordinator_ctx_->commit())) {
LOG_WARN("fail to commit coordinator", KR(ret));
if (OB_FAIL(commit_redef_table(session_info))) {
LOG_WARN("fail to commit redef table", KR(ret));
} else if (OB_FAIL(coordinator_ctx_->set_status_commit())) {
LOG_WARN("fail to set coordinator status commit", KR(ret));
}
@ -687,6 +713,7 @@ int ObTableLoadCoordinator::px_commit_ddl()
int ObTableLoadCoordinator::drive_sql_stat(ObExecContext &ctx, ObTableLoadSqlStatistics &sql_statistics) {
int ret = OB_SUCCESS;
const uint64_t tenant_id = MTL_ID();
const uint64_t table_id = ctx_->ddl_param_.dest_table_id_;
ObSchemaGetterGuard schema_guard;
ObSchemaGetterGuard *tmp_schema_guard = nullptr;
ObSchemaGetterGuard *tmp_schema_guard2 = nullptr;
@ -694,12 +721,9 @@ int ObTableLoadCoordinator::drive_sql_stat(ObExecContext &ctx, ObTableLoadSqlSta
if (sql_statistics.is_empty()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sql statistics is empty", K(ret));
} else if (OB_FAIL(ObTableLoadSchema::get_table_schema(tenant_id, param_.target_table_id_, schema_guard,
table_schema))) {
LOG_WARN("fail to get table schema", KR(ret), K(param_.target_table_id_), K(tenant_id));
} else if (OB_ISNULL(table_schema)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected error", K(ret));
} else if (OB_FAIL(ObTableLoadSchema::get_table_schema(tenant_id, table_id, schema_guard,
table_schema))) {
LOG_WARN("fail to get table schema", KR(ret), K(tenant_id), K(table_id));
} else {
tmp_schema_guard = ctx.get_virtual_table_ctx().schema_guard_;
tmp_schema_guard2 = ctx.get_das_ctx().get_schema_guard();
@ -752,6 +776,7 @@ int ObTableLoadCoordinator::pre_start_trans_peers(ObTableLoadCoordinatorTrans *t
ObTableLoadPreStartTransPeerResult result;
request.credential_ = coordinator_ctx_->credential_;
request.table_id_ = param_.table_id_;
request.task_id_ = ctx_->ddl_param_.task_id_;
request.trans_id_ = trans_id;
for (int64_t i = 0; OB_SUCC(ret) && i < all_addr_array.count(); ++i) {
const ObAddr &addr = all_addr_array.at(i);
@ -783,6 +808,7 @@ int ObTableLoadCoordinator::confirm_start_trans_peers(ObTableLoadCoordinatorTran
ObTableLoadConfirmStartTransPeerResult result;
request.credential_ = coordinator_ctx_->credential_;
request.table_id_ = param_.table_id_;
request.task_id_ = ctx_->ddl_param_.task_id_;
request.trans_id_ = trans_id;
for (int64_t i = 0; OB_SUCC(ret) && i < all_addr_array.count(); ++i) {
const ObAddr &addr = all_addr_array.at(i);
@ -867,6 +893,7 @@ int ObTableLoadCoordinator::pre_finish_trans_peers(ObTableLoadCoordinatorTrans *
ObTableLoadPreFinishTransPeerResult result;
request.credential_ = coordinator_ctx_->credential_;
request.table_id_ = param_.table_id_;
request.task_id_ = ctx_->ddl_param_.task_id_;
request.trans_id_ = trans_id;
for (int64_t i = 0; OB_SUCC(ret) && i < all_addr_array.count(); ++i) {
const ObAddr &addr = all_addr_array.at(i);
@ -898,6 +925,7 @@ int ObTableLoadCoordinator::confirm_finish_trans_peers(ObTableLoadCoordinatorTra
ObTableLoadConfirmFinishTransPeerResult result;
request.credential_ = coordinator_ctx_->credential_;
request.table_id_ = param_.table_id_;
request.task_id_ = ctx_->ddl_param_.task_id_;
request.trans_id_ = trans_id;
for (int64_t i = 0; OB_SUCC(ret) && i < all_addr_array.count(); ++i) {
const ObAddr &addr = all_addr_array.at(i);
@ -956,6 +984,7 @@ int ObTableLoadCoordinator::check_peers_trans_commit(ObTableLoadCoordinatorTrans
ObTableLoadGetTransStatusPeerResult result;
request.credential_ = coordinator_ctx_->credential_;
request.table_id_ = param_.table_id_;
request.task_id_ = ctx_->ddl_param_.task_id_;
request.trans_id_ = trans->get_trans_id();
is_commit = true;
for (int64_t i = 0; OB_SUCC(ret) && i < all_addr_array.count(); ++i) {
@ -1145,6 +1174,7 @@ int ObTableLoadCoordinator::abandon_trans_peers(ObTableLoadCoordinatorTrans *tra
ObTableLoadAbandonTransPeerResult result;
request.credential_ = coordinator_ctx_->credential_;
request.table_id_ = param_.table_id_;
request.task_id_ = ctx_->ddl_param_.task_id_;
request.trans_id_ = trans_id;
for (int64_t i = 0; OB_SUCC(ret) && i < all_addr_array.count(); ++i) {
const ObAddr &addr = all_addr_array.at(i);
@ -1544,6 +1574,7 @@ int ObTableLoadCoordinator::write_peer_leader(const ObTableLoadTransId &trans_id
ObTableLoadPeerResult result;
request.credential_ = coordinator_ctx_->credential_;
request.table_id_ = param_.table_id_;
request.task_id_ = ctx_->ddl_param_.task_id_;
request.trans_id_ = trans_id;
request.session_id_ = session_id;
request.sequence_no_ = sequence_no;

15
src/observer/table_load/ob_table_load_coordinator.h
View File

@ -26,22 +26,23 @@ public:
ObTableLoadCoordinator(ObTableLoadTableCtx *ctx);
static bool is_ctx_inited(ObTableLoadTableCtx *ctx);
static int init_ctx(ObTableLoadTableCtx *ctx, const common::ObIArray<int64_t> &idx_array,
sql::ObSQLSessionInfo *session_info);
static int abort_ctx(ObTableLoadTableCtx *ctx);
uint64_t user_id);
static void abort_ctx(ObTableLoadTableCtx *ctx, sql::ObSQLSessionInfo &session_info);
int init();
bool is_valid() const { return is_inited_; }
private:
int drive_sql_stat(sql::ObExecContext &ctx, table::ObTableLoadSqlStatistics &sql_statistics);
static int abort_active_trans(ObTableLoadTableCtx *ctx);
static int abort_peers_ctx(ObTableLoadTableCtx *ctx);
static int abort_redef_table(ObTableLoadTableCtx *ctx, sql::ObSQLSessionInfo &session_info);
// table load ctrl interface
public:
int begin();
int finish();
int commit(sql::ObExecContext &ctx, table::ObTableLoadResultInfo &result_info);
int px_commit_data(sql::ObExecContext &ctx);
int px_commit_ddl();
int commit(sql::ObExecContext *exec_ctx, sql::ObSQLSessionInfo &session_info,
table::ObTableLoadResultInfo &result_info);
int px_commit_data(sql::ObExecContext *exec_ctx);
int px_commit_ddl(sql::ObSQLSessionInfo &session_info);
int get_status(table::ObTableLoadStatusType &status, int &error_code);
private:
int pre_begin_peers();
@ -49,6 +50,8 @@ private:
int pre_merge_peers();
int start_merge_peers();
int commit_peers(table::ObTableLoadSqlStatistics &sql_statistics);
int commit_redef_table(sql::ObSQLSessionInfo &session_info);
int drive_sql_stat(sql::ObExecContext &ctx, table::ObTableLoadSqlStatistics &sql_statistics);
private:
int add_check_merge_result_task();
int check_peers_merge_result(bool &is_finish);

58
src/observer/table_load/ob_table_load_coordinator_ctx.cpp
View File

@ -22,13 +22,12 @@ using namespace obrpc;
ObTableLoadCoordinatorCtx::ObTableLoadCoordinatorCtx(ObTableLoadTableCtx *ctx)
: ctx_(ctx),
allocator_("TLD_CoordCtx", OB_MALLOC_NORMAL_BLOCK_SIZE, ctx->param_.tenant_id_),
allocator_("TLD_CoordCtx"),
task_scheduler_(nullptr),
last_trans_gid_(1024),
next_session_id_(0),
status_(ObTableLoadStatusType::NONE),
error_code_(OB_SUCCESS),
redef_table_(),
is_inited_(false)
{
}
@ -38,26 +37,22 @@ ObTableLoadCoordinatorCtx::~ObTableLoadCoordinatorCtx()
destroy();
}
int ObTableLoadCoordinatorCtx::init(ObSQLSessionInfo *session_info,
const ObIArray<int64_t> &idx_array)
int ObTableLoadCoordinatorCtx::init(const ObIArray<int64_t> &idx_array, uint64_t user_id)
{
int ret = OB_SUCCESS;
if (IS_INIT) {
ret = OB_INIT_TWICE;
LOG_WARN("ObTableLoadCoordinatorCtx init twice", KR(ret), KP(this));
} else if (idx_array.count() != ctx_->param_.column_count_) {
} else if (OB_UNLIKELY(idx_array.count() != ctx_->param_.column_count_ ||
OB_INVALID_ID == user_id)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid idx_array", KR(ret), K(idx_array.count()), K_(ctx_->param_.column_count));
LOG_WARN("invalid args", KR(ret), K(idx_array.count()), K_(ctx_->param_.column_count),
K(user_id));
} else {
// init redef table
if (OB_FAIL(redef_table_.init(ctx_, session_info))) {
LOG_WARN("failed to init ddl processor", KR(ret));
} else if (OB_FAIL(redef_table_.start())) {
LOG_WARN("failed to create hidden table", KR(ret));
} else if (OB_FAIL(target_schema_.init(ctx_->param_.tenant_id_,
ctx_->param_.target_table_id_))) {
allocator_.set_tenant_id(MTL_ID());
if (OB_FAIL(target_schema_.init(ctx_->param_.tenant_id_, ctx_->ddl_param_.dest_table_id_))) {
LOG_WARN("fail to init table load schema", KR(ret), K(ctx_->param_.tenant_id_),
K(ctx_->param_.target_table_id_));
K(ctx_->ddl_param_.dest_table_id_));
}
// init idx array
else if (OB_FAIL(idx_array_.assign(idx_array))) {
@ -96,8 +91,8 @@ int ObTableLoadCoordinatorCtx::init(ObSQLSessionInfo *session_info,
LOG_WARN("fail to init segment ctx map", KR(ret));
}
// generate credential_
else if (OB_FAIL(generate_credential(session_info->get_priv_user_id()))) {
LOG_WARN("fail to generate credential", KR(ret));
else if (OB_FAIL(generate_credential(user_id))) {
LOG_WARN("fail to generate credential", KR(ret), K(user_id));
}
// init task_scheduler_
else if (OB_ISNULL(task_scheduler_ = OB_NEWx(ObTableLoadTaskThreadPoolScheduler, (&allocator_),
@ -150,7 +145,6 @@ void ObTableLoadCoordinatorCtx::destroy()
trans_ctx_map_.reuse();
segment_ctx_map_.reset();
commited_trans_ctx_array_.reset();
redef_table_.reset();
}
int ObTableLoadCoordinatorCtx::generate_credential(uint64_t user_id)
@ -615,35 +609,5 @@ int ObTableLoadCoordinatorCtx::check_exist_committed_trans(bool &is_exist) const
return ret;
}
int ObTableLoadCoordinatorCtx::commit()
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadCoordinatorCtx not init", KR(ret));
} else if (OB_FAIL(check_status(ObTableLoadStatusType::MERGED))) {
LOG_WARN("fail to check status", KR(ret));
} else if (OB_FAIL(redef_table_.finish())){
LOG_WARN("failed to finish redef table", KR(ret));
}
return ret;
}
int ObTableLoadCoordinatorCtx::abort()
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadCoordinatorCtx not init", KR(ret));
} else if (OB_FAIL(redef_table_.abort())){
LOG_WARN("failed to abort redef table", KR(ret));
}
return ret;
}
int64_t ObTableLoadCoordinatorCtx::get_ddl_task_id() const
{
return redef_table_.get_ddl_task_id();
}
} // namespace observer
} // namespace oceanbase

7
src/observer/table_load/ob_table_load_coordinator_ctx.h
View File

@ -11,7 +11,6 @@
#include "observer/table_load/ob_table_load_partition_calc.h"
#include "observer/table_load/ob_table_load_partition_location.h"
#include "observer/table_load/ob_table_load_schema.h"
#include "observer/table_load/ob_table_load_redef_table.h"
#include "share/table/ob_table_load_array.h"
#include "share/table/ob_table_load_define.h"
@ -29,7 +28,7 @@ class ObTableLoadCoordinatorCtx
public:
ObTableLoadCoordinatorCtx(ObTableLoadTableCtx *ctx);
~ObTableLoadCoordinatorCtx();
int init(sql::ObSQLSessionInfo *session_info, const common::ObIArray<int64_t> &idx_array);
int init(const common::ObIArray<int64_t> &idx_array, uint64_t user_id);
void stop();
void destroy();
bool is_valid() const { return is_inited_; }
@ -91,9 +90,6 @@ public:
common::ObIAllocator &allocator) const;
int check_exist_trans(bool &is_exist) const;
int check_exist_committed_trans(bool &is_exist) const;
int commit();
int abort();
int64_t get_ddl_task_id() const;
private:
int generate_credential(uint64_t user_id);
int alloc_trans_ctx(const table::ObTableLoadTransId &trans_id, ObTableLoadTransCtx *&trans_ctx);
@ -140,7 +136,6 @@ private:
TransCtxMap trans_ctx_map_;
SegmentCtxMap segment_ctx_map_;
common::ObSEArray<ObTableLoadTransCtx *, 64> commited_trans_ctx_array_;
ObTableLoadRedefTable redef_table_;
bool is_inited_;
};

6
src/observer/table_load/ob_table_load_finish_processor.cpp
View File

@ -26,7 +26,7 @@ int ObTableLoadFinishP::process()
LOG_WARN("fail to check_user_access", KR(ret));
} else {
ObTableLoadTableCtx *table_ctx = nullptr;
ObTableLoadKey key(credential_.tenant_id_, arg_.table_id_);
ObTableLoadUniqueKey key(arg_.table_id_, arg_.task_id_);
if (OB_FAIL(ObTableLoadService::get_ctx(key, table_ctx))) {
LOG_WARN("fail to get table ctx", KR(ret), K(key));
} else {
@ -67,7 +67,7 @@ int ObTableLoadPreMergePeerP::process()
LOG_WARN("fail to check_user_access", KR(ret));
} else {
ObTableLoadTableCtx *table_ctx = nullptr;
ObTableLoadKey key(credential_.tenant_id_, arg_.table_id_);
ObTableLoadUniqueKey key(arg_.table_id_, arg_.task_id_);
if (OB_FAIL(ObTableLoadService::get_ctx(key, table_ctx))) {
LOG_WARN("fail to get table ctx", KR(ret), K(key));
} else {
@ -102,7 +102,7 @@ int ObTableLoadStartMergePeerP::process()
LOG_WARN("fail to check_user_access", KR(ret));
} else {
ObTableLoadTableCtx *table_ctx = nullptr;
ObTableLoadKey key(credential_.tenant_id_, arg_.table_id_);
ObTableLoadUniqueKey key(arg_.table_id_, arg_.task_id_);
if (OB_FAIL(ObTableLoadService::get_ctx(key, table_ctx))) {
LOG_WARN("fail to get table ctx", KR(ret), K(key));
} else {

4
src/observer/table_load/ob_table_load_get_status_processor.cpp
View File

@ -26,7 +26,7 @@ int ObTableLoadGetStatusP::process()
LOG_WARN("fail to check_user_access", KR(ret));
} else {
ObTableLoadTableCtx *table_ctx = nullptr;
ObTableLoadKey key(credential_.tenant_id_, arg_.table_id_);
ObTableLoadUniqueKey key(arg_.table_id_, arg_.task_id_);
if (OB_FAIL(ObTableLoadService::get_ctx(key, table_ctx))) {
LOG_WARN("fail to get table ctx", KR(ret), K(key));
} else {
@ -61,7 +61,7 @@ int ObTableLoadGetStatusPeerP::process()
LOG_WARN("fail to check_user_access", KR(ret));
} else {
ObTableLoadTableCtx *table_ctx = nullptr;
ObTableLoadKey key(credential_.tenant_id_, arg_.table_id_);
ObTableLoadUniqueKey key(arg_.table_id_, arg_.task_id_);
if (OB_FAIL(ObTableLoadService::get_ctx(key, table_ctx))) {
LOG_WARN("fail to get table ctx", KR(ret), K(key));
} else {

266
src/observer/table_load/ob_table_load_instance.cpp
View File

@ -5,40 +5,51 @@
#define USING_LOG_PREFIX SERVER
#include "observer/table_load/ob_table_load_instance.h"
#include "share/table/ob_table_load_define.h"
#include "observer/table_load/ob_table_load_exec_ctx.h"
#include "observer/table_load/ob_table_load_coordinator.h"
#include "observer/table_load/ob_table_load_exec_ctx.h"
#include "observer/table_load/ob_table_load_redef_table.h"
#include "observer/table_load/ob_table_load_service.h"
#include "observer/table_load/ob_table_load_table_ctx.h"
#include "share/table/ob_table_load_define.h"
#include "sql/engine/ob_exec_context.h"
namespace oceanbase
{
using namespace table;
using namespace storage;
namespace observer
{
using namespace sql;
using namespace storage;
using namespace table;
ObTableLoadInstance::ObTableLoadInstance()
: allocator_(nullptr),
execute_ctx_(nullptr),
: execute_ctx_(nullptr),
allocator_(nullptr),
session_info_(nullptr),
table_ctx_(nullptr),
coordinator_(nullptr),
job_stat_(nullptr),
px_mode_(false),
online_opt_stat_gather_(false),
sql_mode_(0),
is_committed_(false),
is_inited_(false)
{
}
ObTableLoadInstance::~ObTableLoadInstance()
ObTableLoadInstance::~ObTableLoadInstance() { destroy(); }
void ObTableLoadInstance::destroy()
{
destroy();
trans_ctx_.reset();
if (nullptr != table_ctx_) {
ObTableLoadService::remove_ctx(table_ctx_);
if (!is_committed_) {
ObTableLoadCoordinator::abort_ctx(table_ctx_, *session_info_);
}
ObTableLoadService::put_ctx(table_ctx_);
table_ctx_ = nullptr;
job_stat_ = nullptr;
}
}
int ObTableLoadInstance::init(ObTableLoadParam &param,
const oceanbase::common::ObIArray<int64_t> &idx_array,
ObTableLoadExecCtx *execute_ctx)
int ObTableLoadInstance::init(ObTableLoadParam &param, const ObIArray<int64_t> &idx_array,
ObTableLoadExecCtx *execute_ctx)
{
int ret = OB_SUCCESS;
if (IS_INIT) {
@ -48,48 +59,30 @@ int ObTableLoadInstance::init(ObTableLoadParam &param,
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(param), K(execute_ctx));
} else {
bool is_new = false;
allocator_ = execute_ctx->allocator_;
execute_ctx_ = execute_ctx;
px_mode_ = param.px_mode_;
online_opt_stat_gather_ = param.online_opt_stat_gather_;
sql_mode_ = param.sql_mode_;
ObTableLoadSegmentID segment_id(DEFAULT_SEGMENT_ID);
allocator_ = execute_ctx->allocator_;
session_info_ = execute_ctx_->exec_ctx_->get_my_session();
if (OB_FAIL(param.normalize())) {
LOG_WARN("fail to normalize param", KR(ret));
} else if (OB_FAIL(next_sequence_no_array_.create(param.session_count_, *allocator_))) {
LOG_WARN("fail to create next sequence no array", KR(ret));
}
// check support
else if (OB_FAIL(ObTableLoadService::check_support_direct_load(param.table_id_))) {
LOG_WARN("fail to check support direct load", KR(ret), K(param.table_id_));
}
// create table ctx
else if (OB_FAIL(ObTableLoadService::create_ctx(param, table_ctx_, is_new))) {
LOG_WARN("fail to create table load ctx", KR(ret), K(param));
} else if (OB_UNLIKELY(!is_new)) {
ret = OB_ENTRY_EXIST;
LOG_WARN("table load ctx exists", KR(ret));
} else if (OB_FAIL(
ObTableLoadCoordinator::init_ctx(table_ctx_, idx_array, execute_ctx_->exec_ctx_->get_my_session()))) {
LOG_WARN("fail to coordinator init ctx", KR(ret));
}
// new coordinator
else if (OB_ISNULL(coordinator_ = OB_NEWx(ObTableLoadCoordinator, allocator_, table_ctx_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObTableLoadCoordinator", KR(ret));
} else if (OB_FAIL(coordinator_->init())) {
LOG_WARN("fail to init coordinator", KR(ret));
else if (OB_FAIL(create_table_ctx(param, idx_array))) {
LOG_WARN("fail to create table ctx", KR(ret));
}
// begin
else if (OB_FAIL(coordinator_->begin())) {
LOG_WARN("fail to coordinator begin", KR(ret));
else if (OB_FAIL(begin())) {
LOG_WARN("fail to begin", KR(ret));
}
// start trans
else if (!px_mode_ && OB_FAIL(coordinator_->start_trans(segment_id, trans_id_))) {
LOG_WARN("fail to coordinator start trans", KR(ret));
else if (!param.px_mode_ && OB_FAIL(start_trans())) {
LOG_WARN("fail to start trans", KR(ret));
}
// init succ
else {
for (int64_t i = 0; i < param.session_count_; ++i) {
next_sequence_no_array_[i] = 1;
}
job_stat_ = table_ctx_->job_stat_;
is_inited_ = true;
}
@ -97,22 +90,111 @@ int ObTableLoadInstance::init(ObTableLoadParam &param,
return ret;
}
void ObTableLoadInstance::destroy()
int ObTableLoadInstance::create_table_ctx(ObTableLoadParam &param,
const ObIArray<int64_t> &idx_array)
{
if (nullptr != coordinator_) {
if (coordinator_->is_valid()) {
ObTableLoadCoordinator::abort_ctx(table_ctx_);
int ret = OB_SUCCESS;
ObTableLoadTableCtx *table_ctx = nullptr;
ObTableLoadDDLParam ddl_param;
// start redef table
ObTableLoadRedefTableStartArg start_arg;
ObTableLoadRedefTableStartRes start_res;
start_arg.tenant_id_ = param.tenant_id_;
start_arg.table_id_ = param.table_id_;
start_arg.parallelism_ = param.session_count_;
if (OB_FAIL(ObTableLoadRedefTable::start(start_arg, start_res, *session_info_))) {
LOG_WARN("fail to start redef table", KR(ret), K(start_arg));
} else {
ddl_param.dest_table_id_ = start_res.dest_table_id_;
ddl_param.task_id_ = start_res.task_id_;
ddl_param.schema_version_ = start_res.schema_version_;
}
if (OB_SUCC(ret)) {
if (OB_ISNULL(table_ctx = ObTableLoadService::alloc_ctx())) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to alloc table ctx", KR(ret), K(param));
} else if (OB_FAIL(table_ctx->init(param, ddl_param))) {
LOG_WARN("fail to init table ctx", KR(ret));
} else if (OB_FAIL(ObTableLoadCoordinator::init_ctx(table_ctx, idx_array,
session_info_->get_priv_user_id()))) {
LOG_WARN("fail to coordinator init ctx", KR(ret));
} else if (OB_FAIL(ObTableLoadService::add_ctx(table_ctx))) {
LOG_WARN("fail to add ctx", KR(ret));
} else {
table_ctx_ = table_ctx;
}
coordinator_->~ObTableLoadCoordinator();
allocator_->free(coordinator_);
coordinator_ = nullptr;
}
if (nullptr != table_ctx_) {
ObTableLoadService::remove_ctx(table_ctx_);
ObTableLoadService::put_ctx(table_ctx_);
table_ctx_ = nullptr;
job_stat_ = nullptr;
if (OB_FAIL(ret)) {
int tmp_ret = OB_SUCCESS;
if (ddl_param.is_valid()) {
ObTableLoadRedefTableAbortArg abort_arg;
abort_arg.tenant_id_ = param.tenant_id_;
abort_arg.task_id_ = ddl_param.task_id_;
if (OB_TMP_FAIL(ObTableLoadRedefTable::abort(abort_arg, *session_info_))) {
LOG_WARN("fail to abort redef table", KR(tmp_ret), K(abort_arg));
}
}
if (nullptr != table_ctx) {
ObTableLoadService::free_ctx(table_ctx);
table_ctx = nullptr;
}
}
return ret;
}
int ObTableLoadInstance::begin()
{
int ret = OB_SUCCESS;
ObTableLoadCoordinator coordinator(table_ctx_);
if (OB_FAIL(coordinator.init())) {
LOG_WARN("fail to init coordinator", KR(ret));
} else if (OB_FAIL(coordinator.begin())) {
LOG_WARN("fail to coodrinator begin", KR(ret));
}
return ret;
}
int ObTableLoadInstance::start_trans()
{
int ret = OB_SUCCESS;
ObTableLoadSegmentID segment_id(DEFAULT_SEGMENT_ID);
ObTableLoadCoordinator coordinator(table_ctx_);
if (OB_FAIL(coordinator.init())) {
LOG_WARN("fail to init coordinator", KR(ret));
} else if (OB_FAIL(coordinator.start_trans(segment_id, trans_ctx_.trans_id_))) {
LOG_WARN("fail to coordinator start trans", KR(ret));
} else if (OB_FAIL(trans_ctx_.next_sequence_no_array_.create(table_ctx_->param_.session_count_,
*allocator_))) {
LOG_WARN("fail to create next sequence no array", KR(ret));
} else {
for (int64_t i = 0; i < table_ctx_->param_.session_count_; ++i) {
trans_ctx_.next_sequence_no_array_[i] = 1;
}
}
return ret;
}
int ObTableLoadInstance::write(int32_t session_id, const table::ObTableLoadObjRowArray &obj_rows)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadInstance not init", KR(ret), KP(this));
} else if (OB_UNLIKELY(session_id < 0 || session_id > table_ctx_->param_.session_count_ ||
obj_rows.empty())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(session_id), K(obj_rows.count()));
} else {
uint64_t &next_sequence_no = trans_ctx_.next_sequence_no_array_[session_id - 1];
ObTableLoadCoordinator coordinator(table_ctx_);
if (OB_FAIL(coordinator.init())) {
LOG_WARN("fail to init coordinator", KR(ret));
} else if (OB_FAIL(coordinator.write(trans_ctx_.trans_id_, session_id, next_sequence_no++,
obj_rows))) {
LOG_WARN("fail to write coordinator", KR(ret));
}
}
return ret;
}
int ObTableLoadInstance::check_trans_committed()
@ -120,11 +202,15 @@ int ObTableLoadInstance::check_trans_committed()
int ret = OB_SUCCESS;
ObTableLoadTransStatusType trans_status = ObTableLoadTransStatusType::NONE;
int error_code = OB_SUCCESS;
ObTableLoadCoordinator coordinator(table_ctx_);
if (OB_FAIL(coordinator.init())) {
LOG_WARN("fail to init coordinator", KR(ret));
}
while (OB_SUCC(ret) && ObTableLoadTransStatusType::COMMIT != trans_status &&
OB_SUCC(execute_ctx_->check_status())) {
if (OB_FAIL(coordinator_->get_trans_status(trans_id_, trans_status, error_code))) {
if (OB_FAIL(coordinator.get_trans_status(trans_ctx_.trans_id_, trans_status, error_code))) {
LOG_WARN("fail to coordinator get trans status", KR(ret));
} else{
} else {
switch (trans_status) {
case ObTableLoadTransStatusType::FROZEN:
usleep(WAIT_INTERVAL_US);
@ -150,9 +236,13 @@ int ObTableLoadInstance::check_merged()
int ret = OB_SUCCESS;
ObTableLoadStatusType status = ObTableLoadStatusType::NONE;
int error_code = OB_SUCCESS;
ObTableLoadCoordinator coordinator(table_ctx_);
if (OB_FAIL(coordinator.init())) {
LOG_WARN("fail to init coordinator", KR(ret));
}
while (OB_SUCC(ret) && ObTableLoadStatusType::MERGED != status &&
OB_SUCC(execute_ctx_->check_status())) {
if (OB_FAIL(coordinator_->get_status(status, error_code))) {
if (OB_FAIL(coordinator.get_status(status, error_code))) {
LOG_WARN("fail to coordinator get status", KR(ret));
} else {
switch (status) {
@ -185,8 +275,12 @@ int ObTableLoadInstance::commit(ObTableLoadResultInfo &result_info)
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadInstance not init", KR(ret), KP(this));
} else {
ObTableLoadCoordinator coordinator(table_ctx_);
if (OB_FAIL(coordinator.init())) {
LOG_WARN("fail to init coordinator", KR(ret));
}
// finish trans
if (OB_FAIL(coordinator_->finish_trans(trans_id_))) {
else if (OB_FAIL(coordinator.finish_trans(trans_ctx_.trans_id_))) {
LOG_WARN("fail to finish trans", KR(ret));
}
// wait trans commit
@ -194,7 +288,7 @@ int ObTableLoadInstance::commit(ObTableLoadResultInfo &result_info)
LOG_WARN("fail to check trans committed", KR(ret));
}
// finish
else if (OB_FAIL(coordinator_->finish())) {
else if (OB_FAIL(coordinator.finish())) {
LOG_WARN("fail to finish", KR(ret));
}
// wait merge
@ -202,14 +296,10 @@ int ObTableLoadInstance::commit(ObTableLoadResultInfo &result_info)
LOG_WARN("fail to check merged", KR(ret));
}
// commit
else if (OB_FAIL(coordinator_->commit(*execute_ctx_->exec_ctx_, result_info))) {
else if (OB_FAIL(coordinator.commit(execute_ctx_->exec_ctx_, *session_info_, result_info))) {
LOG_WARN("fail to commit", KR(ret));
}
else {
// Setting coordinator_ to NULL to mark a normal termination
coordinator_->~ObTableLoadCoordinator();
allocator_->free(coordinator_);
coordinator_ = nullptr;
} else {
is_committed_ = true;
}
}
return ret;
@ -223,8 +313,12 @@ int ObTableLoadInstance::px_commit_data()
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadInstance not init", KR(ret), KP(this));
} else {
ObTableLoadCoordinator coordinator(table_ctx_);
if (OB_FAIL(coordinator.init())) {
LOG_WARN("fail to init coordinator", KR(ret));
}
// finish
if (OB_FAIL(coordinator_->finish())) {
else if (OB_FAIL(coordinator.finish())) {
LOG_WARN("fail to finish", KR(ret));
}
// wait merge
@ -232,7 +326,7 @@ int ObTableLoadInstance::px_commit_data()
LOG_WARN("fail to check merged", KR(ret));
}
// commit
else if (OB_FAIL(coordinator_->px_commit_data(*execute_ctx_->exec_ctx_))) {
else if (OB_FAIL(coordinator.px_commit_data(execute_ctx_->exec_ctx_))) {
LOG_WARN("fail to do px_commit_data", KR(ret));
}
}
@ -246,33 +340,11 @@ int ObTableLoadInstance::px_commit_ddl()
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadInstance not init", KR(ret), KP(this));
} else {
if (OB_FAIL(coordinator_->px_commit_ddl())) {
ObTableLoadCoordinator coordinator(table_ctx_);
if (OB_FAIL(coordinator.init())) {
LOG_WARN("fail to init coordinator", KR(ret));
} else if (OB_FAIL(coordinator.px_commit_ddl(*session_info_))) {
LOG_WARN("fail to do px_commit_ddl", KR(ret));
} else {
// Setting coordinator_ to NULL to mark a normal termination
coordinator_->~ObTableLoadCoordinator();
allocator_->free(coordinator_);
coordinator_ = nullptr;
}
}
return ret;
}
int ObTableLoadInstance::write(int32_t session_id,
const table::ObTableLoadObjRowArray &obj_rows)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadInstance not init", KR(ret), KP(this));
} else if (OB_UNLIKELY(session_id < 0 || session_id > table_ctx_->param_.session_count_ ||
obj_rows.empty())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(session_id), K(obj_rows.count()));
} else {
uint64_t &next_sequence_no = next_sequence_no_array_[session_id - 1];
if (OB_FAIL(coordinator_->write(trans_id_, session_id, next_sequence_no++, obj_rows))) {
LOG_WARN("fail to write coordinator", KR(ret));
}
}
return ret;

35
src/observer/table_load/ob_table_load_instance.h
View File

@ -14,7 +14,6 @@ namespace observer
{
class ObTableLoadParam;
class ObTableLoadTableCtx;
class ObTableLoadCoordinator;
class ObTableLoadExecCtx;
class ObTableLoadInstance
@ -23,14 +22,13 @@ class ObTableLoadInstance
public:
ObTableLoadInstance();
~ObTableLoadInstance();
int init(observer::ObTableLoadParam &param,
const oceanbase::common::ObIArray<int64_t> &idx_array,
observer::ObTableLoadExecCtx *execute_ctx);
void destroy();
int init(ObTableLoadParam &param, const common::ObIArray<int64_t> &idx_array,
ObTableLoadExecCtx *execute_ctx);
int write(int32_t session_id, const table::ObTableLoadObjRowArray &obj_rows);
int commit(table::ObTableLoadResultInfo &result_info);
int px_commit_data();
int px_commit_ddl();
int write(int32_t session_id, const table::ObTableLoadObjRowArray &obj_rows);
sql::ObLoadDataStat *get_job_stat() const { return job_stat_; }
void update_job_stat_parsed_rows(int64_t parsed_rows)
{
@ -41,20 +39,33 @@ public:
ATOMIC_AAF(&job_stat_->parsed_bytes_, parsed_bytes);
}
private:
int create_table_ctx(ObTableLoadParam &param, const common::ObIArray<int64_t> &idx_array);
int begin();
int start_trans();
int check_trans_committed();
int check_merged();
private:
struct TransCtx
{
public:
void reset()
{
trans_id_.reset();
next_sequence_no_array_.reset();
}
public:
table::ObTableLoadTransId trans_id_;
table::ObTableLoadArray<uint64_t> next_sequence_no_array_;
};
private:
static const int64_t DEFAULT_SEGMENT_ID = 1;
common::ObIAllocator *allocator_;
ObTableLoadExecCtx *execute_ctx_;
common::ObIAllocator *allocator_;
sql::ObSQLSessionInfo *session_info_;
ObTableLoadTableCtx *table_ctx_;
ObTableLoadCoordinator *coordinator_;
table::ObTableLoadTransId trans_id_;
table::ObTableLoadArray<uint64_t> next_sequence_no_array_;
sql::ObLoadDataStat *job_stat_;
bool px_mode_;
bool online_opt_stat_gather_;
uint64_t sql_mode_;
TransCtx trans_ctx_;
bool is_committed_;
bool is_inited_;
DISALLOW_COPY_AND_ASSIGN(ObTableLoadInstance);
};

213
src/observer/table_load/ob_table_load_manager.cpp
View File

@ -17,74 +17,13 @@ using namespace common::hash;
using namespace lib;
using namespace table;
/**
* TableCtxHandle
*/
ObTableLoadManager::TableCtxHandle::TableCtxHandle()
: table_ctx_(nullptr)
{
}
ObTableLoadManager::TableCtxHandle::TableCtxHandle(ObTableLoadTableCtx *table_ctx)
: table_ctx_(table_ctx)
{
if (OB_NOT_NULL(table_ctx_)) {
table_ctx_->inc_ref_count();
}
}
ObTableLoadManager::TableCtxHandle::TableCtxHandle(const TableCtxHandle &other)
: table_ctx_(other.table_ctx_)
{
if (OB_NOT_NULL(table_ctx_)) {
table_ctx_->inc_ref_count();
}
}
ObTableLoadManager::TableCtxHandle::~TableCtxHandle()
{
reset();
}
void ObTableLoadManager::TableCtxHandle::reset()
{
if (nullptr != table_ctx_) {
table_ctx_->dec_ref_count();
table_ctx_ = nullptr;
}
}
void ObTableLoadManager::TableCtxHandle::set(ObTableLoadTableCtx *table_ctx)
{
reset();
if (OB_NOT_NULL(table_ctx)) {
table_ctx_ = table_ctx;
table_ctx_->inc_ref_count();
}
}
ObTableLoadManager::TableCtxHandle &ObTableLoadManager::TableCtxHandle::operator=(
const TableCtxHandle &other)
{
if (this != &other) {
set(other.table_ctx_);
}
return *this;
}
/**
* ObTableLoadManager
*/
ObTableLoadManager::ObTableLoadManager()
: is_inited_(false)
{
}
ObTableLoadManager::ObTableLoadManager() : is_inited_(false) {}
ObTableLoadManager::~ObTableLoadManager()
{
}
ObTableLoadManager::~ObTableLoadManager() {}
int ObTableLoadManager::init()
{
@ -95,8 +34,11 @@ int ObTableLoadManager::init()
LOG_WARN("ObTableLoadManager init twice", KR(ret), KP(this));
} else {
if (OB_FAIL(
table_ctx_map_.create(bucket_num, "TLD_TableCtxMap", "TLD_TableCtxMap", MTL_ID()))) {
LOG_WARN("fail to create table ctx map", KR(ret), K(bucket_num));
table_ctx_map_.create(bucket_num, "TLD_TableCtxMgr", "TLD_TableCtxMgr", MTL_ID()))) {
LOG_WARN("fail to create hashmap", KR(ret), K(bucket_num));
} else if (OB_FAIL(table_handle_map_.create(bucket_num, "TLD_TableCtxMgr", "TLD_TableCtxMgr",
MTL_ID()))) {
LOG_WARN("fail to create hashmap", KR(ret), K(bucket_num));
} else {
is_inited_ = true;
}
@ -104,64 +46,92 @@ int ObTableLoadManager::init()
return ret;
}
int ObTableLoadManager::add_table_ctx(uint64_t table_id, ObTableLoadTableCtx *table_ctx)
int ObTableLoadManager::add_table_ctx(const ObTableLoadUniqueKey &key,
ObTableLoadTableCtx *table_ctx)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadManager not init", KR(ret), KP(this));
} else if (OB_UNLIKELY(OB_INVALID_ID == table_id || nullptr == table_ctx)) {
} else if (OB_UNLIKELY(!key.is_valid() || nullptr == table_ctx)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(table_id), KP(table_ctx));
LOG_WARN("invalid args", KR(ret), K(key), KP(table_ctx));
} else if (OB_UNLIKELY(table_ctx->is_dirty())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected dirty table ctx", KR(ret), KP(table_ctx));
} else {
TableCtxHandle handle(table_ctx); // inc_ref_count
if (OB_FAIL(table_ctx_map_.set_refactored(table_id, handle))) {
const uint64_t table_id = key.table_id_;
TableHandle table_handle;
table_handle.key_ = key;
table_handle.table_ctx_ = table_ctx;
obsys::ObWLockGuard guard(rwlock_);
if (OB_FAIL(table_ctx_map_.set_refactored(key, table_ctx))) {
if (OB_UNLIKELY(OB_HASH_EXIST != ret)) {
LOG_WARN("fail to set refactored", KR(ret), K(table_id));
LOG_WARN("fail to set refactored", KR(ret), K(key));
} else {
ret = OB_ENTRY_EXIST;
}
}
// force update table index
else if (OB_FAIL(table_handle_map_.set_refactored(table_id, table_handle, 1))) {
LOG_WARN("fail to set refactored", KR(ret), K(table_id));
// erase from table ctx map, avoid wild pointer is been use
int tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(table_ctx_map_.erase_refactored(key))) {
LOG_WARN("fail to erase refactored", KR(tmp_ret), K(key));
}
} else {
handle.take(); // keep reference count
table_ctx->inc_ref_count();
}
}
return ret;
}
int ObTableLoadManager::remove_table_ctx(uint64_t table_id)
int ObTableLoadManager::remove_table_ctx(const ObTableLoadUniqueKey &key)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadManager not init", KR(ret), KP(this));
} else if (OB_UNLIKELY(OB_INVALID_ID == table_id)) {
} else if (OB_UNLIKELY(!key.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), KP(table_id));
LOG_WARN("invalid args", KR(ret), K(key));
} else {
TableCtxHandle handle;
// remove from map
if (OB_FAIL(table_ctx_map_.erase_refactored(table_id, &handle))) {
if (OB_UNLIKELY(OB_HASH_NOT_EXIST != ret)) {
LOG_WARN("fail to get refactored", KR(ret), K(table_id));
} else {
ret = OB_ENTRY_NOT_EXIST;
const uint64_t table_id = key.table_id_;
ObTableLoadTableCtx *table_ctx = nullptr;
TableHandle table_handle;
{
obsys::ObWLockGuard guard(rwlock_);
if (OB_FAIL(table_ctx_map_.erase_refactored(key, &table_ctx))) {
if (OB_UNLIKELY(OB_HASH_NOT_EXIST == ret)) {
LOG_WARN("fail to erase refactored", KR(ret), K(key));
} else {
ret = OB_ENTRY_NOT_EXIST;
}
}
// remove table index if key match
else if (OB_FAIL(table_handle_map_.get_refactored(table_id, table_handle))) {
if (OB_UNLIKELY(OB_HASH_NOT_EXIST == ret)) {
LOG_WARN("fail to get refactored", KR(ret), K(table_id));
} else {
ret = OB_SUCCESS;
}
} else if (table_handle.key_ == key &&
OB_FAIL(table_handle_map_.erase_refactored(table_id))) {
LOG_WARN("fail to erase refactored", KR(ret), K(table_id), K(table_handle));
}
}
// add to dirty list
else {
ObTableLoadTableCtx *table_ctx = handle.get();
table_ctx->set_dirty();
ObMutexGuard guard(mutex_);
OB_ASSERT(dirty_list_.add_last(table_ctx));
if (OB_SUCC(ret)) {
if (OB_FAIL(add_dirty_list(table_ctx))) {
LOG_WARN("fail to add dirty list", KR(ret), K(key), KP(table_ctx));
}
}
}
return ret;
}
int ObTableLoadManager::get_table_ctx(uint64_t table_id, ObTableLoadTableCtx *&table_ctx)
int ObTableLoadManager::get_table_ctx(const ObTableLoadUniqueKey &key,
ObTableLoadTableCtx *&table_ctx)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
@ -169,15 +139,40 @@ int ObTableLoadManager::get_table_ctx(uint64_t table_id, ObTableLoadTableCtx *&t
LOG_WARN("ObTableLoadManager not init", KR(ret), KP(this));
} else {
table_ctx = nullptr;
TableCtxHandle handle;
if (OB_FAIL(table_ctx_map_.get_refactored(table_id, handle))) {
obsys::ObRLockGuard guard(rwlock_);
if (OB_FAIL(table_ctx_map_.get_refactored(key, table_ctx))) {
if (OB_UNLIKELY(OB_HASH_NOT_EXIST != ret)) {
LOG_WARN("fail to get refactored", KR(ret), K(key));
} else {
ret = OB_ENTRY_NOT_EXIST;
}
} else {
table_ctx->inc_ref_count();
}
}
return ret;
}
int ObTableLoadManager::get_table_ctx_by_table_id(uint64_t table_id,
ObTableLoadTableCtx *&table_ctx)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadManager not init", KR(ret), KP(this));
} else {
table_ctx = nullptr;
TableHandle table_handle;
obsys::ObRLockGuard guard(rwlock_);
if (OB_FAIL(table_handle_map_.get_refactored(table_id, table_handle))) {
if (OB_UNLIKELY(OB_HASH_NOT_EXIST != ret)) {
LOG_WARN("fail to get refactored", KR(ret), K(table_id));
} else {
ret = OB_ENTRY_NOT_EXIST;
}
} else {
table_ctx = handle.take(); // keep reference count
table_ctx = table_handle.table_ctx_;
table_ctx->inc_ref_count();
}
}
return ret;
@ -192,21 +187,24 @@ int ObTableLoadManager::get_inactive_table_ctx_list(
LOG_WARN("ObTableLoadManager not init", KR(ret), KP(this));
} else {
table_ctx_array.reset();
auto fn = [&ret, &table_ctx_array](HashMapPair<uint64_t, TableCtxHandle> &entry) -> int {
TableCtxHandle handle = entry.second; // inc_ref_count
ObTableLoadTableCtx *table_ctx = handle.get();
auto fn = [&ret, &table_ctx_array](
const HashMapPair<ObTableLoadUniqueKey, ObTableLoadTableCtx *> &entry) -> int {
ObTableLoadTableCtx *table_ctx = entry.second;
OB_ASSERT(nullptr != table_ctx);
if (table_ctx->get_ref_count() > 2) { // 2 = map + handle
if (table_ctx->get_ref_count() > 0) {
// skip active table ctx
} else if (OB_FAIL(table_ctx_array.push_back(table_ctx))) {
LOG_WARN("fail to push back", KR(ret));
} else {
handle.take(); // keep reference count
table_ctx->inc_ref_count();
}
return ret;
};
if (OB_FAIL(table_ctx_map_.foreach_refactored(fn))) {
LOG_WARN("fail to foreach map", KR(ret));
{
obsys::ObRLockGuard guard(rwlock_);
if (OB_FAIL(table_ctx_map_.foreach_refactored(fn))) {
LOG_WARN("fail to foreach map", KR(ret));
}
}
if (OB_FAIL(ret)) {
for (int64_t i = 0; i < table_ctx_array.count(); ++i) {
@ -230,6 +228,23 @@ void ObTableLoadManager::put_table_ctx(ObTableLoadTableCtx *table_ctx)
}
}
int ObTableLoadManager::add_dirty_list(ObTableLoadTableCtx *table_ctx)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(nullptr == table_ctx)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), KP(table_ctx));
} else if (OB_UNLIKELY(table_ctx->is_dirty())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected dirty table ctx", KR(ret), KPC(table_ctx));
} else {
table_ctx->set_dirty();
ObMutexGuard guard(mutex_);
OB_ASSERT(dirty_list_.add_last(table_ctx));
}
return ret;
}
int ObTableLoadManager::get_releasable_table_ctx_list(
ObIArray<ObTableLoadTableCtx *> &table_ctx_array)
{

43
src/observer/table_load/ob_table_load_manager.h
View File

@ -7,6 +7,7 @@
#include "lib/hash/ob_hashmap.h"
#include "lib/list/ob_dlist.h"
#include "lib/lock/ob_mutex.h"
#include "observer/table_load/ob_table_load_struct.h"
namespace oceanbase
{
@ -20,38 +21,40 @@ public:
ObTableLoadManager();
~ObTableLoadManager();
int init();
int add_table_ctx(uint64_t table_id, ObTableLoadTableCtx *table_ctx);
int remove_table_ctx(uint64_t table_id);
// table ctx holds a reference count
int get_table_ctx(uint64_t table_id, ObTableLoadTableCtx *&table_ctx);
int add_table_ctx(const ObTableLoadUniqueKey &key, ObTableLoadTableCtx *table_ctx);
int remove_table_ctx(const ObTableLoadUniqueKey &key);
// table ctx holds a reference count
int get_table_ctx(const ObTableLoadUniqueKey &key, ObTableLoadTableCtx *&table_ctx);
// table ctx holds a reference count
int get_table_ctx_by_table_id(uint64_t table_id, ObTableLoadTableCtx *&table_ctx);
// all table ctx hold a reference count
int get_inactive_table_ctx_list(common::ObIArray<ObTableLoadTableCtx *> &table_ctx_array);
void put_table_ctx(ObTableLoadTableCtx *table_ctx);
// table ctx no reference counting
int get_releasable_table_ctx_list(common::ObIArray<ObTableLoadTableCtx *> &table_ctx_array);
public:
int add_dirty_list(ObTableLoadTableCtx *table_ctx);
private:
struct TableCtxHandle
struct TableHandle
{
public:
TableCtxHandle();
TableCtxHandle(ObTableLoadTableCtx *table_ctx);
TableCtxHandle(const TableCtxHandle &other);
~TableCtxHandle();
void reset();
void set(ObTableLoadTableCtx *table_ctx);
TableCtxHandle &operator=(const TableCtxHandle &other);
OB_INLINE ObTableLoadTableCtx *get() const { return table_ctx_; }
OB_INLINE ObTableLoadTableCtx *take()
{
ObTableLoadTableCtx *table_ctx = table_ctx_;
table_ctx_ = nullptr;
return table_ctx;
}
private:
TableHandle() : table_ctx_(nullptr) {}
TO_STRING_KV(K_(key), KP_(table_ctx));
public:
ObTableLoadUniqueKey key_;
ObTableLoadTableCtx *table_ctx_;
};
typedef common::hash::ObHashMap<uint64_t, TableCtxHandle> TableCtxMap;
// key => table_ctx
typedef common::hash::ObHashMap<ObTableLoadUniqueKey, ObTableLoadTableCtx *,
common::hash::NoPthreadDefendMode>
TableCtxMap;
// table_id => table_handle
typedef common::hash::ObHashMap<uint64_t, TableHandle, common::hash::NoPthreadDefendMode>
TableHandleMap;
mutable obsys::ObRWLock rwlock_;
TableCtxMap table_ctx_map_;
TableHandleMap table_handle_map_; // index of the latest task
lib::ObMutex mutex_;
common::ObDList<ObTableLoadTableCtx> dirty_list_;
bool is_inited_;

2
src/observer/table_load/ob_table_load_merger.cpp
View File

@ -187,7 +187,7 @@ int ObTableLoadMerger::build_merge_ctx()
int ret = OB_SUCCESS;
ObDirectLoadMergeParam merge_param;
merge_param.table_id_ = param_.table_id_;
merge_param.target_table_id_ = param_.target_table_id_;
merge_param.target_table_id_ = store_ctx_->ctx_->ddl_param_.dest_table_id_;
merge_param.rowkey_column_num_ = store_ctx_->ctx_->schema_.rowkey_column_count_;
merge_param.schema_column_count_ =
(store_ctx_->ctx_->schema_.is_heap_table_ ? store_ctx_->table_data_desc_.column_count_ + 1

4
src/observer/table_load/ob_table_load_processor.cpp
View File

@ -36,7 +36,7 @@ int ObTableLoadP::process()
LOG_WARN("fail to check_user_access", KR(ret));
} else {
ObTableLoadTableCtx *table_ctx = nullptr;
ObTableLoadKey key(credential_.tenant_id_, arg_.table_id_);
ObTableLoadUniqueKey key(arg_.table_id_, arg_.task_id_);
if (OB_FAIL(ObTableLoadService::get_ctx(key, table_ctx))) {
LOG_WARN("fail to get table ctx", KR(ret), K(key));
} else {
@ -173,7 +173,7 @@ int ObTableLoadPeerP::process()
LOG_WARN("fail to check_user_access", KR(ret));
} else {
ObTableLoadTableCtx *table_ctx = nullptr;
ObTableLoadKey key(credential_.tenant_id_, arg_.table_id_);
ObTableLoadUniqueKey key(arg_.table_id_, arg_.task_id_);
ObTableLoadSharedAllocatorHandle allocator_handle = ObTableLoadSharedAllocatorHandle::make_handle();
int64_t data_len = arg_.payload_.length();
char *buf = nullptr;

115
src/observer/table_load/ob_table_load_redef_table.cpp
View File

@ -18,9 +18,11 @@ using namespace sql;
using namespace obrpc;
namespace observer
{
ObTableLoadRedefTable::ObTableLoadRedefTable()
: ctx_(nullptr),
session_info_(nullptr),
dest_table_id_(OB_INVALID_ID),
ddl_task_id_(0),
schema_version_(0),
is_finish_or_abort_called_(false),
@ -87,7 +89,8 @@ int ObTableLoadRedefTable::start()
} else {
ddl_task_id_ = create_table_res.task_id_;
schema_version_ = create_table_res.schema_version_;
const_cast<ObTableLoadParam &>(ctx_->param_).target_table_id_ = create_table_res.dest_table_id_;
dest_table_id_ = create_table_res.dest_table_id_;
// const_cast<ObTableLoadParam &>(ctx_->param_).target_table_id_ = create_table_res.dest_table_id_;
LOG_INFO("succeed to create hidden table", K(create_table_res), K(create_table_res));
}
THIS_WORKER.set_timeout_ts(origin_timeout_ts);
@ -127,7 +130,7 @@ int ObTableLoadRedefTable::finish()
build_single_replica_response_arg.ls_id_ = share::ObLSID(1);
build_single_replica_response_arg.tablet_id_ = ObTableID(-1);
build_single_replica_response_arg.source_table_id_ = ctx_->param_.table_id_;
build_single_replica_response_arg.dest_schema_id_ = ctx_->param_.target_table_id_;
build_single_replica_response_arg.dest_schema_id_ = dest_table_id_;
build_single_replica_response_arg.ret_code_ = ret;
build_single_replica_response_arg.task_id_ = ddl_task_id_;
build_single_replica_response_arg.snapshot_version_ = 1;
@ -167,5 +170,113 @@ int ObTableLoadRedefTable::abort()
}
return ret;
}
int ObTableLoadRedefTable::start(const ObTableLoadRedefTableStartArg &arg,
ObTableLoadRedefTableStartRes &res, ObSQLSessionInfo &session_info)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!arg.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(arg));
} else {
const int64_t origin_timeout_ts = THIS_WORKER.get_timeout_ts();
ObCreateHiddenTableArg create_table_arg;
ObCreateHiddenTableRes create_table_res;
create_table_arg.reset();
create_table_arg.tenant_id_ = arg.tenant_id_;
create_table_arg.table_id_ = arg.table_id_;
create_table_arg.dest_tenant_id_ = arg.tenant_id_;
create_table_arg.parallelism_ = arg.parallelism_;
create_table_arg.ddl_type_ = share::DDL_DIRECT_LOAD;
create_table_arg.session_id_ = session_info.get_sessid_for_table();
create_table_arg.sql_mode_ = session_info.get_sql_mode();
create_table_arg.tz_info_ = session_info.get_tz_info_wrap().get_tz_info_offset();
create_table_arg.nls_formats_[ObNLSFormatEnum::NLS_DATE] = session_info.get_local_nls_date_format();
create_table_arg.nls_formats_[ObNLSFormatEnum::NLS_TIMESTAMP] = session_info.get_local_nls_timestamp_format();
create_table_arg.nls_formats_[ObNLSFormatEnum::NLS_TIMESTAMP_TZ] = session_info.get_local_nls_timestamp_tz_format();
if (OB_FAIL(create_table_arg.tz_info_wrap_.deep_copy(session_info.get_tz_info_wrap()))) {
LOG_WARN("failed to deep copy tz_info_wrap", KR(ret));
} else if (OB_FAIL(ObDDLServerClient::create_hidden_table(create_table_arg, create_table_res, session_info))) {
LOG_WARN("failed to create hidden table", KR(ret), K(create_table_arg));
} else {
res.dest_table_id_ = create_table_res.dest_table_id_;
res.task_id_ = create_table_res.task_id_;
res.schema_version_ = create_table_res.schema_version_;
LOG_INFO("succeed to create hidden table", K(arg), K(res));
}
THIS_WORKER.set_timeout_ts(origin_timeout_ts);
}
return ret;
}
int ObTableLoadRedefTable::finish(const ObTableLoadRedefTableFinishArg &arg,
ObSQLSessionInfo &session_info)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!arg.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(arg));
} else {
const int64_t origin_timeout_ts = THIS_WORKER.get_timeout_ts();
ObCopyTableDependentsArg copy_table_dependents_arg;
copy_table_dependents_arg.task_id_ = arg.task_id_;
copy_table_dependents_arg.tenant_id_ = arg.tenant_id_;
copy_table_dependents_arg.copy_indexes_ = true;
copy_table_dependents_arg.copy_constraints_ = true;
copy_table_dependents_arg.copy_triggers_ = false;
copy_table_dependents_arg.ignore_errors_ = false;
if (OB_FAIL(ObDDLServerClient::copy_table_dependents(copy_table_dependents_arg))) {
LOG_WARN("failed to copy table dependents", KR(ret), K(copy_table_dependents_arg));
} else {
LOG_INFO("succeed to copy table dependents", K(copy_table_dependents_arg));
ObFinishRedefTableArg finish_redef_table_arg;
finish_redef_table_arg.task_id_ = arg.task_id_;
finish_redef_table_arg.tenant_id_ = arg.tenant_id_;
ObDDLBuildSingleReplicaResponseArg build_single_replica_response_arg;
build_single_replica_response_arg.task_id_ = arg.task_id_;
build_single_replica_response_arg.tenant_id_ = arg.tenant_id_;
build_single_replica_response_arg.ls_id_ = share::ObLSID(1);
build_single_replica_response_arg.tablet_id_ = ObTableID(-1);
build_single_replica_response_arg.source_table_id_ = arg.table_id_;
build_single_replica_response_arg.dest_schema_id_ = arg.dest_table_id_;
build_single_replica_response_arg.ret_code_ = ret;
build_single_replica_response_arg.snapshot_version_ = 1;
build_single_replica_response_arg.schema_version_ = arg.schema_version_;
build_single_replica_response_arg.execution_id_ = 1;
if (OB_FAIL(ObDDLServerClient::finish_redef_table(
finish_redef_table_arg, build_single_replica_response_arg, session_info))) {
LOG_WARN("failed to finish redef table", KR(ret), K(finish_redef_table_arg));
} else {
LOG_INFO("succeed to finish redef table", KR(ret), K(finish_redef_table_arg));
}
}
THIS_WORKER.set_timeout_ts(origin_timeout_ts);
}
return ret;
}
int ObTableLoadRedefTable::abort(const ObTableLoadRedefTableAbortArg &arg,
ObSQLSessionInfo &session_info)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!arg.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(arg));
} else {
const int64_t origin_timeout_ts = THIS_WORKER.get_timeout_ts();
ObAbortRedefTableArg abort_redef_table_arg;
abort_redef_table_arg.task_id_ = arg.task_id_;
abort_redef_table_arg.tenant_id_ = arg.tenant_id_;
if (OB_FAIL(ObDDLServerClient::abort_redef_table(abort_redef_table_arg, session_info))) {
LOG_WARN("failed to abort redef table", KR(ret), K(abort_redef_table_arg));
} else {
LOG_INFO("succeed to abort hidden table", K(arg));
}
THIS_WORKER.set_timeout_ts(origin_timeout_ts);
}
return ret;
}
} // namespace observer
} // namespace oceanbase

110
src/observer/table_load/ob_table_load_redef_table.h
View File

@ -16,6 +16,104 @@ namespace observer
{
class ObTableLoadTableCtx;
struct ObTableLoadRedefTableStartArg
{
public:
ObTableLoadRedefTableStartArg()
: tenant_id_(common::OB_INVALID_ID), table_id_(common::OB_INVALID_ID), parallelism_(0)
{
}
~ObTableLoadRedefTableStartArg() = default;
void reset()
{
tenant_id_ = common::OB_INVALID_ID;
table_id_ = common::OB_INVALID_ID;
parallelism_ = 0;
}
bool is_valid() const
{
return common::OB_INVALID_ID != tenant_id_ && common::OB_INVALID_ID != table_id_ &&
0 != parallelism_;
}
TO_STRING_KV(K_(tenant_id), K_(table_id), K_(parallelism));
public:
uint64_t tenant_id_;
uint64_t table_id_;
uint64_t parallelism_;
};
struct ObTableLoadRedefTableStartRes
{
public:
ObTableLoadRedefTableStartRes()
: dest_table_id_(common::OB_INVALID_ID), task_id_(0), schema_version_(0)
{
}
~ObTableLoadRedefTableStartRes() = default;
void reset()
{
dest_table_id_ = common::OB_INVALID_ID;
task_id_ = 0;
schema_version_ = 0;
}
TO_STRING_KV(K_(dest_table_id), K_(task_id), K_(schema_version));
public:
uint64_t dest_table_id_;
int64_t task_id_;
int64_t schema_version_;
};
struct ObTableLoadRedefTableFinishArg
{
public:
ObTableLoadRedefTableFinishArg()
: tenant_id_(common::OB_INVALID_ID),
table_id_(common::OB_INVALID_ID),
dest_table_id_(common::OB_INVALID_ID),
task_id_(0),
schema_version_(0)
{
}
~ObTableLoadRedefTableFinishArg() = default;
void reset()
{
tenant_id_ = common::OB_INVALID_ID;
table_id_ = common::OB_INVALID_ID;
dest_table_id_ = common::OB_INVALID_ID;
task_id_ = 0;
schema_version_ = 0;
}
bool is_valid() const
{
return common::OB_INVALID_ID != tenant_id_ && common::OB_INVALID_ID != table_id_ &&
common::OB_INVALID_ID != dest_table_id_ && 0 != task_id_ && 0 != schema_version_;
}
TO_STRING_KV(K_(tenant_id), K_(table_id), K_(dest_table_id), K_(task_id), K_(schema_version));
public:
uint64_t tenant_id_;
uint64_t table_id_;
uint64_t dest_table_id_;
int64_t task_id_;
int64_t schema_version_;
};
struct ObTableLoadRedefTableAbortArg
{
public:
ObTableLoadRedefTableAbortArg() : tenant_id_(common::OB_INVALID_ID), task_id_(0) {}
~ObTableLoadRedefTableAbortArg() = default;
void reset()
{
tenant_id_ = common::OB_INVALID_ID;
task_id_ = 0;
}
bool is_valid() const { return common::OB_INVALID_ID != tenant_id_ && 0 != task_id_; }
TO_STRING_KV(K_(tenant_id), K_(task_id));
public:
uint64_t tenant_id_;
int64_t task_id_;
};
class ObTableLoadRedefTable
{
public:
@ -27,13 +125,19 @@ public:
int finish();
int abort();
OB_INLINE int64_t get_ddl_task_id() const { return ddl_task_id_; }
static int start(const ObTableLoadRedefTableStartArg &arg, ObTableLoadRedefTableStartRes &res,
sql::ObSQLSessionInfo &session_info);
static int finish(const ObTableLoadRedefTableFinishArg &arg, sql::ObSQLSessionInfo &session_info);
static int abort(const ObTableLoadRedefTableAbortArg &arg, sql::ObSQLSessionInfo &session_info);
private:
ObTableLoadTableCtx * ctx_;
ObTableLoadTableCtx *ctx_;
sql::ObSQLSessionInfo *session_info_;
uint64_t dest_table_id_;
int64_t ddl_task_id_;
int64_t schema_version_;
bool is_finish_or_abort_called_;
bool is_inited_;
};
} // namespace observer
} // namespace oceanbase
} // namespace observer
} // namespace oceanbase

3
src/observer/table_load/ob_table_load_schema.cpp
View File

@ -122,8 +122,7 @@ int ObTableLoadSchema::get_schema_version(uint64_t tenant_id, uint64_t table_id,
return ret;
}
int ObTableLoadSchema::check_constraints(uint64_t tenant_id,
ObSchemaGetterGuard &schema_guard,
int ObTableLoadSchema::check_constraints(ObSchemaGetterGuard &schema_guard,
const ObTableSchema *table_schema)
{
int ret = OB_SUCCESS;

3
src/observer/table_load/ob_table_load_schema.h
View File

@ -32,8 +32,7 @@ public:
common::ObIAllocator &allocator,
table::ObTableLoadArray<common::ObString> &column_names);
static int get_schema_version(uint64_t tenant_id, uint64_t table_id, int64_t &schema_version);
static int check_constraints(uint64_t tenant_id,
share::schema::ObSchemaGetterGuard &schema_guard,
static int check_constraints(share::schema::ObSchemaGetterGuard &schema_guard,
const share::schema::ObTableSchema *table_schema);
public:
ObTableLoadSchema();

186
src/observer/table_load/ob_table_load_service.cpp
View File

@ -44,21 +44,22 @@ void ObTableLoadService::ObGCTask::runTimerTask()
LOG_WARN("ObTableLoadService::ObGCTask not init", KR(ret), KP(this));
} else {
LOG_DEBUG("table load start gc", K(tenant_id_));
ObTableLoadManager &manager = service_.get_manager();
ObArray<ObTableLoadTableCtx *> inactive_table_ctx_array;
if (OB_FAIL(service_.manager_.get_inactive_table_ctx_list(inactive_table_ctx_array))) {
if (OB_FAIL(manager.get_inactive_table_ctx_list(inactive_table_ctx_array))) {
LOG_WARN("fail to get inactive table ctx list", KR(ret), K(tenant_id_));
}
for (int64_t i = 0; i < inactive_table_ctx_array.count(); ++i) {
ObTableLoadTableCtx *table_ctx = inactive_table_ctx_array.at(i);
const uint64_t table_id = table_ctx->param_.table_id_;
const uint64_t target_table_id = table_ctx->param_.target_table_id_;
const uint64_t hidden_table_id = table_ctx->ddl_param_.dest_table_id_;
// check if table ctx is removed
if (table_ctx->is_dirty()) {
LOG_DEBUG("table load ctx is dirty", K(tenant_id_), "table_id", table_ctx->param_.table_id_,
"ref_count", table_ctx->get_ref_count());
}
// check if table ctx is activated
else if (table_ctx->get_ref_count() > 2) {
else if (table_ctx->get_ref_count() > 1) {
LOG_DEBUG("table load ctx is active", K(tenant_id_), "table_id",
table_ctx->param_.table_id_, "ref_count", table_ctx->get_ref_count());
}
@ -66,28 +67,28 @@ void ObTableLoadService::ObGCTask::runTimerTask()
else {
ObSchemaGetterGuard schema_guard;
const ObTableSchema *table_schema = nullptr;
if (target_table_id == OB_INVALID_ID) {
if (hidden_table_id == OB_INVALID_ID) {
LOG_INFO("hidden table has not been created, gc table load ctx", K(tenant_id_),
K(table_id), K(target_table_id));
service_.remove_table_ctx(table_ctx);
} else if (OB_FAIL(ObTableLoadSchema::get_table_schema(tenant_id_, target_table_id,
K(table_id), K(hidden_table_id));
ObTableLoadService::remove_ctx(table_ctx);
} else if (OB_FAIL(ObTableLoadSchema::get_table_schema(tenant_id_, hidden_table_id,
schema_guard, table_schema))) {
if (OB_UNLIKELY(OB_TABLE_NOT_EXIST != ret)) {
LOG_WARN("fail to get table schema", KR(ret), K(tenant_id_), K(target_table_id));
LOG_WARN("fail to get table schema", KR(ret), K(tenant_id_), K(hidden_table_id));
} else {
LOG_INFO("hidden table not exist, gc table load ctx", K(tenant_id_), K(table_id),
K(target_table_id));
service_.remove_table_ctx(table_ctx);
K(hidden_table_id));
ObTableLoadService::remove_ctx(table_ctx);
}
} else if (table_schema->is_in_recyclebin()) {
LOG_INFO("hidden table is in recyclebin, gc table load ctx", K(tenant_id_), K(table_id),
K(target_table_id));
service_.remove_table_ctx(table_ctx);
K(hidden_table_id));
ObTableLoadService::remove_ctx(table_ctx);
} else {
LOG_DEBUG("table load ctx is running", K(tenant_id_), K(table_id), K(target_table_id));
LOG_DEBUG("table load ctx is running", K(tenant_id_), K(table_id), K(hidden_table_id));
}
}
service_.put_table_ctx(table_ctx);
manager.put_table_ctx(table_ctx);
}
}
}
@ -124,9 +125,9 @@ void ObTableLoadService::ObReleaseTask::runTimerTask()
for (int64_t i = 0; i < releasable_table_ctx_array.count(); ++i) {
ObTableLoadTableCtx *table_ctx = releasable_table_ctx_array.at(i);
const uint64_t table_id = table_ctx->param_.table_id_;
const uint64_t target_table_id = table_ctx->param_.target_table_id_;
LOG_INFO("free table ctx", K(tenant_id_), K(table_id), K(target_table_id), KP(table_ctx));
OB_DELETE(ObTableLoadTableCtx, "TLD_TableCtxVal", table_ctx);
const uint64_t hidden_table_id = table_ctx->ddl_param_.dest_table_id_;
LOG_INFO("free table ctx", K(tenant_id_), K(table_id), K(hidden_table_id), KP(table_ctx));
ObTableLoadService::free_ctx(table_ctx);
}
}
}
@ -148,8 +149,40 @@ int ObTableLoadService::mtl_init(ObTableLoadService *&service)
return ret;
}
int ObTableLoadService::create_ctx(const ObTableLoadParam &param, ObTableLoadTableCtx *&table_ctx,
bool &is_new)
int ObTableLoadService::check_support_direct_load(uint64_t table_id)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(OB_INVALID_ID == table_id)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(table_id));
} else {
const uint64_t tenant_id = MTL_ID();
ObSchemaGetterGuard schema_guard;
const ObTableSchema *table_schema = nullptr;
if (OB_FAIL(
ObTableLoadSchema::get_table_schema(tenant_id, table_id, schema_guard, table_schema))) {
LOG_WARN("fail to get table schema", KR(ret), K(tenant_id), K(table_id));
} else if (OB_FAIL(ObTableLoadSchema::check_constraints(schema_guard, table_schema))) {
LOG_WARN("fail to check schema constraints", KR(ret), K(tenant_id), K(table_id));
}
}
return ret;
}
ObTableLoadTableCtx *ObTableLoadService::alloc_ctx()
{
return OB_NEW(ObTableLoadTableCtx, ObMemAttr(MTL_ID(), "TLD_TableCtxVal"));
}
void ObTableLoadService::free_ctx(ObTableLoadTableCtx *table_ctx)
{
if (OB_NOT_NULL(table_ctx)) {
OB_DELETE(ObTableLoadTableCtx, "TLD_TableCtxVal", table_ctx);
table_ctx = nullptr;
}
}
int ObTableLoadService::add_ctx(ObTableLoadTableCtx *table_ctx)
{
int ret = OB_SUCCESS;
ObTableLoadService *service = nullptr;
@ -157,7 +190,35 @@ int ObTableLoadService::create_ctx(const ObTableLoadParam &param, ObTableLoadTab
ret = OB_ERR_SYS;
LOG_WARN("null table load service", KR(ret));
} else {
ret = service->create_table_ctx(param, table_ctx, is_new);
ObTableLoadUniqueKey key(table_ctx->param_.table_id_, table_ctx->ddl_param_.task_id_);
ret = service->get_manager().add_table_ctx(key, table_ctx);
}
return ret;
}
int ObTableLoadService::remove_ctx(ObTableLoadTableCtx *table_ctx)
{
int ret = OB_SUCCESS;
ObTableLoadService *service = nullptr;
if (OB_ISNULL(service = MTL(ObTableLoadService *))) {
ret = OB_ERR_SYS;
LOG_WARN("null table load service", KR(ret));
} else {
ObTableLoadUniqueKey key(table_ctx->param_.table_id_, table_ctx->ddl_param_.task_id_);
ret = service->get_manager().remove_table_ctx(key);
}
return ret;
}
int ObTableLoadService::get_ctx(const ObTableLoadUniqueKey &key, ObTableLoadTableCtx *&table_ctx)
{
int ret = OB_SUCCESS;
ObTableLoadService *service = nullptr;
if (OB_ISNULL(service = MTL(ObTableLoadService *))) {
ret = OB_ERR_SYS;
LOG_WARN("null table load service", KR(ret));
} else {
ret = service->get_manager().get_table_ctx(key, table_ctx);
}
return ret;
}
@ -170,7 +231,7 @@ int ObTableLoadService::get_ctx(const ObTableLoadKey &key, ObTableLoadTableCtx *
ret = OB_ERR_SYS;
LOG_WARN("null table load service", KR(ret));
} else {
ret = service->get_table_ctx(key.table_id_, table_ctx);
ret = service->get_manager().get_table_ctx_by_table_id(key.table_id_, table_ctx);
}
return ret;
}
@ -183,23 +244,10 @@ void ObTableLoadService::put_ctx(ObTableLoadTableCtx *table_ctx)
ret = OB_ERR_SYS;
LOG_WARN("null table load service", KR(ret));
} else {
service->put_table_ctx(table_ctx);
service->get_manager().put_table_ctx(table_ctx);
}
}
int ObTableLoadService::remove_ctx(ObTableLoadTableCtx *table_ctx)
{
int ret = OB_SUCCESS;
ObTableLoadService *service = nullptr;
if (OB_ISNULL(service = MTL(ObTableLoadService *))) {
ret = OB_ERR_SYS;
LOG_WARN("null table load service", KR(ret));
} else {
ret = service->remove_table_ctx(table_ctx);
}
return ret;
}
ObTableLoadService::ObTableLoadService()
: gc_task_(*this), release_task_(*this), is_inited_(false)
{
@ -260,73 +308,5 @@ void ObTableLoadService::destroy()
gc_timer_.destroy();
}
int ObTableLoadService::create_table_ctx(const ObTableLoadParam &param,
ObTableLoadTableCtx *&table_ctx, bool &is_new)
{
int ret = OB_SUCCESS;
table_ctx = nullptr;
is_new = false;
if (OB_UNLIKELY(!param.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(param));
} else {
const uint64_t table_id = param.table_id_;
ObTableLoadTableCtx *new_table_ctx = nullptr;
if (OB_FAIL(manager_.get_table_ctx(table_id, table_ctx))) {
if (OB_UNLIKELY(OB_ENTRY_NOT_EXIST != ret)) {
LOG_WARN("fail to get table ctx", KR(ret), K(table_id));
} else {
table_ctx = nullptr;
ret = OB_SUCCESS;
}
}
if (OB_SUCC(ret) && nullptr == table_ctx) {
if (OB_ISNULL(new_table_ctx =
OB_NEW(ObTableLoadTableCtx, ObMemAttr(MTL_ID(), "TLD_TableCtxVal"), param))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new table ctx", KR(ret), K(param));
} else if (OB_FAIL(new_table_ctx->init())) {
LOG_WARN("fail to init table ctx", KR(ret));
} else if (OB_FAIL(manager_.add_table_ctx(table_id, new_table_ctx))) {
LOG_WARN("fail to add table ctx", KR(ret), K(table_id));
} else {
table_ctx = new_table_ctx;
is_new = true;
}
}
if (OB_FAIL(ret)) {
if (nullptr != new_table_ctx) {
OB_DELETE(ObTableLoadTableCtx, "TLD_TableCtxVal", new_table_ctx);
new_table_ctx = nullptr;
}
}
}
return ret;
}
int ObTableLoadService::remove_table_ctx(ObTableLoadTableCtx *table_ctx)
{
int ret = OB_SUCCESS;
const uint64_t table_id = table_ctx->param_.table_id_;
if (OB_FAIL(manager_.remove_table_ctx(table_id))) {
LOG_WARN("fail to remove table ctx", KR(ret), K(table_id));
}
return ret;
}
int ObTableLoadService::get_table_ctx(uint64_t table_id, ObTableLoadTableCtx *&ctx)
{
int ret = OB_SUCCESS;
if (OB_FAIL(manager_.get_table_ctx(table_id, ctx))) {
LOG_WARN("fail to get table ctx", KR(ret), K(table_id));
}
return ret;
}
void ObTableLoadService::put_table_ctx(ObTableLoadTableCtx *table_ctx)
{
manager_.put_table_ctx(table_ctx);
}
} // namespace observer
} // namespace oceanbase

20
src/observer/table_load/ob_table_load_service.h
View File

@ -18,10 +18,16 @@ class ObTableLoadService
{
public:
static int mtl_init(ObTableLoadService *&service);
static int create_ctx(const ObTableLoadParam &param, ObTableLoadTableCtx *&ctx, bool &is_new);
static int get_ctx(const ObTableLoadKey &key, ObTableLoadTableCtx *&ctx);
static void put_ctx(ObTableLoadTableCtx *ctx);
static int remove_ctx(ObTableLoadTableCtx *ctx);
static int check_support_direct_load(uint64_t table_id);
static ObTableLoadTableCtx *alloc_ctx();
static void free_ctx(ObTableLoadTableCtx *table_ctx);
static int add_ctx(ObTableLoadTableCtx *table_ctx);
static int remove_ctx(ObTableLoadTableCtx *table_ctx);
// get ctx
static int get_ctx(const ObTableLoadUniqueKey &key, ObTableLoadTableCtx *&table_ctx);
// get ctx by table_id
static int get_ctx(const ObTableLoadKey &key, ObTableLoadTableCtx *&table_ctx);
static void put_ctx(ObTableLoadTableCtx *table_ctx);
public:
ObTableLoadService();
int init(uint64_t tenant_id);
@ -29,11 +35,7 @@ public:
int stop();
void wait();
void destroy();
public:
int create_table_ctx(const ObTableLoadParam &param, ObTableLoadTableCtx *&ctx, bool &is_new);
int remove_table_ctx(ObTableLoadTableCtx *ctx);
int get_table_ctx(uint64_t table_id, ObTableLoadTableCtx *&ctx);
void put_table_ctx(ObTableLoadTableCtx *ctx);
ObTableLoadManager &get_manager() { return manager_; }
private:
static const int64_t GC_INTERVAL = 30LL * 1000 * 1000; // 30s
static const int64_t RELEASE_INTERVAL = 1LL * 1000 * 1000; // 1s

15
src/observer/table_load/ob_table_load_store.cpp
View File

@ -28,22 +28,22 @@ ObTableLoadStore::ObTableLoadStore(ObTableLoadTableCtx *ctx)
{
}
int ObTableLoadStore::init_ctx(ObTableLoadTableCtx *ctx,
int64_t ddl_task_id,
const ObTableLoadArray<ObTableLoadLSIdAndPartitionId> &partition_id_array,
const ObTableLoadArray<ObTableLoadLSIdAndPartitionId> &target_partition_id_array)
int ObTableLoadStore::init_ctx(
ObTableLoadTableCtx *ctx,
const ObTableLoadArray<ObTableLoadLSIdAndPartitionId> &partition_id_array,
const ObTableLoadArray<ObTableLoadLSIdAndPartitionId> &target_partition_id_array)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(ctx)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid agrs", KR(ret));
} else if (OB_FAIL(ctx->init_store_ctx(ddl_task_id, partition_id_array, target_partition_id_array))) {
} else if (OB_FAIL(ctx->init_store_ctx(partition_id_array, target_partition_id_array))) {
LOG_WARN("fail to init store ctx", KR(ret));
}
return ret;
}
int ObTableLoadStore::abort_ctx(ObTableLoadTableCtx *ctx)
void ObTableLoadStore::abort_ctx(ObTableLoadTableCtx *ctx)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!ctx->is_valid())) {
@ -58,11 +58,10 @@ int ObTableLoadStore::abort_ctx(ObTableLoadTableCtx *ctx)
LOG_WARN("fail to set store status abort", KR(ret));
}
// 2. mark all active trans abort
else if (OB_FAIL(abort_active_trans(ctx))) {
if (OB_FAIL(abort_active_trans(ctx))) {
LOG_WARN("fail to abort active trans", KR(ret));
}
}
return ret;
}
int ObTableLoadStore::abort_active_trans(ObTableLoadTableCtx *ctx)

3
src/observer/table_load/ob_table_load_store.h
View File

@ -24,10 +24,9 @@ public:
ObTableLoadStore(ObTableLoadTableCtx *ctx);
static int init_ctx(
ObTableLoadTableCtx *ctx,
int64_t ddl_task_id,
const table::ObTableLoadArray<table::ObTableLoadLSIdAndPartitionId> &partition_id_array,
const table::ObTableLoadArray<table::ObTableLoadLSIdAndPartitionId> &target_partition_id_array);
static int abort_ctx(ObTableLoadTableCtx *ctx);
static void abort_ctx(ObTableLoadTableCtx *ctx);
int init();
private:
static int abort_active_trans(ObTableLoadTableCtx *ctx);

25
src/observer/table_load/ob_table_load_store_ctx.cpp
View File

@ -57,9 +57,9 @@ ObTableLoadStoreCtx::~ObTableLoadStoreCtx()
destroy();
}
int ObTableLoadStoreCtx::init(int64_t ddl_task_id,
const ObTableLoadArray<ObTableLoadLSIdAndPartitionId> &partition_id_array,
const ObTableLoadArray<ObTableLoadLSIdAndPartitionId> &target_partition_id_array)
int ObTableLoadStoreCtx::init(
const ObTableLoadArray<ObTableLoadLSIdAndPartitionId> &partition_id_array,
const ObTableLoadArray<ObTableLoadLSIdAndPartitionId> &target_partition_id_array)
{
int ret = OB_SUCCESS;
if (IS_INIT) {
@ -71,9 +71,9 @@ int ObTableLoadStoreCtx::init(int64_t ddl_task_id,
} else {
ObDirectLoadInsertTableParam insert_table_param;
insert_table_param.table_id_ = ctx_->param_.table_id_;
insert_table_param.schema_version_ = ctx_->schema_.schema_version_;
insert_table_param.schema_version_ = ctx_->ddl_param_.schema_version_;
insert_table_param.snapshot_version_ = ObTimeUtil::current_time_ns();
insert_table_param.ddl_task_id_ = ddl_task_id;
insert_table_param.ddl_task_id_ = ctx_->ddl_param_.task_id_;
insert_table_param.execution_id_ = 1; //仓氐说暂时设置为1，不然后面检测过不了
for (int64_t i = 0; OB_SUCC(ret) && i < partition_id_array.count(); ++i) {
const ObLSID &ls_id = partition_id_array[i].ls_id_;
@ -543,21 +543,16 @@ int ObTableLoadStoreCtx::generate_autoinc_params(AutoincParam &autoinc_param)
int ObTableLoadStoreCtx::init_sequence()
{
int ret = OB_SUCCESS;
const uint64_t tenant_id = ctx_->param_.tenant_id_;
const uint64_t table_id = ctx_->ddl_param_.dest_table_id_;
share::schema::ObSchemaGetterGuard table_schema_guard;
share::schema::ObSchemaGetterGuard sequence_schema_guard;
const ObSequenceSchema *sequence_schema = nullptr;
uint64_t tenant_id = ctx_->param_.tenant_id_;
const ObTableSchema *target_table_schema = nullptr;
uint64_t sequence_id = OB_INVALID_ID;
if (OB_FAIL(ObTableLoadSchema::get_table_schema(ctx_->param_.tenant_id_,
ctx_->param_.target_table_id_,
table_schema_guard, target_table_schema))) {
LOG_WARN("fail to get table schema", KR(ret), K(ctx_->param_.tenant_id_),
K(ctx_->param_.target_table_id_));
} else if (OB_ISNULL(target_table_schema)) {
ret = OB_TABLE_NOT_EXIST;
LOG_WARN("table not exist", KR(ret), K(tenant_id),
K(ctx_->param_.target_table_id_));
if (OB_FAIL(ObTableLoadSchema::get_table_schema(tenant_id, table_id, table_schema_guard,
target_table_schema))) {
LOG_WARN("fail to get table schema", KR(ret), K(tenant_id), K(table_id));
} else {
//ddl对于identity是建表的时候进行自增值同步，对于sequence参数初始化得用隐藏表table id的table schema
for (ObTableSchema::const_column_iterator iter = target_table_schema->column_begin();

6
src/observer/table_load/ob_table_load_store_ctx.h
View File

@ -40,9 +40,9 @@ static const int64_t MACRO_BLOCK_WRITER_MEM_SIZE = 10 * 1024LL * 1024LL;
public:
ObTableLoadStoreCtx(ObTableLoadTableCtx *ctx);
~ObTableLoadStoreCtx();
int init(int64_t ddl_task_id,
const table::ObTableLoadArray<table::ObTableLoadLSIdAndPartitionId> &partition_id_array,
const table::ObTableLoadArray<table::ObTableLoadLSIdAndPartitionId> &target_partition_id_array);
int init(
const table::ObTableLoadArray<table::ObTableLoadLSIdAndPartitionId> &partition_id_array,
const table::ObTableLoadArray<table::ObTableLoadLSIdAndPartitionId> &target_partition_id_array);
void stop();
void destroy();
bool is_valid() const { return is_inited_; }

91
src/observer/table_load/ob_table_load_struct.h
View File

@ -34,8 +34,6 @@ struct ObTableLoadKey
public:
ObTableLoadKey() : tenant_id_(common::OB_INVALID_ID), table_id_(common::OB_INVALID_ID) {}
ObTableLoadKey(uint64_t tenant_id, uint64_t table_id) : tenant_id_(tenant_id), table_id_(table_id) {}
uint64_t tenant_id_;
uint64_t table_id_;
bool is_valid() const
{
return common::OB_INVALID_ID != tenant_id_ && common::OB_INVALID_ID != table_id_;
@ -60,6 +58,39 @@ public:
: table_id_ - other.table_id_);
}
TO_STRING_KV(K_(tenant_id), K_(table_id));
public:
uint64_t tenant_id_;
uint64_t table_id_;
};
struct ObTableLoadUniqueKey
{
public:
ObTableLoadUniqueKey() : table_id_(common::OB_INVALID_ID), task_id_(0) {}
ObTableLoadUniqueKey(uint64_t table_id, int64_t task_id) : table_id_(table_id), task_id_(task_id)
{
}
bool is_valid() const { return common::OB_INVALID_ID != table_id_ && 0 != task_id_; }
bool operator==(const ObTableLoadUniqueKey &other) const
{
return (table_id_ == other.table_id_ && task_id_ == other.task_id_);
}
bool operator!=(const ObTableLoadUniqueKey &other) const
{
return !(*this == other);
}
uint64_t hash() const
{
return common::murmurhash(this, sizeof(*this), 0);
}
int compare(const ObTableLoadUniqueKey &other) const
{
return (table_id_ != other.table_id_ ? table_id_ - other.table_id_ : task_id_ - other.task_id_);
}
TO_STRING_KV(K_(table_id), K_(task_id));
public:
uint64_t table_id_;
int64_t task_id_;
};
struct ObTableLoadParam
@ -67,7 +98,6 @@ struct ObTableLoadParam
ObTableLoadParam()
: tenant_id_(common::OB_INVALID_ID),
table_id_(common::OB_INVALID_ID),
target_table_id_(common::OB_INVALID_ID),
session_count_(0),
batch_size_(0),
max_error_row_count_(0),
@ -80,19 +110,6 @@ struct ObTableLoadParam
dup_action_(sql::ObLoadDupActionType::LOAD_INVALID_MODE)
{
}
uint64_t tenant_id_;
uint64_t table_id_;
uint64_t target_table_id_;
int32_t session_count_;
int32_t batch_size_;
uint64_t max_error_row_count_;
uint64_t sql_mode_;
int32_t column_count_;
bool need_sort_;
bool px_mode_;
bool online_opt_stat_gather_;
ObTableLoadDataType data_type_;
sql::ObLoadDupActionType dup_action_;
int normalize()
{
@ -109,15 +126,48 @@ struct ObTableLoadParam
{
return common::OB_INVALID_ID != tenant_id_ &&
common::OB_INVALID_ID != table_id_ &&
//common::OB_INVALID_ID != target_table_id_ &&
session_count_ > 0 && session_count_ <= MAX_TABLE_LOAD_SESSION_COUNT &&
batch_size_ > 0 &&
column_count_ > 0;
}
TO_STRING_KV(K_(tenant_id), K_(table_id), K_(target_table_id), K_(session_count),
K_(batch_size), K_(max_error_row_count), K_(column_count),
K_(need_sort), K_(px_mode), K_(data_type), K_(dup_action));
TO_STRING_KV(K_(tenant_id), K_(table_id), K_(session_count), K_(batch_size),
K_(max_error_row_count), K_(sql_mode), K_(column_count), K_(need_sort), K_(px_mode),
K_(online_opt_stat_gather), K_(data_type), K_(dup_action));
public:
uint64_t tenant_id_;
uint64_t table_id_;
int32_t session_count_;
int32_t batch_size_;
uint64_t max_error_row_count_;
uint64_t sql_mode_;
int32_t column_count_;
bool need_sort_;
bool px_mode_;
bool online_opt_stat_gather_;
ObTableLoadDataType data_type_;
sql::ObLoadDupActionType dup_action_;
};
struct ObTableLoadDDLParam
{
public:
ObTableLoadDDLParam() : dest_table_id_(common::OB_INVALID_ID), task_id_(0), schema_version_(0) {}
void reset()
{
dest_table_id_ = common::OB_INVALID_ID;
task_id_ = 0;
schema_version_ = 0;
}
bool is_valid() const
{
return common::OB_INVALID_ID != dest_table_id_ && 0 != task_id_ && 0 != schema_version_;
}
TO_STRING_KV(K_(dest_table_id), K_(task_id), K_(schema_version));
public:
uint64_t dest_table_id_;
int64_t task_id_;
int64_t schema_version_;
};
class ObTableLoadMutexGuard
@ -133,7 +183,6 @@ public:
int ret = common::OB_SUCCESS;
if (nullptr != mutex_) {
if (OB_FAIL(mutex_->unlock())) {
SERVER_LOG(WARN, "fail to unlock mutex", KR(ret));
}
mutex_ = nullptr;
}

40
src/observer/table_load/ob_table_load_table_ctx.cpp
View File

@ -23,12 +23,11 @@ using namespace sql;
using namespace table;
using namespace obrpc;
ObTableLoadTableCtx::ObTableLoadTableCtx(const ObTableLoadParam &param)
: param_(param),
coordinator_ctx_(nullptr),
ObTableLoadTableCtx::ObTableLoadTableCtx()
: coordinator_ctx_(nullptr),
store_ctx_(nullptr),
job_stat_(nullptr),
allocator_("TLD_TableCtx", OB_MALLOC_NORMAL_BLOCK_SIZE, param.tenant_id_),
allocator_("TLD_TableCtx"),
ref_count_(0),
is_dirty_(false),
is_inited_(false)
@ -40,13 +39,19 @@ ObTableLoadTableCtx::~ObTableLoadTableCtx()
destroy();
}
int ObTableLoadTableCtx::init()
int ObTableLoadTableCtx::init(const ObTableLoadParam &param, const ObTableLoadDDLParam &ddl_param)
{
int ret = OB_SUCCESS;
if (IS_INIT) {
ret = OB_INIT_TWICE;
LOG_WARN("ObTableLoadTableCtx init twice", KR(ret));
} else if (OB_UNLIKELY(!param.is_valid() || !ddl_param.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(param), K(ddl_param));
} else {
param_ = param;
ddl_param_ = ddl_param;
allocator_.set_tenant_id(MTL_ID());
if (OB_FAIL(schema_.init(param_.tenant_id_, param_.table_id_))) {
LOG_WARN("fail to init table load schema", KR(ret), K(param_.tenant_id_),
K(param_.table_id_));
@ -142,24 +147,20 @@ void ObTableLoadTableCtx::unregister_job_stat()
}
}
int ObTableLoadTableCtx::init_coordinator_ctx(const ObIArray<int64_t> &idx_array,
ObSQLSessionInfo *session_info)
int ObTableLoadTableCtx::init_coordinator_ctx(const ObIArray<int64_t> &idx_array, uint64_t user_id)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadTableCtx not init", KR(ret));
} else if (OB_NOT_NULL(coordinator_ctx_)) {
ret = OB_ENTRY_EXIST;
ret = OB_ERR_UNEXPECTED;
LOG_WARN("coordinator ctx already exist", KR(ret));
} else {
if (session_info == nullptr) {
session_info = &session_info_;
}
if (OB_ISNULL(coordinator_ctx_ = OB_NEWx(ObTableLoadCoordinatorCtx, (&allocator_), this))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObTableLoadCoordinatorCtx", KR(ret));
} else if (OB_FAIL(coordinator_ctx_->init(session_info, idx_array))) {
} else if (OB_FAIL(coordinator_ctx_->init(idx_array, user_id))) {
LOG_WARN("fail to init coordinator ctx", KR(ret));
} else if (OB_FAIL(coordinator_ctx_->set_status_inited())) {
LOG_WARN("fail to set coordinator status inited", KR(ret));
@ -168,20 +169,7 @@ int ObTableLoadTableCtx::init_coordinator_ctx(const ObIArray<int64_t> &idx_array
return ret;
}
int ObTableLoadTableCtx::init_session_info(uint64_t user_id)
{
int ret = OB_SUCCESS;
if (OB_FAIL(session_info_.init(0, 0, nullptr, nullptr, ObTimeUtility::current_time(), MTL_ID()))) {
LOG_WARN("fail to init session info", KR(ret));
}
OZ (session_info_.load_default_sys_variable(false, false)); //加载默认的session参数
OZ (session_info_.load_default_configs_in_pc());
OX (session_info_.set_priv_user_id(user_id));
return ret;
}
int ObTableLoadTableCtx::init_store_ctx(
int64_t ddl_task_id,
const ObTableLoadArray<ObTableLoadLSIdAndPartitionId> &partition_id_array,
const ObTableLoadArray<ObTableLoadLSIdAndPartitionId> &target_partition_id_array)
{
@ -196,7 +184,7 @@ int ObTableLoadTableCtx::init_store_ctx(
if (OB_ISNULL(store_ctx_ = OB_NEWx(ObTableLoadStoreCtx, (&allocator_), this))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObTableLoadStoreCtx", KR(ret));
} else if (OB_FAIL(store_ctx_->init(ddl_task_id, partition_id_array, target_partition_id_array))) {
} else if (OB_FAIL(store_ctx_->init(partition_id_array, target_partition_id_array))) {
LOG_WARN("fail to init store ctx", KR(ret));
} else if (OB_FAIL(store_ctx_->set_status_inited())) {
LOG_WARN("fail to set store status inited", KR(ret));

15
src/observer/table_load/ob_table_load_table_ctx.h
View File

@ -26,9 +26,9 @@ class ObTableLoadTransCtx;
class ObTableLoadTableCtx : public common::ObDLinkBase<ObTableLoadTableCtx>
{
public:
ObTableLoadTableCtx(const ObTableLoadParam &param);
ObTableLoadTableCtx();
~ObTableLoadTableCtx();
int init();
int init(const ObTableLoadParam &param, const ObTableLoadDDLParam &ddl_param);
void stop();
void destroy();
bool is_valid() const { return is_inited_; }
@ -40,13 +40,10 @@ public:
TO_STRING_KV(K_(param), KP_(coordinator_ctx), KP_(store_ctx), "ref_count", get_ref_count(),
K_(is_dirty), K_(is_inited));
public:
int init_coordinator_ctx(const common::ObIArray<int64_t> &idx_array,
sql::ObSQLSessionInfo *session_info);
int init_coordinator_ctx(const common::ObIArray<int64_t> &idx_array, uint64_t user_id);
int init_store_ctx(
int64_t ddl_task_id,
const table::ObTableLoadArray<table::ObTableLoadLSIdAndPartitionId> &partition_id_array,
const table::ObTableLoadArray<table::ObTableLoadLSIdAndPartitionId> &target_partition_id_array);
int init_session_info(uint64_t user_id);
public:
int alloc_task(ObTableLoadTask *&task);
void free_task(ObTableLoadTask *task);
@ -57,15 +54,15 @@ private:
void unregister_job_stat();
public:
const ObTableLoadParam param_;
// 只在初始化的时候使用, 线程不安全
ObTableLoadParam param_;
ObTableLoadDDLParam ddl_param_;
ObTableLoadSchema schema_;
ObTableLoadCoordinatorCtx *coordinator_ctx_; // 只在控制节点构造
ObTableLoadStoreCtx *store_ctx_; // 只在数据节点构造
sql::ObLoadDataGID gid_;
sql::ObLoadDataStat *job_stat_;
private:
sql::ObSQLSessionInfo session_info_;
// 只在初始化的时候使用, 线程不安全
common::ObArenaAllocator allocator_;
ObTableLoadObjectAllocator<ObTableLoadTask> task_allocator_; // 多线程安全
ObTableLoadObjectAllocator<ObTableLoadTransCtx> trans_ctx_allocator_; // 多线程安全

20
src/observer/table_load/ob_table_load_trans_processor.cpp
View File

@ -26,7 +26,7 @@ int ObTableLoadStartTransP::process()
LOG_WARN("fail to check_user_access", KR(ret));
} else {
ObTableLoadTableCtx *table_ctx = nullptr;
ObTableLoadKey key(credential_.tenant_id_, arg_.table_id_);
ObTableLoadUniqueKey key(arg_.table_id_, arg_.task_id_);
if (OB_FAIL(ObTableLoadService::get_ctx(key, table_ctx))) {
LOG_WARN("fail to get table ctx", KR(ret), K(key));
} else {
@ -64,7 +64,7 @@ int ObTableLoadPreStartTransPeerP::process()
LOG_WARN("fail to check_user_access", KR(ret));
} else {
ObTableLoadTableCtx *table_ctx = nullptr;
ObTableLoadKey key(credential_.tenant_id_, arg_.table_id_);
ObTableLoadUniqueKey key(arg_.table_id_, arg_.task_id_);
if (OB_FAIL(ObTableLoadService::get_ctx(key, table_ctx))) {
LOG_WARN("fail to get table ctx", KR(ret), K(key));
} else {
@ -99,7 +99,7 @@ int ObTableLoadConfirmStartTransPeerP::process()
LOG_WARN("fail to check_user_access", KR(ret));
} else {
ObTableLoadTableCtx *table_ctx = nullptr;
ObTableLoadKey key(credential_.tenant_id_, arg_.table_id_);
ObTableLoadUniqueKey key(arg_.table_id_, arg_.task_id_);
if (OB_FAIL(ObTableLoadService::get_ctx(key, table_ctx))) {
LOG_WARN("fail to get table ctx", KR(ret), K(key));
} else {
@ -136,7 +136,7 @@ int ObTableLoadFinishTransP::process()
LOG_WARN("fail to check_user_access", KR(ret));
} else {
ObTableLoadTableCtx *table_ctx = nullptr;
ObTableLoadKey key(credential_.tenant_id_, arg_.table_id_);
ObTableLoadUniqueKey key(arg_.table_id_, arg_.task_id_);
if (OB_FAIL(ObTableLoadService::get_ctx(key, table_ctx))) {
LOG_WARN("fail to get table ctx", KR(ret), K(key));
} else {
@ -171,7 +171,7 @@ int ObTableLoadPreFinishTransPeerP::process()
LOG_WARN("fail to check_user_access", KR(ret));
} else {
ObTableLoadTableCtx *table_ctx = nullptr;
ObTableLoadKey key(credential_.tenant_id_, arg_.table_id_);
ObTableLoadUniqueKey key(arg_.table_id_, arg_.task_id_);
if (OB_FAIL(ObTableLoadService::get_ctx(key, table_ctx))) {
LOG_WARN("fail to get table ctx", KR(ret), K(key));
} else {
@ -206,7 +206,7 @@ int ObTableLoadConfirmFinishTransPeerP::process()
LOG_WARN("fail to check_user_access", KR(ret));
} else {
ObTableLoadTableCtx *table_ctx = nullptr;
ObTableLoadKey key(credential_.tenant_id_, arg_.table_id_);
ObTableLoadUniqueKey key(arg_.table_id_, arg_.task_id_);
if (OB_FAIL(ObTableLoadService::get_ctx(key, table_ctx))) {
LOG_WARN("fail to get table ctx", KR(ret), K(key));
} else {
@ -243,7 +243,7 @@ int ObTableLoadAbandonTransP::process()
LOG_WARN("fail to check_user_access", KR(ret));
} else {
ObTableLoadTableCtx *table_ctx = nullptr;
ObTableLoadKey key(credential_.tenant_id_, arg_.table_id_);
ObTableLoadUniqueKey key(arg_.table_id_, arg_.task_id_);
if (OB_FAIL(ObTableLoadService::get_ctx(key, table_ctx))) {
LOG_WARN("fail to get table ctx", KR(ret), K(key));
} else {
@ -278,7 +278,7 @@ int ObTableLoadAbandonTransPeerP::process()
LOG_WARN("fail to check_user_access", KR(ret));
} else {
ObTableLoadTableCtx *table_ctx = nullptr;
ObTableLoadKey key(credential_.tenant_id_, arg_.table_id_);
ObTableLoadUniqueKey key(arg_.table_id_, arg_.task_id_);
if (OB_FAIL(ObTableLoadService::get_ctx(key, table_ctx))) {
LOG_WARN("fail to get table ctx", KR(ret), K(key));
} else {
@ -315,7 +315,7 @@ int ObTableLoadGetTransStatusP::process()
LOG_WARN("fail to check_user_access", KR(ret));
} else {
ObTableLoadTableCtx *table_ctx = nullptr;
ObTableLoadKey key(credential_.tenant_id_, arg_.table_id_);
ObTableLoadUniqueKey key(arg_.table_id_, arg_.task_id_);
if (OB_FAIL(ObTableLoadService::get_ctx(key, table_ctx))) {
LOG_WARN("fail to get table ctx", KR(ret), K(key));
} else {
@ -352,7 +352,7 @@ int ObTableLoadGetTransStatusPeerP::process()
LOG_WARN("fail to check_user_access", KR(ret));
} else {
ObTableLoadTableCtx *table_ctx = nullptr;
ObTableLoadKey key(credential_.tenant_id_, arg_.table_id_);
ObTableLoadUniqueKey key(arg_.table_id_, arg_.task_id_);
if (OB_FAIL(ObTableLoadService::get_ctx(key, table_ctx))) {
LOG_WARN("fail to get table ctx", KR(ret), K(key));
} else {

15
src/observer/table_load/ob_table_load_utils.cpp
View File

@ -8,8 +8,9 @@
#include "common/object/ob_object.h"
#include "observer/ob_server.h"
#include "observer/table/ob_table_rpc_processor.h"
#include "storage/blocksstable/ob_datum_row.h"
#include "sql/session/ob_sql_session_info.h"
#include "storage/blocksstable/ob_datum_range.h"
#include "storage/blocksstable/ob_datum_row.h"
namespace oceanbase
{
@ -184,6 +185,18 @@ bool ObTableLoadUtils::is_local_addr(const ObAddr &addr)
return (ObServer::get_instance().get_self() == addr);
}
int ObTableLoadUtils::init_session_info(uint64_t user_id, ObSQLSessionInfo &session_info)
{
int ret = OB_SUCCESS;
if (OB_FAIL(session_info.init(0, 0, nullptr, nullptr, ObTimeUtility::current_time(), MTL_ID()))) {
LOG_WARN("fail to init session info", KR(ret));
}
OZ(session_info.load_default_sys_variable(false, false)); //加载默认的session参数
OZ(session_info.load_default_configs_in_pc());
OX(session_info.set_priv_user_id(user_id));
return ret;
}
int ObTableLoadUtils::generate_credential(uint64_t tenant_id, uint64_t user_id,
uint64_t database_id, int64_t expire_ts,
uint64_t user_token, ObIAllocator &allocator,

7
src/observer/table_load/ob_table_load_utils.h
View File

@ -28,7 +28,11 @@ class ObNewRow;
namespace table
{
class ObTableApiCredential;
}
} // namespace table
namespace sql
{
class ObSQLSessionInfo;
} // namespace sql
namespace observer
{
class ObGlobalContext;
@ -58,6 +62,7 @@ public:
static int deep_copy(const common::ObIArray<T> &src, table::ObTableLoadArray<T> &dest, common::ObIAllocator &allocator);
static bool is_local_addr(const common::ObAddr &addr);
static int init_session_info(uint64_t user_id, sql::ObSQLSessionInfo &session_info);
static const int64_t CREDENTIAL_BUF_SIZE = 256;
static int generate_credential(uint64_t tenant_id, uint64_t user_id, uint64_t database_id,

55
src/share/table/ob_table_load_rpc_struct.cpp
View File

@ -20,10 +20,12 @@ using namespace common;
OB_SERIALIZE_MEMBER(ObTableLoadBeginRequest,
credential_,
table_name_,
config_);
config_,
timeout_);
OB_SERIALIZE_MEMBER(ObTableLoadBeginResult,
table_id_,
task_id_,
column_names_,
status_,
error_code_);
@ -31,22 +33,24 @@ OB_SERIALIZE_MEMBER(ObTableLoadBeginResult,
OB_SERIALIZE_MEMBER(ObTableLoadPreBeginPeerRequest,
credential_,
table_id_,
target_table_id_,
config_,
column_count_,
partition_id_array_,
target_partition_id_array_,
dup_action_,
ddl_task_id_,
px_mode_,
online_opt_stat_gather_);
online_opt_stat_gather_,
dest_table_id_,
task_id_,
schema_version_,
partition_id_array_,
target_partition_id_array_);
OB_SERIALIZE_MEMBER(ObTableLoadPreBeginPeerResult,
ret_code_);
OB_SERIALIZE_MEMBER(ObTableLoadConfirmBeginPeerRequest,
credential_,
table_id_)
table_id_,
task_id_)
OB_SERIALIZE_MEMBER(ObTableLoadConfirmBeginPeerResult,
ret_code_);
@ -57,7 +61,8 @@ OB_SERIALIZE_MEMBER(ObTableLoadConfirmBeginPeerResult,
OB_SERIALIZE_MEMBER(ObTableLoadFinishRequest,
credential_,
table_id_);
table_id_,
task_id_);
OB_SERIALIZE_MEMBER(ObTableLoadFinishResult,
ret_code_);
@ -65,6 +70,7 @@ OB_SERIALIZE_MEMBER(ObTableLoadFinishResult,
OB_SERIALIZE_MEMBER(ObTableLoadPreMergePeerRequest,
credential_,
table_id_,
task_id_,
committed_trans_id_array_);
OB_SERIALIZE_MEMBER(ObTableLoadPreMergePeerResult,
@ -72,7 +78,8 @@ OB_SERIALIZE_MEMBER(ObTableLoadPreMergePeerResult,
OB_SERIALIZE_MEMBER(ObTableLoadStartMergePeerRequest,
credential_,
table_id_);
table_id_,
task_id_);
OB_SERIALIZE_MEMBER(ObTableLoadStartMergePeerResult,
ret_code_);
@ -83,7 +90,8 @@ OB_SERIALIZE_MEMBER(ObTableLoadStartMergePeerResult,
OB_SERIALIZE_MEMBER(ObTableLoadCommitRequest,
credential_,
table_id_);
table_id_,
task_id_);
OB_SERIALIZE_MEMBER(ObTableLoadCommitResult,
ret_code_,
@ -92,7 +100,8 @@ OB_SERIALIZE_MEMBER(ObTableLoadCommitResult,
OB_SERIALIZE_MEMBER(ObTableLoadCommitPeerRequest,
credential_,
table_id_);
table_id_,
task_id_);
OB_SERIALIZE_MEMBER(ObTableLoadCommitPeerResult,
ret_code_,
@ -105,14 +114,16 @@ OB_SERIALIZE_MEMBER(ObTableLoadCommitPeerResult,
OB_SERIALIZE_MEMBER(ObTableLoadAbortRequest,
credential_,
table_id_);
table_id_,
task_id_);
OB_SERIALIZE_MEMBER(ObTableLoadAbortResult,
ret_code_);
OB_SERIALIZE_MEMBER(ObTableLoadAbortPeerRequest,
credential_,
table_id_);
table_id_,
task_id_);
OB_SERIALIZE_MEMBER(ObTableLoadAbortPeerResult,
ret_code_);
@ -123,7 +134,8 @@ OB_SERIALIZE_MEMBER(ObTableLoadAbortPeerResult,
OB_SERIALIZE_MEMBER(ObTableLoadGetStatusRequest,
credential_,
table_id_);
table_id_,
task_id_);
OB_SERIALIZE_MEMBER(ObTableLoadGetStatusResult,
status_,
@ -131,7 +143,8 @@ OB_SERIALIZE_MEMBER(ObTableLoadGetStatusResult,
OB_SERIALIZE_MEMBER(ObTableLoadGetStatusPeerRequest,
credential_,
table_id_);
table_id_,
task_id_);
OB_SERIALIZE_MEMBER(ObTableLoadGetStatusPeerResult,
status_,
@ -144,6 +157,7 @@ OB_SERIALIZE_MEMBER(ObTableLoadGetStatusPeerResult,
OB_SERIALIZE_MEMBER(ObTableLoadRequest,
credential_,
table_id_,
task_id_,
trans_id_,
session_id_,
sequence_no_,
@ -155,6 +169,7 @@ OB_SERIALIZE_MEMBER(ObTableLoadResult,
OB_SERIALIZE_MEMBER(ObTableLoadPeerRequest,
credential_,
table_id_,
task_id_,
trans_id_,
session_id_,
sequence_no_,
@ -170,6 +185,7 @@ OB_SERIALIZE_MEMBER(ObTableLoadPeerResult,
OB_SERIALIZE_MEMBER(ObTableLoadStartTransRequest,
credential_,
table_id_,
task_id_,
segment_id_);
OB_SERIALIZE_MEMBER(ObTableLoadStartTransResult,
@ -180,6 +196,7 @@ OB_SERIALIZE_MEMBER(ObTableLoadStartTransResult,
OB_SERIALIZE_MEMBER(ObTableLoadPreStartTransPeerRequest,
credential_,
table_id_,
task_id_,
trans_id_);
OB_SERIALIZE_MEMBER(ObTableLoadPreStartTransPeerResult,
@ -188,6 +205,7 @@ OB_SERIALIZE_MEMBER(ObTableLoadPreStartTransPeerResult,
OB_SERIALIZE_MEMBER(ObTableLoadConfirmStartTransPeerRequest,
credential_,
table_id_,
task_id_,
trans_id_);
OB_SERIALIZE_MEMBER(ObTableLoadConfirmStartTransPeerResult,
@ -200,6 +218,7 @@ OB_SERIALIZE_MEMBER(ObTableLoadConfirmStartTransPeerResult,
OB_SERIALIZE_MEMBER(ObTableLoadFinishTransRequest,
credential_,
table_id_,
task_id_,
trans_id_);
OB_SERIALIZE_MEMBER(ObTableLoadFinishTransResult,
@ -208,6 +227,7 @@ OB_SERIALIZE_MEMBER(ObTableLoadFinishTransResult,
OB_SERIALIZE_MEMBER(ObTableLoadPreFinishTransPeerRequest,
credential_,
table_id_,
task_id_,
trans_id_);
OB_SERIALIZE_MEMBER(ObTableLoadPreFinishTransPeerResult,
@ -216,6 +236,7 @@ OB_SERIALIZE_MEMBER(ObTableLoadPreFinishTransPeerResult,
OB_SERIALIZE_MEMBER(ObTableLoadConfirmFinishTransPeerRequest,
credential_,
table_id_,
task_id_,
trans_id_);
OB_SERIALIZE_MEMBER(ObTableLoadConfirmFinishTransPeerResult,
@ -228,6 +249,7 @@ OB_SERIALIZE_MEMBER(ObTableLoadConfirmFinishTransPeerResult,
OB_SERIALIZE_MEMBER(ObTableLoadAbandonTransRequest,
credential_,
table_id_,
task_id_,
trans_id_);
OB_SERIALIZE_MEMBER(ObTableLoadAbandonTransResult,
@ -236,6 +258,7 @@ OB_SERIALIZE_MEMBER(ObTableLoadAbandonTransResult,
OB_SERIALIZE_MEMBER(ObTableLoadAbandonTransPeerRequest,
credential_,
table_id_,
task_id_,
trans_id_);
OB_SERIALIZE_MEMBER(ObTableLoadAbandonTransPeerResult,
@ -248,6 +271,7 @@ OB_SERIALIZE_MEMBER(ObTableLoadAbandonTransPeerResult,
OB_SERIALIZE_MEMBER(ObTableLoadGetTransStatusRequest,
credential_,
table_id_,
task_id_,
trans_id_);
OB_SERIALIZE_MEMBER(ObTableLoadGetTransStatusResult,
@ -257,6 +281,7 @@ OB_SERIALIZE_MEMBER(ObTableLoadGetTransStatusResult,
OB_SERIALIZE_MEMBER(ObTableLoadGetTransStatusPeerRequest,
credential_,
table_id_,
task_id_,
trans_id_);
OB_SERIALIZE_MEMBER(ObTableLoadGetTransStatusPeerResult,

208
src/share/table/ob_table_load_rpc_struct.h
View File

@ -26,12 +26,12 @@ class ObTableLoadBeginRequest final
OB_UNIS_VERSION(1);
public:
ObTableLoadBeginRequest() {}
TO_STRING_KV(K_(table_name), K_(config), K_(timeout));
public:
ObString credential_;
ObString table_name_;
ObTableLoadConfig config_;
TO_STRING_KV(K_(table_name), K_(config));
int64_t timeout_;
};
class ObTableLoadBeginResult final
@ -40,13 +40,15 @@ class ObTableLoadBeginResult final
public:
ObTableLoadBeginResult()
: table_id_(common::OB_INVALID_ID),
task_id_(0),
status_(ObTableLoadStatusType::NONE),
error_code_(common::OB_SUCCESS)
{
}
TO_STRING_KV(K_(table_id), K_(column_names), K_(status), K_(error_code));
TO_STRING_KV(K_(table_id), K_(task_id), K_(column_names), K_(status), K_(error_code));
public:
uint64_t table_id_;
int64_t task_id_;
ObTableLoadArray<ObString> column_names_;
ObTableLoadStatusType status_;
int32_t error_code_;
@ -56,28 +58,43 @@ class ObTableLoadPreBeginPeerRequest final
{
OB_UNIS_VERSION(1);
public:
ObTableLoadPreBeginPeerRequest() : table_id_(0), target_table_id_(0), column_count_(0), ddl_task_id_(0), px_mode_(0) {}
ObString credential_;
uint64_t table_id_;
uint64_t target_table_id_;
ObTableLoadConfig config_;
uint64_t column_count_;
ObTableLoadArray<ObTableLoadLSIdAndPartitionId> partition_id_array_;//orig table
ObTableLoadArray<ObTableLoadLSIdAndPartitionId> target_partition_id_array_;//FIXME: target table
sql::ObLoadDupActionType dup_action_;
int64_t ddl_task_id_;
bool px_mode_;
bool online_opt_stat_gather_;
ObTableLoadPreBeginPeerRequest()
: table_id_(common::OB_INVALID_ID),
column_count_(0),
dup_action_(sql::ObLoadDupActionType::LOAD_INVALID_MODE),
px_mode_(false),
online_opt_stat_gather_(false),
dest_table_id_(common::OB_INVALID_ID),
task_id_(0),
schema_version_(0)
{
}
TO_STRING_KV(K_(table_id),
K_(target_table_id),
K_(config),
K_(column_count),
K_(partition_id_array),
K_(target_partition_id_array),
K_(dup_action),
K_(ddl_task_id),
K_(px_mode),
K_(online_opt_stat_gather));
K_(online_opt_stat_gather),
K_(dest_table_id),
K_(task_id),
K_(schema_version),
K_(partition_id_array),
K_(target_partition_id_array));
public:
ObString credential_;
uint64_t table_id_;
ObTableLoadConfig config_;
uint64_t column_count_;
sql::ObLoadDupActionType dup_action_;
bool px_mode_;
bool online_opt_stat_gather_;
// ddl param
uint64_t dest_table_id_;
int64_t task_id_;
int64_t schema_version_;
// partition info
ObTableLoadArray<ObTableLoadLSIdAndPartitionId> partition_id_array_;//orig table
ObTableLoadArray<ObTableLoadLSIdAndPartitionId> target_partition_id_array_;//FIXME: target table
};
class ObTableLoadPreBeginPeerResult final
@ -95,12 +112,12 @@ class ObTableLoadConfirmBeginPeerRequest final
{
OB_UNIS_VERSION(1);
public:
ObTableLoadConfirmBeginPeerRequest() {}
ObTableLoadConfirmBeginPeerRequest() : table_id_(common::OB_INVALID_ID), task_id_(0) {}
TO_STRING_KV(K_(table_id), K_(task_id));
public:
ObString credential_;
uint64_t table_id_;
TO_STRING_KV(K_(table_id));
int64_t task_id_;
};
class ObTableLoadConfirmBeginPeerResult final
@ -122,11 +139,12 @@ class ObTableLoadFinishRequest final
{
OB_UNIS_VERSION(1);
public:
ObTableLoadFinishRequest() : table_id_(common::OB_INVALID_ID) {}
TO_STRING_KV(K_(table_id));
ObTableLoadFinishRequest() : table_id_(common::OB_INVALID_ID), task_id_(0) {}
TO_STRING_KV(K_(table_id), K_(task_id));
public:
ObString credential_;
uint64_t table_id_;
int64_t task_id_;
};
class ObTableLoadFinishResult final
@ -143,11 +161,12 @@ class ObTableLoadPreMergePeerRequest final
{
OB_UNIS_VERSION(1);
public:
ObTableLoadPreMergePeerRequest() : table_id_(common::OB_INVALID_ID) {}
TO_STRING_KV(K_(table_id), K_(committed_trans_id_array));
ObTableLoadPreMergePeerRequest() : table_id_(common::OB_INVALID_ID), task_id_(0) {}
TO_STRING_KV(K_(table_id), K_(task_id), K_(committed_trans_id_array));
public:
ObString credential_;
uint64_t table_id_;
int64_t task_id_;
ObTableLoadArray<ObTableLoadTransId> committed_trans_id_array_;
};
@ -165,11 +184,12 @@ class ObTableLoadStartMergePeerRequest final
{
OB_UNIS_VERSION(1);
public:
ObTableLoadStartMergePeerRequest() : table_id_(common::OB_INVALID_ID) {}
TO_STRING_KV(K_(table_id));
ObTableLoadStartMergePeerRequest() : table_id_(common::OB_INVALID_ID), task_id_(0) {}
TO_STRING_KV(K_(table_id), K_(task_id));
public:
ObString credential_;
uint64_t table_id_;
int64_t task_id_;
};
class ObTableLoadStartMergePeerResult final
@ -190,11 +210,12 @@ class ObTableLoadCommitRequest final
{
OB_UNIS_VERSION(1);
public:
ObTableLoadCommitRequest() : table_id_(common::OB_INVALID_ID) {}
ObTableLoadCommitRequest() : table_id_(common::OB_INVALID_ID), task_id_(0) {}
TO_STRING_KV(K_(table_id));
public:
ObString credential_;
uint64_t table_id_;
int64_t task_id_;
};
class ObTableLoadCommitResult final
@ -213,11 +234,12 @@ class ObTableLoadCommitPeerRequest final
{
OB_UNIS_VERSION(1);
public:
ObTableLoadCommitPeerRequest() : table_id_(common::OB_INVALID_ID) {}
TO_STRING_KV(K_(table_id));
ObTableLoadCommitPeerRequest() : table_id_(common::OB_INVALID_ID), task_id_(0) {}
TO_STRING_KV(K_(table_id), K_(task_id));
public:
ObString credential_;
uint64_t table_id_;
int64_t task_id_;
};
class ObTableLoadCommitPeerResult final
@ -240,12 +262,12 @@ class ObTableLoadAbortRequest final
{
OB_UNIS_VERSION(1);
public:
ObTableLoadAbortRequest() : table_id_(common::OB_INVALID_ID) {}
ObTableLoadAbortRequest() : table_id_(common::OB_INVALID_ID), task_id_(0) {}
TO_STRING_KV(K_(table_id), K_(task_id));
public:
ObString credential_;
uint64_t table_id_;
TO_STRING_KV(K_(table_id));
int64_t task_id_;
};
class ObTableLoadAbortResult final
@ -263,12 +285,12 @@ class ObTableLoadAbortPeerRequest final
{
OB_UNIS_VERSION(1);
public:
ObTableLoadAbortPeerRequest() : table_id_(common::OB_INVALID_ID) {}
ObTableLoadAbortPeerRequest() : table_id_(common::OB_INVALID_ID), task_id_(0) {}
TO_STRING_KV(K_(table_id), K_(task_id));
public:
ObString credential_;
uint64_t table_id_;
TO_STRING_KV(K_(table_id));
int64_t task_id_;
};
class ObTableLoadAbortPeerResult final
@ -290,11 +312,12 @@ class ObTableLoadGetStatusRequest final
{
OB_UNIS_VERSION(1);
public:
ObTableLoadGetStatusRequest() : table_id_(common::OB_INVALID_ID) {}
TO_STRING_KV(K_(table_id));
ObTableLoadGetStatusRequest() : table_id_(common::OB_INVALID_ID), task_id_(0) {}
TO_STRING_KV(K_(table_id), K_(task_id));
public:
ObString credential_;
uint64_t table_id_;
int64_t task_id_;
};
class ObTableLoadGetStatusResult final
@ -315,11 +338,12 @@ class ObTableLoadGetStatusPeerRequest final
{
OB_UNIS_VERSION(1);
public:
ObTableLoadGetStatusPeerRequest() : table_id_(common::OB_INVALID_ID) {}
TO_STRING_KV(K_(table_id));
ObTableLoadGetStatusPeerRequest() : table_id_(common::OB_INVALID_ID), task_id_(0) {}
TO_STRING_KV(K_(table_id), K_(task_id));
public:
ObString credential_;
uint64_t table_id_;
int64_t task_id_;
};
class ObTableLoadGetStatusPeerResult final
@ -345,16 +369,22 @@ class ObTableLoadRequest final
OB_UNIS_VERSION(1);
public:
ObTableLoadRequest()
: table_id_(common::OB_INVALID_ID), session_id_(0), sequence_no_(common::OB_INVALID_ID) {}
: table_id_(common::OB_INVALID_ID),
task_id_(0),
session_id_(0),
sequence_no_(common::OB_INVALID_ID)
{
}
TO_STRING_KV(K_(table_id), K_(task_id), K_(trans_id), K_(session_id), K_(sequence_no),
K(payload_.length()));
public:
ObString credential_; //这个里面会包含tenant_id, database等信息
uint64_t table_id_;
int64_t task_id_;
ObTableLoadTransId trans_id_;
int32_t session_id_; // 从1开始
uint64_t sequence_no_; // 从1开始
ObString payload_; //里面包的是ObTableLoadObjArray / ObTableLoadStrArray / Raw String
TO_STRING_KV(K_(table_id), K_(trans_id), K_(session_id), K_(sequence_no), K(payload_.length()));
};
class ObTableLoadResult final
@ -374,17 +404,18 @@ class ObTableLoadPeerRequest final
public:
ObTableLoadPeerRequest()
: table_id_(common::OB_INVALID_ID),
task_id_(0),
session_id_(0),
sequence_no_(common::OB_INVALID_ID) {}
TO_STRING_KV(K_(table_id), K_(task_id), K_(trans_id), K_(session_id), K_(sequence_no));
public:
ObString credential_; //这个里面会包含tenant_id, database等信息
uint64_t table_id_;
int64_t task_id_;
ObTableLoadTransId trans_id_;
int32_t session_id_; // 从1开始
uint64_t sequence_no_; // 从1开始
ObString payload_; //里面包的是ObTableLoadObjArray
TO_STRING_KV(K_(table_id), K_(trans_id), K_(session_id), K_(sequence_no));
};
class ObTableLoadPeerResult final
@ -406,11 +437,12 @@ class ObTableLoadStartTransRequest final
{
OB_UNIS_VERSION(1);
public:
ObTableLoadStartTransRequest() : table_id_(common::OB_INVALID_ID) {}
TO_STRING_KV(K_(table_id), K_(segment_id));
ObTableLoadStartTransRequest() : table_id_(common::OB_INVALID_ID), task_id_(0) {}
TO_STRING_KV(K_(table_id), K_(task_id), K_(segment_id));
public:
ObString credential_;
uint64_t table_id_;
int64_t task_id_;
ObTableLoadSegmentID segment_id_;
};
@ -433,13 +465,13 @@ class ObTableLoadPreStartTransPeerRequest final
{
OB_UNIS_VERSION(1);
public:
ObTableLoadPreStartTransPeerRequest() : table_id_(common::OB_INVALID_ID) {}
ObTableLoadPreStartTransPeerRequest() : table_id_(common::OB_INVALID_ID), task_id_(0) {}
TO_STRING_KV(K_(table_id), K_(task_id), K_(trans_id));
public:
ObString credential_;
uint64_t table_id_;
int64_t task_id_;
ObTableLoadTransId trans_id_;
TO_STRING_KV(K_(table_id), K_(trans_id));
};
class ObTableLoadPreStartTransPeerResult final
@ -458,13 +490,13 @@ class ObTableLoadConfirmStartTransPeerRequest final
{
OB_UNIS_VERSION(1);
public:
ObTableLoadConfirmStartTransPeerRequest() : table_id_(common::OB_INVALID_ID) {}
ObTableLoadConfirmStartTransPeerRequest() : table_id_(common::OB_INVALID_ID), task_id_(0) {}
TO_STRING_KV(K_(table_id), K_(task_id), K_(trans_id));
public:
ObString credential_;
uint64_t table_id_;
int64_t task_id_;
ObTableLoadTransId trans_id_;
TO_STRING_KV(K_(table_id), K_(trans_id));
};
class ObTableLoadConfirmStartTransPeerResult final
@ -486,13 +518,13 @@ class ObTableLoadFinishTransRequest final
{
OB_UNIS_VERSION(1);
public:
ObTableLoadFinishTransRequest() : table_id_(common::OB_INVALID_ID) {}
ObTableLoadFinishTransRequest() : table_id_(common::OB_INVALID_ID), task_id_(0) {}
TO_STRING_KV(K_(table_id), K_(task_id), K_(trans_id));
public:
ObString credential_;
uint64_t table_id_;
int64_t task_id_;
ObTableLoadTransId trans_id_;
TO_STRING_KV(K_(table_id), K_(trans_id));
};
class ObTableLoadFinishTransResult final
@ -510,13 +542,13 @@ class ObTableLoadPreFinishTransPeerRequest final
{
OB_UNIS_VERSION(1);
public:
ObTableLoadPreFinishTransPeerRequest() : table_id_(common::OB_INVALID_ID) {}
ObTableLoadPreFinishTransPeerRequest() : table_id_(common::OB_INVALID_ID), task_id_(0) {}
TO_STRING_KV(K_(table_id), K_(task_id), K_(trans_id));
public:
ObString credential_;
uint64_t table_id_;
int64_t task_id_;
ObTableLoadTransId trans_id_;
TO_STRING_KV(K_(table_id), K_(trans_id));
};
class ObTableLoadPreFinishTransPeerResult final
@ -535,13 +567,13 @@ class ObTableLoadConfirmFinishTransPeerRequest final
{
OB_UNIS_VERSION(1);
public:
ObTableLoadConfirmFinishTransPeerRequest() : table_id_(common::OB_INVALID_ID) {}
ObTableLoadConfirmFinishTransPeerRequest() : table_id_(common::OB_INVALID_ID), task_id_(0) {}
TO_STRING_KV(K_(table_id), K_(task_id), K_(trans_id));
public:
ObString credential_;
uint64_t table_id_;
int64_t task_id_;
ObTableLoadTransId trans_id_;
TO_STRING_KV(K_(table_id), K_(trans_id));
};
class ObTableLoadConfirmFinishTransPeerResult final
@ -563,13 +595,13 @@ class ObTableLoadAbandonTransRequest final
{
OB_UNIS_VERSION(1);
public:
ObTableLoadAbandonTransRequest() : table_id_(common::OB_INVALID_ID) {}
ObTableLoadAbandonTransRequest() : table_id_(common::OB_INVALID_ID), task_id_(0) {}
TO_STRING_KV(K_(table_id), K_(task_id), K_(trans_id));
public:
ObString credential_;
uint64_t table_id_;
int64_t task_id_;
ObTableLoadTransId trans_id_;
TO_STRING_KV(K_(table_id), K_(trans_id));
};
class ObTableLoadAbandonTransResult final
@ -587,13 +619,13 @@ class ObTableLoadAbandonTransPeerRequest final
{
OB_UNIS_VERSION(1);
public:
ObTableLoadAbandonTransPeerRequest() : table_id_(common::OB_INVALID_ID) {}
ObTableLoadAbandonTransPeerRequest() : table_id_(common::OB_INVALID_ID), task_id_(0) {}
TO_STRING_KV(K_(table_id), K_(task_id), K_(trans_id));
public:
ObString credential_;
uint64_t table_id_;
int64_t task_id_;
ObTableLoadTransId trans_id_;
TO_STRING_KV(K_(table_id), K_(trans_id));
};
class ObTableLoadAbandonTransPeerResult final
@ -615,11 +647,12 @@ class ObTableLoadGetTransStatusRequest final
{
OB_UNIS_VERSION(1);
public:
ObTableLoadGetTransStatusRequest() : table_id_(common::OB_INVALID_ID) {}
TO_STRING_KV(K_(table_id), K_(trans_id));
ObTableLoadGetTransStatusRequest() : table_id_(common::OB_INVALID_ID), task_id_(0) {}
TO_STRING_KV(K_(table_id), K_(task_id), K_(trans_id));
public:
ObString credential_;
uint64_t table_id_;
int64_t task_id_;
ObTableLoadTransId trans_id_;
};
@ -641,11 +674,12 @@ class ObTableLoadGetTransStatusPeerRequest final
{
OB_UNIS_VERSION(1);
public:
ObTableLoadGetTransStatusPeerRequest() : table_id_(common::OB_INVALID_ID) {}
TO_STRING_KV(K_(table_id), K_(trans_id));
ObTableLoadGetTransStatusPeerRequest() : table_id_(common::OB_INVALID_ID), task_id_(0) {}
TO_STRING_KV(K_(table_id), K_(task_id), K_(trans_id));
public:
ObString credential_;
uint64_t table_id_;
int64_t task_id_;
ObTableLoadTransId trans_id_;
};