hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[FEAT MERGE] incremental direct load phase I

Browse Source 
Co-authored-by: Monk-Liu <1152761042@qq.com>
Co-authored-by: suz-yang <suz.yang@foxmail.com>
Co-authored-by: ZenoWang <wzybuaasoft@163.com>
This commit is contained in:
coolfishchen 2024-04-22 09:23:47 +00:00 committed by ob-robot
parent 4d7b31b518
commit 9de65fb1d7
278 changed files with 13417 additions and 6679 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
deps/init/oceanbase.el7.aarch64.deps vendored
View File

@ -42,7 +42,7 @@ obdevtools-gcc9-9.3.0-52022092914.el7.aarch64.rpm
obdevtools-llvm-11.0.1-312022092921.el7.aarch64.rpm
[tools-deps]
devdeps-oblogmsg-1.0-182024030421.el7.aarch64.rpm
devdeps-oblogmsg-1.0-192024032014.el7.aarch64.rpm
devdeps-rocksdb-6.22.1.1-52022100420.el7.aarch64.rpm
obshell-4.2.3.0-102024031414.el7.aarch64.rpm target=community

2
deps/init/oceanbase.el7.x86_64.deps vendored
View File

@ -45,7 +45,7 @@ obdevtools-gcc9-9.3.0-52022092914.el7.x86_64.rpm
obdevtools-llvm-11.0.1-312022092921.el7.x86_64.rpm
[tools-deps]
devdeps-oblogmsg-1.0-182024030421.el7.x86_64.rpm
devdeps-oblogmsg-1.0-192024032014.el7.x86_64.rpm
devdeps-rocksdb-6.22.1.1-52022100420.el7.x86_64.rpm
obshell-4.2.3.0-102024031414.el7.x86_64.rpm target=community

2
deps/init/oceanbase.el8.aarch64.deps vendored
View File

@ -42,7 +42,7 @@ obdevtools-gcc9-9.3.0-52022092914.el8.aarch64.rpm
obdevtools-llvm-11.0.1-312022092921.el8.aarch64.rpm
[tools-deps]
devdeps-oblogmsg-1.0-182024030421.el8.aarch64.rpm
devdeps-oblogmsg-1.0-192024032014.el8.aarch64.rpm
devdeps-rocksdb-6.22.1.1-52022100420.el8.aarch64.rpm
obshell-4.2.3.0-102024031414.el8.aarch64.rpm target=community

2
deps/init/oceanbase.el8.x86_64.deps vendored
View File

@ -44,7 +44,7 @@ obdevtools-gcc9-9.3.0-52022092914.el8.x86_64.rpm
obdevtools-llvm-11.0.1-312022092921.el8.x86_64.rpm
[tools-deps]
devdeps-oblogmsg-1.0-182024030421.el8.x86_64.rpm
devdeps-oblogmsg-1.0-192024032014.el8.x86_64.rpm
devdeps-rocksdb-6.22.1.1-52022100420.el8.x86_64.rpm
obshell-4.2.3.0-102024031414.el8.x86_64.rpm target=community

2
deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h vendored
View File

@ -845,7 +845,7 @@ PCODE_DEF(OB_CLEAR_TABLET_AUTOINC_SEQ_CACHE, 0x962)
PCODE_DEF(OB_CHECK_AND_CANCEL_DDL_COMPLEMENT_DAG, 0x964)
//PCODE_DEF(OB_CLEAN_SPLITTED_TABLET, 0x965)
//PCODE_DEF(OB_REMOTE_WRITE_DDL_FINISH_LOG, 0x966)
//PCODE_DEF(OB_REMOTE_WRITE_DDL_INC_COMMIT_LOG, 0x967)
PCODE_DEF(OB_REMOTE_WRITE_DDL_INC_COMMIT_LOG, 0x967)
//PCODE_DEF(OB_REMOTE_WRITE_DDL_INC_REDO_LOG, 0x968)
// Depedency Detector

2
deps/oblib/src/rpc/obrpc/ob_rpc_proxy.h vendored
View File

@ -166,7 +166,7 @@ public:
|| pcode == OB_CLEAN_SEQUENCE_CACHE || pcode == OB_FETCH_TABLET_AUTOINC_SEQ_CACHE
|| pcode == OB_BATCH_GET_TABLET_AUTOINC_SEQ || pcode == OB_BATCH_SET_TABLET_AUTOINC_SEQ
|| pcode == OB_CALC_COLUMN_CHECKSUM_REQUEST || pcode == OB_REMOTE_WRITE_DDL_REDO_LOG
|| pcode == OB_REMOTE_WRITE_DDL_COMMIT_LOG);
|| pcode == OB_REMOTE_WRITE_DDL_COMMIT_LOG || pcode == OB_REMOTE_WRITE_DDL_INC_COMMIT_LOG);
}
// when active is set as false, all RPC calls will simply return OB_INACTIVE_RPC_PROXY.

4
mittest/logservice/test_ob_simple_log_single_replica_func.cpp
View File

@ -348,7 +348,7 @@ TEST_F(TestObSimpleLogClusterSingleReplica, single_replica_flashback)
EXPECT_GT(leader.palf_handle_impl_->sw_.get_max_scn(), flashback_scn);
leader.palf_handle_impl_->state_mgr_.role_ = FOLLOWER;
leader.palf_handle_impl_->state_mgr_.state_ = ACTIVE;
leader.palf_handle_impl_->state_mgr_.state_ = ObReplicaState::ACTIVE;
EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->flashback(mode_version, max_scn, timeout_ts_us));
EXPECT_LT(leader.palf_handle_impl_->sw_.get_max_scn(), flashback_scn);
@ -356,7 +356,7 @@ TEST_F(TestObSimpleLogClusterSingleReplica, single_replica_flashback)
EXPECT_EQ(new_log_tail, leader.palf_handle_impl_->get_end_lsn());
EXPECT_EQ(OB_ITER_END, read_log(leader));
leader.palf_handle_impl_->state_mgr_.role_ = LEADER;
leader.palf_handle_impl_->state_mgr_.state_ = ACTIVE;
leader.palf_handle_impl_->state_mgr_.state_ = ObReplicaState::ACTIVE;
dynamic_cast<palf::PalfEnvImpl*>(get_cluster()[0]->get_palf_env())->log_loop_thread_.start();
switch_flashback_to_append(leader, mode_version);

7
mittest/mtlenv/storage/checkpoint/test_checkpoint_executor.cpp
View File

@ -71,6 +71,7 @@ void ObTableHandleV2::reset()
}
table_ = nullptr;
t3m_ = nullptr;
table_type_ = ObITable::TableType::MAX_TABLE_TYPE;
}
}
}
@ -120,9 +121,9 @@ public:
bool ready_for_flush() { return ready_for_flush_; }
bool is_frozen_checkpoint() const { return true; }
bool is_frozen_checkpoint() { return true; }
bool is_active_checkpoint() const { return false; }
bool is_active_checkpoint() { return false; }
void set_ready_for_flush(bool ready) { ready_for_flush_ = ready; }
@ -151,7 +152,7 @@ public:
return rec_scn_;
}
int flush(share::SCN recycle_scn, bool need_freeze = true) override {
int flush(share::SCN recycle_scn, const int64_t trace_id, bool need_freeze) override {
share::SCN tmp;
tmp.val_ = rec_scn_.val_ + 20;
rec_scn_ = tmp;

5
mittest/mtlenv/storage/checkpoint/test_data_checkpoint.cpp
View File

@ -67,6 +67,7 @@ void ObTableHandleV2::reset()
}
table_ = nullptr;
t3m_ = nullptr;
table_type_ = ObITable::TableType::MAX_TABLE_TYPE;
}
}
}
@ -107,9 +108,9 @@ public:
bool ready_for_flush() { return ready_for_flush_; }
bool is_frozen_checkpoint() const { return is_frozen_; }
bool is_frozen_checkpoint() { return is_frozen_; }
bool is_active_checkpoint() const { return !is_frozen_; }
bool is_active_checkpoint() { return !is_frozen_; }
ObTabletID get_tablet_id() const { return ObTabletID(111111111111); }

1
mittest/mtlenv/storage/test_co_merge.cpp
View File

@ -16,7 +16,6 @@
#include "lib/container/ob_iarray.h"
#include "storage/column_store/ob_column_oriented_sstable.h"
#include "storage/memtable/ob_memtable_interface.h"
#include "storage/ob_partition_component_factory.h"
#include "storage/blocksstable/ob_data_file_prepare.h"
#include "storage/blocksstable/ob_row_generate.h"
#include "observer/ob_service.h"

2
mittest/mtlenv/storage/test_ls_tablet_service.cpp
View File

@ -1048,7 +1048,7 @@ TEST_F(TestLSTabletService, update_tablet_release_memtable_for_offline)
ObTabletHandle tablet_handle;
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet_svr()->get_tablet(data_tablet_id, tablet_handle));
ASSERT_EQ(0, tablet_handle.get_obj()->memtable_count_);
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet_svr()->create_memtable(data_tablet_id, 100));
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet_svr()->create_memtable(data_tablet_id, 100, false, false));
ASSERT_EQ(1, tablet_handle.get_obj()->memtable_count_);
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet_svr()->update_tablet_release_memtable_for_offline(data_tablet_id, SCN::max_scn()));

10
mittest/mtlenv/storage/test_memtable_v2.cpp
View File

@ -3580,10 +3580,13 @@ int ObTxCtxTable::release_ref_()
return ret;
}
void ObITabletMemtable::unset_logging_blocked_for_active_memtable_()
{
}
} // namespace storage
namespace memtable
{
namespace memtable{
int ObMemtable::lock_row_on_frozen_stores_(
const storage::ObTableIterParam &,
const ObTxNodeArg &,
@ -3599,9 +3602,6 @@ int ObMemtable::lock_row_on_frozen_stores_(
return OB_SUCCESS;
}
}
void ObMemtable::unset_logging_blocked_for_active_memtable()
{
}
}
namespace transaction

1
mittest/mtlenv/storage/test_multi_version_merge_recycle.cpp
View File

@ -14,7 +14,6 @@
#define protected public
#include "lib/container/ob_iarray.h"
#include "storage/memtable/ob_memtable_interface.h"
#include "storage/ob_partition_component_factory.h"
#include "storage/blocksstable/ob_data_file_prepare.h"
#include "storage/blocksstable/ob_row_generate.h"
#include "observer/ob_service.h"

1
mittest/mtlenv/storage/test_multi_version_sstable_merge.cpp
View File

@ -16,7 +16,6 @@
#define UNITTEST
#include "lib/container/ob_iarray.h"
#include "storage/memtable/ob_memtable_interface.h"
#include "storage/ob_partition_component_factory.h"
#include "storage/blocksstable/ob_data_file_prepare.h"
#include "storage/blocksstable/ob_row_generate.h"
#include "observer/ob_service.h"

4
mittest/mtlenv/storage/test_tablet_create_delete_helper.cpp
View File

@ -3142,7 +3142,7 @@ TEST_F(TestTabletCreateDeleteHelper, empty_memtable_replay_commit)
ret = ObTabletCreateDeleteHelper::get_tablet(key, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
ObTableHandleV2 handle;
memtable::ObIMemtable *memtable;
ObIMemtable *memtable;
ret = tablet_handle.get_obj()->get_active_memtable(handle);
ASSERT_EQ(OB_SUCCESS, ret);
ret = handle.get_memtable(memtable);
@ -3158,7 +3158,7 @@ TEST_F(TestTabletCreateDeleteHelper, empty_memtable_replay_commit)
ASSERT_EQ(OB_SUCCESS, ret);
ObTableHandleV2 new_handle;
memtable::ObIMemtable *new_memtable;
ObIMemtable *new_memtable;
ret = tablet_handle.get_obj()->get_active_memtable(new_handle);
ASSERT_EQ(OB_SUCCESS, ret);
ret = new_handle.get_memtable(new_memtable);

42
mittest/mtlenv/storage/test_tenant_meta_mem_mgr.cpp
View File

@ -36,27 +36,25 @@ namespace oceanbase
{
using namespace share;
namespace memtable
namespace memtable {
int ObMemtable::batch_remove_unused_callback_for_uncommited_txn(const ObLSID, const memtable::ObMemtableSet *)
{
int ObMemtable::get_ls_id(share::ObLSID &ls_id)
int ret = OB_SUCCESS;
return ret;
}
} // namespace memtable
namespace storage
{
int ObIMemtable::get_ls_id(share::ObLSID &ls_id)
{
ls_id = share::ObLSID(1001);
return OB_SUCCESS;
}
int ObMemtable::batch_remove_unused_callback_for_uncommited_txn(
const ObLSID , const memtable::ObMemtableSet *)
{
int ret = OB_SUCCESS;
return ret;
}
}
namespace storage
{
int ObTablet::check_and_set_initial_state()
{
return OB_SUCCESS;
@ -435,9 +433,13 @@ TEST_F(TestTenantMetaMemMgr, test_memtable)
int ret = OB_SUCCESS;
ObTableHandleV2 handle;
ret = t3m_.acquire_memtable(handle);
bool for_inc_direct_load = false;
ret = t3m_.acquire_data_memtable(handle);
ASSERT_EQ(common::OB_SUCCESS, ret);
ASSERT_TRUE(nullptr != handle.table_);
handle.get_table()->set_table_type(ObITable::TableType::DATA_MEMTABLE);
ASSERT_EQ(1, t3m_.memtable_pool_.inner_used_num_);
ASSERT_EQ(1, handle.get_table()->get_ref());
@ -467,6 +469,8 @@ TEST_F(TestTenantMetaMemMgr, test_tx_ctx_memtable)
ret = t3m_.acquire_tx_ctx_memtable(handle);
ASSERT_EQ(common::OB_SUCCESS, ret);
ASSERT_TRUE(nullptr != handle.table_);
handle.get_table()->set_table_type(ObITable::TableType::TX_CTX_MEMTABLE);
ASSERT_EQ(1, t3m_.tx_ctx_memtable_pool_.inner_used_num_);
ASSERT_EQ(1, handle.get_table()->get_ref());
@ -496,6 +500,8 @@ TEST_F(TestTenantMetaMemMgr, test_tx_data_memtable)
ret = t3m_.acquire_tx_data_memtable(handle);
ASSERT_EQ(common::OB_SUCCESS, ret);
ASSERT_TRUE(nullptr != handle.table_);
handle.get_table()->set_table_type(ObITable::TableType::TX_DATA_MEMTABLE);
ASSERT_EQ(1, t3m_.tx_data_memtable_pool_.inner_used_num_);
ASSERT_EQ(1, handle.get_table()->get_ref());
@ -525,6 +531,8 @@ TEST_F(TestTenantMetaMemMgr, test_lock_memtable)
ret = t3m_.acquire_lock_memtable(handle);
ASSERT_EQ(common::OB_SUCCESS, ret);
ASSERT_TRUE(nullptr != handle.table_);
handle.get_table()->set_table_type(ObITable::TableType::LOCK_MEMTABLE);
ASSERT_EQ(1, t3m_.lock_memtable_pool_.inner_used_num_);
ASSERT_EQ(1, handle.get_table()->get_ref());
@ -1396,7 +1404,7 @@ TEST_F(TestTenantMetaMemMgr, test_table_gc)
ObTableHandleV2 tx_data_memtable_handle;
ObTableHandleV2 tx_ctx_memtable_handle;
ObTableHandleV2 lock_memtable_handle;
ASSERT_EQ(OB_SUCCESS, t3m_.acquire_memtable(memtable_handle));
ASSERT_EQ(OB_SUCCESS, t3m_.acquire_data_memtable(memtable_handle));
ASSERT_EQ(OB_SUCCESS, t3m_.acquire_tx_data_memtable(tx_data_memtable_handle));
ASSERT_EQ(OB_SUCCESS, t3m_.acquire_tx_ctx_memtable(tx_ctx_memtable_handle));
ASSERT_EQ(OB_SUCCESS, t3m_.acquire_lock_memtable(lock_memtable_handle));

15
mittest/mtlenv/test_tx_data_table.cpp
View File

@ -90,7 +90,6 @@ public:
} else {
tablet_id_ = tablet_id;
t3m_ = t3m;
table_type_ = ObITable::TableType::TX_DATA_MEMTABLE;
freezer_ = freezer;
tx_data_table_ = tx_data_table;
ls_tablet_svr_ = ls_handle.get_ls()->get_tablet_svr();
@ -128,6 +127,11 @@ public:
return ret;
}
void destroy()
{
mgr_.destroy();
}
private:
virtual ObTxDataMemtableMgr *get_memtable_mgr_()
{
@ -349,7 +353,7 @@ void TestTxDataTable::init_memtable_mgr_(ObTxDataMemtableMgr *memtable_mgr)
{
ASSERT_NE(nullptr, memtable_mgr);
memtable_mgr->set_freezer(&tx_data_table_.freezer_);
ASSERT_EQ(OB_SUCCESS, memtable_mgr->create_memtable(SCN::min_scn(), 1, SCN::min_scn()));
ASSERT_EQ(OB_SUCCESS, memtable_mgr->create_memtable(CreateMemtableArg(1, SCN::min_scn(), SCN::min_scn(), false, false)));
ASSERT_EQ(1, memtable_mgr->get_memtable_count_());
}
@ -385,6 +389,7 @@ void TestTxDataTable::do_basic_test()
ObTxDataMemtableMgr *memtable_mgr = tx_data_table_.get_memtable_mgr_();
init_memtable_mgr_(memtable_mgr);
fprintf(stdout, "start insert tx data\n");
insert_tx_data_();
fprintf(stdout, "start insert rollback tx data\n");
@ -445,8 +450,7 @@ void TestTxDataTable::do_basic_test()
}
// free memtable
freezing_memtable->reset();
active_memtable->reset();
memtable_mgr->offline();
}
void TestTxDataTable::do_undo_status_test()
@ -564,6 +568,7 @@ void TestTxDataTable::do_tx_data_serialize_test()
test_serialize_with_action_cnt_(TX_DATA_UNDO_ACT_MAX_NUM_PER_NODE * 100);
test_serialize_with_action_cnt_(TX_DATA_UNDO_ACT_MAX_NUM_PER_NODE * 100 + 1);
test_commit_versions_serialize_();
memtable_mgr->offline();
}
void TestTxDataTable::test_commit_versions_serialize_()
@ -632,7 +637,7 @@ void TestTxDataTable::do_repeat_insert_test() {
ObTxDataMemtableMgr *memtable_mgr = tx_data_table_.get_memtable_mgr_();
ASSERT_NE(nullptr, memtable_mgr);
memtable_mgr->set_freezer(&tx_data_table_.freezer_);
ASSERT_EQ(OB_SUCCESS, memtable_mgr->create_memtable(SCN::min_scn(), 1, SCN::min_scn()));
ASSERT_EQ(OB_SUCCESS, memtable_mgr->create_memtable(CreateMemtableArg(1, SCN::min_scn(), SCN::min_scn(), false, false)));
ASSERT_EQ(1, memtable_mgr->get_memtable_count_());
insert_start_scn.convert_for_logservice(ObTimeUtil::current_time_ns());

1
mittest/multi_replica/CMakeLists.txt
View File

@ -37,3 +37,4 @@ ob_unittest_multi_replica(test_ob_dup_table_tablet_gc)
ob_unittest_multi_replica(test_ob_dup_table_new_gc)
ob_unittest_multi_replica(test_mds_replay_from_ctx_table)
ob_unittest_multi_replica_longer_timeout(test_multi_transfer_tx)
ob_unittest_multi_replica(test_ob_direct_load_inc_log)

191
mittest/multi_replica/test_ob_direct_load_inc_log.cpp Normal file
View File

@ -0,0 +1,191 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#define USING_LOG_PREFIX SERVER
#define protected public
#define private public
#include "env/ob_fast_bootstrap.h"
#include "env/ob_multi_replica_util.h"
#include "lib/mysqlclient/ob_mysql_result.h"
#include "storage/tx/ob_dup_table_lease.h"
using namespace oceanbase::transaction;
using namespace oceanbase::storage;
#define CUR_TEST_CASE_NAME ObDirectLoadIncLogTest
DEFINE_MULTI_ZONE_TEST_CASE_CLASS
MULTI_REPLICA_TEST_MAIN_FUNCTION(test_direct_load_inc_);
#define DEFAULT_LOAD_ROW_CNT 30
namespace oceanbase
{
namespace unittest
{
class ObTestExtraLogCb : public logservice::AppendCb
{
public:
int on_success()
{
int ret = OB_SUCCESS;
TRANS_LOG(INFO, "[ObMultiReplicaTestBase] invoke test on_success", K(ret), KP(this));
return ret;
}
int on_failure()
{
int ret = OB_SUCCESS;
TRANS_LOG(INFO, "[ObMultiReplicaTestBase] invoke test on_failure", K(ret), KP(this));
return ret;
}
};
void submit_ddl_redo_log(transaction::ObPartTransCtx *tx_ctx)
{
ObDDLRedoLog ddl_redo;
for (int i = 0; i < 5; i++) {
ObTestExtraLogCb *test_log_cb =
static_cast<ObTestExtraLogCb *>(ob_malloc(sizeof(ObTestExtraLogCb), "TestExtralCb"));
new (test_log_cb) ObTestExtraLogCb();
int64_t replay_hint = GETTID();
share::SCN scn;
scn.set_max();
ASSERT_EQ(OB_SUCCESS,
tx_ctx->submit_direct_load_inc_redo_log(ddl_redo, test_log_cb, replay_hint, scn));
usleep(1000);
}
}
void submit_ddl_start_log(transaction::ObPartTransCtx *tx_ctx)
{
storage::ObDDLIncStartLog ddl_start;
ddl_start.log_basic_.tablet_id_ = 101;//GETTID();
ObTestExtraLogCb *test_log_cb =
static_cast<ObTestExtraLogCb *>(ob_malloc(sizeof(ObTestExtraLogCb), "TestExtralCb"));
new (test_log_cb) ObTestExtraLogCb();
int64_t replay_hint = 0;
share::SCN scn;
scn.set_max();
ASSERT_EQ(OB_SUCCESS, tx_ctx->submit_direct_load_inc_start_log(ddl_start, test_log_cb, scn));
}
void submit_ddl_end_log(transaction::ObPartTransCtx *tx_ctx)
{
storage::ObDDLIncCommitLog ddl_commit;
ddl_commit.log_basic_.tablet_id_ = 101;//GETTID();
ObTestExtraLogCb *test_log_cb =
static_cast<ObTestExtraLogCb *>(ob_malloc(sizeof(ObTestExtraLogCb), "TestExtralCb"));
new (test_log_cb) ObTestExtraLogCb();
int64_t replay_hint = 0;
share::SCN scn;
scn.set_max();
ASSERT_EQ(OB_SUCCESS, tx_ctx->submit_direct_load_inc_commit_log(ddl_commit, test_log_cb, scn));
}
TEST_F(GET_ZONE_TEST_CLASS_NAME(1), create_table)
{
int ret = OB_SUCCESS;
CREATE_TEST_TENANT(test_tenant_id);
SERVER_LOG(INFO, "[ObMultiReplicaTestBase] create test tenant success", K(test_tenant_id));
common::ObMySQLProxy &test_tenant_sql_proxy = get_curr_simple_server().get_sql_proxy2();
ACQUIRE_CONN_FROM_SQL_PROXY(test_conn, test_tenant_sql_proxy);
WRITE_SQL_BY_CONN(test_conn,
"CREATE TABLE test_t1( "
"id_x int, "
"id_y int, "
"id_z int, "
"PRIMARY KEY(id_x)"
") duplicate_scope='cluster' PARTITION BY hash(id_x) partitions 10;");
std::string primary_zone_sql = "ALTER TENANT " + std::string(DEFAULT_TEST_TENANT_NAME)
+ " set primary_zone='zone1; zone3; zone2';";
WRITE_SQL_BY_CONN(test_conn, primary_zone_sql.c_str());
READ_SQL_BY_CONN(test_conn, table_info_result,
"select table_id, duplicate_scope from "
"oceanbase.__all_table where table_name = 'test_t1' ");
ASSERT_EQ(OB_SUCCESS, table_info_result->next());
int64_t table_id;
int64_t dup_scope;
ASSERT_EQ(OB_SUCCESS, table_info_result->get_int("table_id", table_id));
ASSERT_EQ(OB_SUCCESS, table_info_result->get_int("duplicate_scope", dup_scope));
ASSERT_EQ(true, table_id > 0);
ASSERT_EQ(true, dup_scope != 0);
std::string tablet_count_sql =
"select count(*), ls_id from oceanbase.__all_tablet_to_ls where table_id = "
+ std::to_string(table_id) + " group by ls_id order by count(*)";
READ_SQL_BY_CONN(test_conn, tablet_count_result, tablet_count_sql.c_str());
int64_t tablet_count = 0;
int64_t ls_id_num = 0;
ASSERT_EQ(OB_SUCCESS, tablet_count_result->next());
ASSERT_EQ(OB_SUCCESS, tablet_count_result->get_int("count(*)", tablet_count));
ASSERT_EQ(OB_SUCCESS, tablet_count_result->get_int("ls_id", ls_id_num));
ASSERT_EQ(10, tablet_count);
ASSERT_EQ(true, share::ObLSID(ls_id_num).is_valid());
GET_LS(test_tenant_id, ls_id_num, ls_handle);
WRITE_SQL_BY_CONN(test_conn, "set autocommit = false;");
WRITE_SQL_BY_CONN(test_conn, "begin;");
for (int i = 1; i <= DEFAULT_LOAD_ROW_CNT; i++) {
std::string insert_sql_str = "INSERT INTO test_t1 VALUES(" + std::to_string(i) + ", 0 , 0)";
WRITE_SQL_BY_CONN(test_conn, insert_sql_str.c_str());
}
int64_t tx_id_num;
// GET_RUNNGING_TRX_ID(test_conn, tx_id_num);
std::string template_sql_str = "INSERT INTO test_t1 VALUES(" + std::to_string(1) + ", 0 , 0)";
GET_TX_ID_FROM_SQL_AUDIT(test_conn, template_sql_str.c_str(), tx_id_num);
transaction::ObPartTransCtx *tx_ctx = nullptr;
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tx_ctx(ObTransID(tx_id_num), false, tx_ctx));
TRANS_LOG(INFO, "[ObMultiReplicaTestBase] get target tx ctx", K(ret), K(tx_id_num), KPC(tx_ctx));
submit_ddl_start_log(tx_ctx);
std::thread t1(submit_ddl_redo_log, tx_ctx);
std::thread t2(submit_ddl_redo_log, tx_ctx);
std::thread t3(submit_ddl_redo_log, tx_ctx);
std::thread t4(submit_ddl_redo_log, tx_ctx);
t1.join();
t2.join();
t3.join();
t4.join();
// submit_ddl_end_log(tx_ctx);
WRITE_SQL_BY_CONN(test_conn, "commit;");
// Don't free log cb in thread
}
} // namespace unittest
} // namespace oceanbase

1
mittest/simple_server/CMakeLists.txt
View File

@ -105,6 +105,7 @@ ob_unittest_observer(test_ls_replica test_ls_replica.cpp)
ob_unittest_observer(test_create_clone_tenant_resource_pool test_create_clone_tenant_resource_pool.cpp)
ob_unittest_observer(test_ob_admin_arg test_ob_admin_arg.cpp)
ob_unittest_observer(test_tablet_autoinc_mgr test_tablet_autoinc_mgr.cpp)
ob_unittest_observer(test_tablet_memtable_mit test_tablet_memtable_mit.cpp)
ob_unittest_observer(test_tenant_snapshot_service test_tenant_snapshot_service.cpp)
ob_unittest_observer(test_callbacks_with_reverse_order test_callbacks_with_reverse_order.cpp)
ob_unittest_observer(test_transfer_tx_data test_transfer_with_smaller_tx_data.cpp)

6
mittest/simple_server/test_memtable_new_safe_to_destroy.cpp
View File

@ -194,14 +194,12 @@ TEST_F(ObTestMemtableNewSafeToDestroy, test_safe_to_destroy)
WRITE_SQL_BY_CONN(user_connection, "begin;");
WRITE_SQL_FMT_BY_CONN(user_connection, "insert into qcc values(1);");
ASSERT_EQ(0, SSH::find_tx(user_connection, qcc_tx_id));
ObTableHandleV2 handle;
ObTabletID tablet_id;
ASSERT_EQ(0, SSH::select_table_tablet(tenant_id, "qcc", tablet_id));
get_memtable(tenant_id, share::ObLSID(1001), tablet_id, handle);
memtable::ObIMemtable *imemtable;
ObIMemtable *imemtable;
handle.get_memtable(imemtable);
memtable::ObMemtable *memtable = dynamic_cast<memtable::ObMemtable *>(imemtable);
TRANS_LOG(INFO, "qcc print", KPC(memtable));
@ -221,7 +219,7 @@ TEST_F(ObTestMemtableNewSafeToDestroy, test_safe_to_destroy)
handle.reset();
storage::ObTabletMemtableMgr *memtable_mgr = memtable->get_memtable_mgr_();
storage::ObTabletMemtableMgr *memtable_mgr = memtable->get_memtable_mgr();
EXPECT_EQ(OB_SUCCESS, memtable_mgr->release_memtables());
TRANS_LOG(INFO, "qcc print2", KPC(memtable));;

35
mittest/simple_server/test_ob_minor_freeze.cpp
View File

@ -37,15 +37,15 @@
namespace oceanbase
{
const int SLEEP_TIME = 100;
namespace memtable
namespace storage
{
int64_t ObMemtable::inc_write_ref_()
int64_t ObITabletMemtable::inc_write_ref_()
{
ob_usleep(rand() % SLEEP_TIME);
return ATOMIC_AAF(&write_ref_cnt_, 1);
}
int64_t ObMemtable::dec_write_ref_()
int64_t ObITabletMemtable::dec_write_ref_()
{
ob_usleep(rand() % SLEEP_TIME);
int64_t write_ref_cnt = ATOMIC_SAF(&write_ref_cnt_, 1);
@ -55,13 +55,13 @@ int64_t ObMemtable::dec_write_ref_()
return write_ref_cnt;
}
int64_t ObMemtable::inc_unsubmitted_cnt_()
int64_t ObITabletMemtable::inc_unsubmitted_cnt_()
{
ob_usleep(rand() % SLEEP_TIME);
return ATOMIC_AAF(&unsubmitted_cnt_, 1);
}
int64_t ObMemtable::dec_unsubmitted_cnt_()
int64_t ObITabletMemtable::dec_unsubmitted_cnt_()
{
ob_usleep(rand() % SLEEP_TIME);
int64_t unsubmitted_cnt = ATOMIC_SAF(&unsubmitted_cnt_, 1);
@ -129,7 +129,6 @@ public:
void tenant_freeze();
void logstream_freeze();
void tablet_freeze();
void tablet_freeze_for_replace_tablet_meta();
void batch_tablet_freeze();
void insert_and_freeze();
void empty_memtable_flush();
@ -374,28 +373,6 @@ void ObMinorFreezeTest::tablet_freeze()
}
}
void ObMinorFreezeTest::tablet_freeze_for_replace_tablet_meta()
{
common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2();
share::ObTenantSwitchGuard tenant_guard;
OB_LOG(INFO, "tenant_id", K(RunCtx.tenant_id_));
ASSERT_EQ(OB_SUCCESS, tenant_guard.switch_to(RunCtx.tenant_id_));
const int64_t start = ObTimeUtility::current_time();
while (ObTimeUtility::current_time() - start <= freeze_duration_) {
for (int j = 0; j < OB_DEFAULT_TABLE_COUNT; ++j) {
ObTableHandleV2 handle;
int ret = ls_handles_.at(j).get_ls()->get_freezer()->tablet_freeze_for_replace_tablet_meta(tablet_ids_.at(j), handle);
if (OB_EAGAIN == ret || OB_ENTRY_EXIST == ret) {
ret = OB_SUCCESS;
}
ASSERT_EQ(OB_SUCCESS, ret);
if (OB_SUCC(ret)) {
ASSERT_EQ(OB_SUCCESS, ls_handles_.at(j).get_ls()->get_freezer()->handle_frozen_memtable_for_replace_tablet_meta(tablet_ids_.at(j), handle));
}
}
}
}
void ObMinorFreezeTest::batch_tablet_freeze()
{
common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2();
@ -416,7 +393,6 @@ void ObMinorFreezeTest::insert_and_freeze()
std::thread tenant_freeze_thread([this]() { tenant_freeze(); });
std::thread tablet_freeze_thread([this]() { tablet_freeze(); });
std::thread logstream_freeze_thread([this]() { logstream_freeze(); });
std::thread tablet_freeze_for_replace_tablet_meta_thread([this]() { tablet_freeze_for_replace_tablet_meta(); });
std::thread check_frozen_memtable_thread([this]() { check_frozen_memtable(); });
std::thread batch_tablet_freeze_thread([this]() { batch_tablet_freeze(); });
std::vector<std::thread> insert_threads;
@ -428,7 +404,6 @@ void ObMinorFreezeTest::insert_and_freeze()
tenant_freeze_thread.join();
tablet_freeze_thread.join();
logstream_freeze_thread.join();
tablet_freeze_for_replace_tablet_meta_thread.join();
check_frozen_memtable_thread.join();
batch_tablet_freeze_thread.join();
for (int i = 0; i < insert_thread_num_; ++i) {

360
mittest/simple_server/test_tablet_memtable_mit.cpp Normal file
View File

@ -0,0 +1,360 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#define protected public
#define private public
#include "env/ob_simple_cluster_test_base.h"
#include "env/ob_simple_server_restart_helper.h"
#include "storage/ddl/ob_tablet_ddl_kv.h"
#include "storage/tx_storage/ob_ls_service.h"
#include "storage/ob_direct_load_table_guard.h"
#undef private
#undef protected
static const char *TEST_FILE_NAME = "test_tablet_memtable_mit";
static const char *BORN_CASE_NAME = "TestTabletMemtable";
static const char *RESTART_CASE_NAME = "TestTabletMemtableRestart";
ObSimpleServerRestartHelper *helper_ptr = nullptr;
namespace oceanbase {
using namespace observer;
using namespace memtable;
using namespace storage;
using namespace share;
class TestRunCtx
{
public:
uint64_t tenant_id_ = 0;
};
TestRunCtx RunCtx;
ObLSID LS_ID;
ObTabletID TABLET_ID;
namespace storage {
int ObDDLKV::flush(ObLSID ls_id)
{
int ret = OB_SUCCESS;
if (!ready_for_flush()) {
return OB_ERR_UNEXPECTED;
} else {
ObTabletMemtableMgr *mgr = get_memtable_mgr();
ObTableHandleV2 handle;
EXPECT_EQ(OB_SUCCESS, mgr->get_first_frozen_memtable(handle));
if (handle.get_table() == this) {
mgr->release_head_memtable_(this);
STORAGE_LOG(INFO, "flush and release one ddl kv ddl_kv", KP(this));
}
}
return ret;
}
}
namespace unittest {
class TestTabletMemtable : public ObSimpleClusterTestBase {
public:
TestTabletMemtable() : ObSimpleClusterTestBase(TEST_FILE_NAME) {}
~TestTabletMemtable() {}
void basic_test();
void create_test_table(const char* table_name);
private:
common::ObArenaAllocator allocator_; // for medium info
};
#define EXE_SQL(sql_str) \
ASSERT_EQ(OB_SUCCESS, sql.assign(sql_str)); \
ASSERT_EQ(OB_SUCCESS, sql_proxy.write(sql.ptr(), affected_rows));
#define EXE_SQL_FMT(...) \
ASSERT_EQ(OB_SUCCESS, sql.assign_fmt(__VA_ARGS__)); \
ASSERT_EQ(OB_SUCCESS, sql_proxy.write(sql.ptr(), affected_rows));
#define WRITE_SQL_BY_CONN(conn, sql_str) \
ASSERT_EQ(OB_SUCCESS, sql.assign(sql_str)); \
ASSERT_EQ(OB_SUCCESS, conn->execute_write(OB_SYS_TENANT_ID, sql.ptr(), affected_rows));
#define WRITE_SQL_FMT_BY_CONN(conn, ...) \
ASSERT_EQ(OB_SUCCESS, sql.assign_fmt(__VA_ARGS__)); \
ASSERT_EQ(OB_SUCCESS, conn->execute_write(OB_SYS_TENANT_ID, sql.ptr(), affected_rows));
#define READ_SQL_FMT_BY_CONN(conn, res, ...) \
ASSERT_EQ(OB_SUCCESS, sql.assign_fmt(__VA_ARGS__)); \
ASSERT_EQ(OB_SUCCESS, conn->execute_read(OB_SYS_TENANT_ID, sql.ptr(), res));
#define DEF_VAL_FOR_SQL \
int ret = OB_SUCCESS; \
ObSqlString sql; \
int64_t affected_rows = 0; \
sqlclient::ObISQLConnection *connection = nullptr;
void TestTabletMemtable::create_test_table(const char* table_name)
{
DEF_VAL_FOR_SQL
ObString create_sql;
ASSERT_EQ(OB_SUCCESS, get_curr_simple_server().get_sql_proxy2().acquire(connection));
WRITE_SQL_FMT_BY_CONN(connection, "create table if not exists %s (c1 int, c2 int)", table_name);
HEAP_VAR(ObMySQLProxy::MySQLResult, res)
{
int64_t int_tablet_id = 0;
int64_t int_ls_id = 0;
int64_t retry_times = 10;
READ_SQL_FMT_BY_CONN(connection,
res,
"SELECT tablet_id, ls_id FROM oceanbase.DBA_OB_TABLE_LOCATIONS WHERE TABLE_NAME = \'%s\'",
table_name);
common::sqlclient::ObMySQLResult *result = res.mysql_result();
ASSERT_EQ(OB_SUCCESS, result->next());
ASSERT_EQ(OB_SUCCESS, result->get_int("tablet_id", int_tablet_id));
ASSERT_EQ(OB_SUCCESS, result->get_int("ls_id", int_ls_id));
TABLET_ID = int_tablet_id;
LS_ID = int_ls_id;
fprintf(stdout, "get table info finish : tablet_id = %ld, ls_id = %ld\n", TABLET_ID.id(), LS_ID.id());
}
}
void TestTabletMemtable::basic_test() {
share::ObTenantSwitchGuard tenant_guard;
ASSERT_EQ(OB_SUCCESS, tenant_guard.switch_to(RunCtx.tenant_id_));
create_test_table("test_tablet_memtable");
ObLSService *ls_svr = MTL(ObLSService *);
ObLSHandle ls_handle;
ASSERT_EQ(1001, LS_ID.id());
ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(LS_ID, ls_handle, ObLSGetMod::STORAGE_MOD));
ObLS *ls = nullptr;
ObLSIterator *ls_iter = nullptr;
ASSERT_NE(nullptr, ls = ls_handle.get_ls());
ObTabletHandle tablet_handle;
ObTablet *tablet = nullptr;
ASSERT_EQ(OB_SUCCESS, ls->get_tablet(TABLET_ID, tablet_handle));
ASSERT_NE(nullptr, tablet = tablet_handle.get_obj());
// *********** CREATE DATA METMABLE ************
ASSERT_EQ(OB_SUCCESS, tablet->create_memtable(1, SCN::min_scn(), false /*for_direct_load*/, false /*for_replay*/));
ObProtectedMemtableMgrHandle *protected_handle = nullptr;
ASSERT_EQ(OB_SUCCESS, tablet->get_protected_memtable_mgr_handle(protected_handle));
ASSERT_NE(nullptr, protected_handle);
ASSERT_EQ(true, protected_handle->has_memtable());
// *********** CHECK METMABLE TYPE ************
ObTableHandleV2 memtable_handle;
ObITabletMemtable *memtable = nullptr;
ASSERT_EQ(OB_SUCCESS, protected_handle->get_active_memtable(memtable_handle));
ASSERT_EQ(OB_SUCCESS, memtable_handle.get_tablet_memtable(memtable));
ASSERT_EQ(ObITable::TableType::DATA_MEMTABLE, memtable->get_table_type());
// *********** DO TABLET FREEZE ************
ObFreezer *freezer = nullptr;
ASSERT_NE(nullptr, freezer = ls->get_freezer());
ASSERT_EQ(OB_SUCCESS, ls->tablet_freeze(TABLET_ID, true /* is_sync */));
ASSERT_EQ(OB_ENTRY_NOT_EXIST, protected_handle->get_active_memtable(memtable_handle));
ASSERT_EQ(OB_SUCCESS, protected_handle->get_boundary_memtable(memtable_handle));
ASSERT_EQ(OB_SUCCESS, memtable_handle.get_tablet_memtable(memtable));
STORAGE_LOG(INFO, "finish freeze data memtable", KPC(memtable));
// *********** CREATE DIRECT LOAD MEMTABLE ************
ASSERT_EQ(OB_SUCCESS, tablet->create_memtable(1, SCN::min_scn(), true /*for_direct_load*/, false /*for_replay*/));
ASSERT_EQ(OB_SUCCESS, protected_handle->get_active_memtable(memtable_handle));
ASSERT_EQ(OB_SUCCESS, memtable_handle.get_tablet_memtable(memtable));
ASSERT_EQ(ObITable::TableType::DIRECT_LOAD_MEMTABLE, memtable->get_table_type());
STORAGE_LOG(INFO, "finish create direct load memtable", KPC(memtable));
// *********** GET DIRECT LOAD MEMTABLE FOR WRITE ************
memtable->inc_write_ref();
// *********** CONCURRENT TABLET FREEZE ************
ObFuture<int> result;
int64_t freeze_start_time = ObClockGenerator::getClock();
ASSERT_EQ(OB_SUCCESS, ls->tablet_freeze_with_rewrite_meta(TABLET_ID, &result));
sleep(2);
ASSERT_EQ(TabletMemtableFreezeState::ACTIVE, memtable->get_freeze_state());
STORAGE_LOG(INFO, "waiting write finish", KPC(memtable));
// *********** DIRECT LOAD MEMTABLE WRITE FINISH ************
memtable->dec_write_ref();
ASSERT_EQ(OB_SUCCESS, ls->wait_freeze_finished(result));
STORAGE_LOG(INFO, "write_ref_cnt should be zero", KPC(memtable));
// *********** CHECK FREEZE RESULT ************
ASSERT_EQ(OB_ENTRY_NOT_EXIST, protected_handle->get_active_memtable(memtable_handle));
ASSERT_EQ(OB_SUCCESS, protected_handle->get_boundary_memtable(memtable_handle));
ASSERT_EQ(OB_SUCCESS, memtable_handle.get_tablet_memtable(memtable));
STORAGE_LOG(INFO, "get boundary memtable", KPC(memtable));
ASSERT_EQ(ObITable::TableType::DIRECT_LOAD_MEMTABLE, memtable->get_table_type());
ASSERT_NE(TabletMemtableFreezeState::ACTIVE, memtable->get_freeze_state());
ASSERT_EQ(true, memtable->is_in_prepare_list_of_data_checkpoint());
// *********** CREATE ANOTHER DIRECT LOAD MEMTABLE ************
ASSERT_EQ(OB_SUCCESS, tablet->create_memtable(1, SCN::min_scn(), true /*for_direct_load*/, false /*for_replay*/));
ASSERT_EQ(OB_SUCCESS, protected_handle->get_active_memtable(memtable_handle));
ASSERT_EQ(OB_SUCCESS, memtable_handle.get_tablet_memtable(memtable));
STORAGE_LOG(INFO, "create a new direct load memtable", KPC(memtable));
// *********** CONSTURCT A DIRECT LOAD TABLE GUARD ************
SCN fake_ddl_redo_scn = SCN::plus(SCN::min_scn(), 10);
ObDirectLoadTableGuard direct_load_guard(*tablet, fake_ddl_redo_scn, false);
// *********** CHECK DIRECT LOAD TABLE GUARD UASBLE ************
ObDDLKV *memtable_for_direct_load = nullptr;
ASSERT_EQ(OB_SUCCESS, direct_load_guard.prepare_memtable(memtable_for_direct_load));
ASSERT_EQ(OB_ERR_UNEXPECTED, direct_load_guard.prepare_memtable(memtable_for_direct_load));
ASSERT_EQ(OB_SUCCESS, memtable_for_direct_load->set_rec_scn(fake_ddl_redo_scn));
ASSERT_EQ(memtable, memtable_for_direct_load);
ASSERT_EQ(1, memtable_for_direct_load->get_write_ref());
direct_load_guard.reset();
ASSERT_EQ(0, memtable_for_direct_load->get_write_ref());
// *********** DO LOGSTREAM FREEZE ************
ASSERT_EQ(OB_SUCCESS, ls->logstream_freeze(true /* is_sync */));
STORAGE_LOG(INFO, "finish logstream freeze");
// *********** CHECK LOGSTREAM FREEZE RESULT ************
ASSERT_EQ(OB_ENTRY_NOT_EXIST, protected_handle->get_active_memtable(memtable_handle));
ASSERT_EQ(OB_SUCCESS, protected_handle->get_boundary_memtable(memtable_handle));
ASSERT_EQ(OB_SUCCESS, memtable_handle.get_tablet_memtable(memtable));
ASSERT_EQ(ObITable::TableType::DIRECT_LOAD_MEMTABLE, memtable->get_table_type());
ASSERT_NE(TabletMemtableFreezeState::ACTIVE, memtable->get_freeze_state());
ASSERT_EQ(true, memtable->is_in_prepare_list_of_data_checkpoint());
STORAGE_LOG(INFO, "finish check logstream freeze result", KPC(memtable));
// *********** WAIT ALL MEMTABLES FLUSH ************
int64_t retry_times = 0;
while (retry_times <= 10) {
sleep(1);
if (protected_handle->has_memtable()) {
ObSEArray<ObTableHandleV2, 2> handles;
ASSERT_EQ(OB_SUCCESS, protected_handle->get_all_memtables(handles));
STORAGE_LOG(INFO, "wait all memtable flushed", K(retry_times));
for (int i = 0; i < handles.count(); i++) {
ObITabletMemtable *memtable = static_cast<ObITabletMemtable*>(handles.at(i).get_table());
STORAGE_LOG(INFO, "PRINT Table", K(memtable->key_), K(memtable->get_freeze_state()));
}
retry_times++;
} else {
break;
}
}
ASSERT_FALSE(protected_handle->has_memtable());
fprintf(stdout, "finish basic test\n");
}
TEST_F(TestTabletMemtable, observer_start) { SERVER_LOG(INFO, "observer_start succ"); }
TEST_F(TestTabletMemtable, add_tenant)
{
// create tenant
ASSERT_EQ(OB_SUCCESS, create_tenant());
ASSERT_EQ(OB_SUCCESS, get_tenant_id(RunCtx.tenant_id_));
ASSERT_NE(0, RunCtx.tenant_id_);
ASSERT_EQ(OB_SUCCESS, get_curr_simple_server().init_sql_proxy2());
}
TEST_F(TestTabletMemtable, init_config)
{
DEF_VAL_FOR_SQL
common::ObMySQLProxy &sys_proxy = get_curr_simple_server().get_sql_proxy();
sqlclient::ObISQLConnection *sys_conn = nullptr;
ASSERT_EQ(OB_SUCCESS, sys_proxy.acquire(sys_conn));
ASSERT_NE(nullptr, sys_conn);
sleep(2);
}
TEST_F(TestTabletMemtable, create_table)
{
common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2();
OB_LOG(INFO, "create_table start");
ObSqlString sql;
int64_t affected_rows = 0;
EXE_SQL("create table if not exists test_tablet_memtable (c1 int, c2 int, primary key(c1))");
OB_LOG(INFO, "create_table succ");
}
TEST_F(TestTabletMemtable, basic_test)
{
basic_test();
}
class TestTabletMemtableRestart : public ObSimpleClusterTestBase
{
public:
TestTabletMemtableRestart() : ObSimpleClusterTestBase(TEST_FILE_NAME) {}
private:
};
TEST_F(TestTabletMemtableRestart, observer_restart)
{
SERVER_LOG(INFO, "observer restart succ");
}
} // namespace unittest
} // namespace oceanbase
int main(int argc, char **argv)
{
int c = 0;
int time_sec = 0;
int concurrency = 1;
char *log_level = (char *)"DEBUG";
while (EOF != (c = getopt(argc, argv, "t:l:"))) {
switch (c) {
case 't':
time_sec = atoi(optarg);
break;
case 'l':
log_level = optarg;
oceanbase::unittest::ObSimpleClusterTestBase::enable_env_warn_log_ = false;
break;
case 'c':
concurrency = atoi(optarg);
break;
default:
break;
}
}
std::string gtest_file_name = std::string(TEST_FILE_NAME) + "_gtest.log";
oceanbase::unittest::init_gtest_output(gtest_file_name);
int ret = 0;
ObSimpleServerRestartHelper restart_helper(argc, argv, TEST_FILE_NAME, BORN_CASE_NAME,
RESTART_CASE_NAME);
helper_ptr = &restart_helper;
restart_helper.set_sleep_sec(time_sec);
restart_helper.run();
return ret;
}

2
mittest/simple_server/test_transfer_in_after_abort.cpp
View File

@ -221,7 +221,7 @@ public:
void check_memtable_cleanout(ObTableHandleV2 &handle,
const bool memtable_is_all_delay_cleanout)
{
memtable::ObIMemtable *imemtable;
ObIMemtable *imemtable;
handle.get_memtable(imemtable);
memtable::ObMemtable *memtable = dynamic_cast<memtable::ObMemtable *>(imemtable);

3
mittest/simple_server/test_tx_data_table_mit.cpp
View File

@ -375,7 +375,8 @@ void ObTxDataTableTest::check_minor_merge()
sleep(1);
}
}
ASSERT_GT(cnt, 0);
// ASSERT_GT(cnt, 0);
// 确认没有未能转储的memtable
retry_times = 10;

8
src/logservice/libobcdc/src/ob_cdc_lob_ctx.cpp
View File

@ -75,7 +75,7 @@ void ObLobDataGetCtx::reset()
type_ = ObLobDataGetTaskType::FULL_LOB;
host_ = nullptr;
column_id_ = common::OB_INVALID_ID;
dml_flag_ = blocksstable::ObDmlFlag::DF_MAX;
dml_flag_.reset();
new_lob_data_ = nullptr;
old_lob_data_ = nullptr;
lob_col_value_handle_done_count_ = 0;
@ -87,7 +87,7 @@ void ObLobDataGetCtx::reset()
void ObLobDataGetCtx::reset(
void *host,
const uint64_t column_id,
const blocksstable::ObDmlFlag &dml_flag,
const blocksstable::ObDmlRowFlag &dml_flag,
const common::ObLobData *new_lob_data)
{
host_ = host;
@ -222,8 +222,8 @@ int64_t ObLobDataGetCtx::to_string(char *buf, const int64_t buf_len) const
}
(void)common::databuff_printf(buf, buf_len, pos,
"column_id=%ld, dml=%s, ref_cnt[new=%d, old=%d], handle_cnt=%d, type=%d, ",
column_id_, print_dml_flag(dml_flag_), new_lob_col_ctx_.get_col_ref_cnt(),
"column_id=%ld, dml_flag=%s, dml_serialize_flag=%d, ref_cnt[new=%d, old=%d], handle_cnt=%d, type=%d, ",
column_id_, dml_flag_.getFlagStr(), dml_flag_.get_serialize_flag(), new_lob_col_ctx_.get_col_ref_cnt(),
old_lob_col_ctx_.get_col_ref_cnt(), lob_col_value_handle_done_count_, type_);
if (nullptr != new_lob_data_) {

18
src/logservice/libobcdc/src/ob_cdc_lob_ctx.h
View File

@ -13,12 +13,12 @@
#ifndef OCEANBASE_LIBOBCDC_LOB_CTX_H_
#define OCEANBASE_LIBOBCDC_LOB_CTX_H_
#include "common/object/ob_object.h" // ObLobData, ObLobDataOutRowCtx
#include "storage/blocksstable/ob_datum_row.h" // ObDmlFlag
#include "lib/atomic/ob_atomic.h" // ATOMIC_**
#include "common/object/ob_object.h" // ObLobData, ObLobDataOutRowCtx
#include "storage/blocksstable/ob_datum_row.h" // ObDmlRowFlag
#include "lib/atomic/ob_atomic.h" // ATOMIC_**
#include "lib/allocator/ob_allocator.h"
#include "lib/ob_define.h"
#include "ob_log_lighty_list.h" // LightyList
#include "ob_log_lighty_list.h" // LightyList
namespace oceanbase
{
@ -75,13 +75,13 @@ struct ObLobDataGetCtx
void reset(
void *host,
const uint64_t column_id,
const blocksstable::ObDmlFlag &dml_flag,
const blocksstable::ObDmlRowFlag &dml_flag,
const common::ObLobData *new_lob_data);
void set_old_lob_data(const common::ObLobData *old_lob_data) { old_lob_data_ = old_lob_data; }
bool is_insert() const { return blocksstable::ObDmlFlag::DF_INSERT == dml_flag_; }
bool is_update() const { return blocksstable::ObDmlFlag::DF_UPDATE == dml_flag_; }
bool is_delete() const { return blocksstable::ObDmlFlag::DF_DELETE == dml_flag_; }
bool is_insert() const { return dml_flag_.is_insert(); }
bool is_update() const { return dml_flag_.is_update(); }
bool is_delete() const { return dml_flag_.is_delete(); }
bool is_ext_info_log() const { return ObLobDataGetTaskType::EXT_INFO_LOG == type_; }
const common::ObLobData *get_lob_data(const bool is_new_col) const
@ -143,7 +143,7 @@ struct ObLobDataGetCtx
ObLobDataGetTaskType type_;
void *host_; // ObLobDataOutRowCtxList
uint64_t column_id_;
blocksstable::ObDmlFlag dml_flag_;
blocksstable::ObDmlRowFlag dml_flag_;
const common::ObLobData *new_lob_data_;
const common::ObLobData *old_lob_data_;
int8_t lob_col_value_handle_done_count_;

72
src/logservice/libobcdc/src/ob_cdc_part_trans_resolver.cpp
View File

@ -17,6 +17,7 @@
#include "ob_cdc_part_trans_resolver.h"
#include "ob_log_cluster_id_filter.h" // ClusterIdFilter
#include "logservice/logfetcher/ob_log_part_serve_info.h" // PartServeInfo
#include "ob_log_config.h" // TCONF
namespace oceanbase
{
@ -141,7 +142,8 @@ ObCDCPartTransResolver::ObCDCPartTransResolver(
offlined_(false),
tls_id_(),
part_trans_dispatcher_(tls_id_str, task_pool, task_map, dispatcher),
cluster_id_filter_(cluster_id_filter)
cluster_id_filter_(cluster_id_filter),
enable_direct_load_inc_(false)
{}
ObCDCPartTransResolver::~ObCDCPartTransResolver()
@ -150,9 +152,11 @@ ObCDCPartTransResolver::~ObCDCPartTransResolver()
int ObCDCPartTransResolver::init(
const logservice::TenantLSID &tls_id,
const int64_t start_commit_version)
const int64_t start_commit_version,
const bool enable_direct_load_inc)
{
tls_id_ = tls_id;
enable_direct_load_inc_ = enable_direct_load_inc;
return part_trans_dispatcher_.init(tls_id, start_commit_version);
}
@ -429,6 +433,16 @@ int ObCDCPartTransResolver::read_trans_log_(
}
break;
}
case transaction::ObTxLogType::TX_DIRECT_LOAD_INC_LOG:
{
if (OB_FAIL(handle_direct_load_inc_log_(tx_id, lsn, submit_ts, handling_miss_log, tx_log_block))) {
if (OB_IN_STOP_STATE != ret) {
LOG_ERROR("handle_direct_load_inc_log_ fail", KR(ret), K_(tls_id), K(tx_id), K(tx_id), K(lsn), K(tx_log_header),
K(missing_info));
}
}
break;
}
case transaction::ObTxLogType::TX_RECORD_LOG:
{
if (OB_FAIL(handle_record_(tx_id, lsn, missing_info, tx_log_block))) {
@ -588,6 +602,60 @@ int ObCDCPartTransResolver::handle_multi_data_source_log_(
return ret;
}
int ObCDCPartTransResolver::handle_direct_load_inc_log_(
const transaction::ObTransID &tx_id,
const palf::LSN &lsn,
const int64_t submit_ts,
const bool handling_miss_log,
transaction::ObTxLogBlock &tx_log_block)
{
int ret = OB_SUCCESS;
transaction::ObTxDirectLoadIncLog::TempRef tmp_ref;
transaction::ObTxDirectLoadIncLog direct_load_inc_log(tmp_ref);
PartTransTask *task = NULL;
if (OB_FAIL(obtain_task_(tx_id, task, handling_miss_log))) {
LOG_ERROR("obtain_task_ fail", KR(ret), K_(tls_id), K(tx_id), K(lsn), K(handling_miss_log));
} else if (OB_FAIL(push_fetched_log_entry_(lsn, *task))) {
if (OB_ENTRY_EXIST == ret) {
LOG_WARN("redo already fetched, ignore", KR(ret), K_(tls_id), K(tx_id), K(lsn),
"task_sorted_log_entry_info", task->get_sorted_log_entry_info());
ret = OB_SUCCESS;
} else {
LOG_ERROR("push_fetched_log_entry failed", KR(ret), K_(tls_id), K(tx_id), K(lsn), KPC(task));
}
} else if (!enable_direct_load_inc_) {
// ignore all direct load inc log
} else if (OB_FAIL(tx_log_block.deserialize_log_body(direct_load_inc_log))) {
LOG_ERROR("deserialize_direct_load_inc_log failed", KR(ret), K_(tls_id), K(tx_id), K(lsn));
} else if (transaction::ObTxDirectLoadIncLog::DirectLoadIncLogType::DLI_REDO != direct_load_inc_log.get_ddl_log_type()) {
LOG_DEBUG("ignore DLI_START and DLI_END log", K_(tls_id), K(tx_id), K(lsn), K(direct_load_inc_log));
} else {
transaction::ObTxDLIncLogBuf &log_buf = const_cast<transaction::ObTxDLIncLogBuf &>(direct_load_inc_log.get_dli_buf());
if (OB_FAIL(task->push_direct_load_inc_log(
tx_id,
lsn,
submit_ts,
log_buf.get_buf(),
log_buf.get_buf_size()))) {
if (OB_ENTRY_EXIST == ret) {
LOG_DEBUG("direct_load_inc_log duplication", KR(ret), K_(tls_id), K(tx_id), K(lsn), K(submit_ts),
K(direct_load_inc_log), K(handling_miss_log), K(task));
ret = OB_SUCCESS;
} else if (OB_IN_STOP_STATE != ret) {
LOG_ERROR("push_direct_load_inc_log into PartTransTask fail", KR(ret), K_(tls_id), K(tx_id), K(lsn),
K(handling_miss_log), K(task), K(direct_load_inc_log));
}
} else {
LOG_DEBUG("handle_direct_load_inc_log", K_(tls_id), K(tx_id), K(lsn), K(submit_ts), K(direct_load_inc_log),
K(handling_miss_log), KPC(task));
}
}
return ret;
}
int ObCDCPartTransResolver::handle_record_(
const transaction::ObTransID &tx_id,
const palf::LSN &lsn,

15
src/logservice/libobcdc/src/ob_cdc_part_trans_resolver.h
View File

@ -146,7 +146,8 @@ public:
// init part_trans_resolver
virtual int init(
const logservice::TenantLSID &ls_id,
const int64_t start_commit_version) = 0;
const int64_t start_commit_version,
const bool enable_direct_load_inc) = 0;
public:
/// read log entry
@ -219,7 +220,8 @@ public:
public:
virtual int init(
const logservice::TenantLSID &tls_id,
const int64_t start_commit_version);
const int64_t start_commit_version,
const bool enable_direct_load_inc);
virtual int read(
const char *buf,
@ -294,6 +296,14 @@ private:
const bool handling_miss_log,
transaction::ObTxLogBlock &tx_log_block);
// read ObTxDirectLoadIncLog
int handle_direct_load_inc_log_(
const transaction::ObTransID &tx_id,
const palf::LSN &lsn,
const int64_t submit_ts,
const bool handling_miss_log,
transaction::ObTxLogBlock &tx_log_block);
// read ObTxRecordLog
int handle_record_(
const transaction::ObTransID &tx_id,
@ -427,6 +437,7 @@ private:
logservice::TenantLSID tls_id_;
PartTransDispatcher part_trans_dispatcher_;
IObLogClusterIDFilter &cluster_id_filter_;
bool enable_direct_load_inc_;
private:
DISALLOW_COPY_AND_ASSIGN(ObCDCPartTransResolver);
};

3
src/logservice/libobcdc/src/ob_log_config.h
View File

@ -607,6 +607,9 @@ public:
DEF_CAP(print_mod_memory_usage_threshold, OB_CLUSTER_PARAMETER, "0M", "[0M,]", "print mod memory usage threshold");
DEF_STR(print_mod_memory_usage_label, OB_CLUSTER_PARAMETER, "|", "mod label for print memmory usage");
// Whether to sync the incremental direct load data
T_DEF_BOOL(enable_direct_load_inc, OB_CLUSTER_PARAMETER, 0, "0:disabled, 1:enabled");
#undef OB_CLUSTER_PARAMETER
public:

3
src/logservice/libobcdc/src/ob_log_entry_task_pool.cpp
View File

@ -70,6 +70,7 @@ void ObLogEntryTaskPool::destroy()
}
int ObLogEntryTaskPool::alloc(
const bool is_direct_load_inc_log,
ObLogEntryTask *&log_entry_task,
PartTransTask &host)
{
@ -83,7 +84,7 @@ int ObLogEntryTaskPool::alloc(
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("alloc log_entry_task failed", KR(ret), K_(alloc_cnt), "memory_hold", allocator_.hold());
} else {
log_entry_task = new(ptr) ObLogEntryTask(host);
log_entry_task = new(ptr) ObLogEntryTask(host, is_direct_load_inc_log);
ATOMIC_INC(&alloc_cnt_);
}

2
src/logservice/libobcdc/src/ob_log_entry_task_pool.h
View File

@ -29,6 +29,7 @@ public:
public:
virtual int alloc(
const bool is_direct_load_inc_log,
ObLogEntryTask *&task,
PartTransTask &host) = 0;
virtual void free(ObLogEntryTask *task) = 0;
@ -49,6 +50,7 @@ public:
public:
int alloc(
const bool is_direct_load_inc_log,
ObLogEntryTask *&log_entry_task,
PartTransTask &host) override;
void free(ObLogEntryTask *log_entry_task) override;

6
src/logservice/libobcdc/src/ob_log_fetcher.cpp
View File

@ -84,6 +84,7 @@ ObLogFetcher::~ObLogFetcher()
int ObLogFetcher::init(
const bool is_loading_data_dict_baseline_data,
const bool enable_direct_load_inc,
const ClientFetchingMode fetching_mode,
const ObBackupPathString &archive_dest,
IObLogFetcherDispatcher *dispatcher,
@ -200,6 +201,7 @@ int ObLogFetcher::init(
heartbeat_dispatch_tid_ = 0;
last_timestamp_ = OB_INVALID_TIMESTAMP;
is_loading_data_dict_baseline_data_ = is_loading_data_dict_baseline_data;
enable_direct_load_inc_ = enable_direct_load_inc;
fetching_mode_ = fetching_mode;
archive_dest_ = archive_dest;
log_ext_handler_concurrency_ = cfg.cdc_read_archive_log_concurrency;
@ -212,7 +214,7 @@ int ObLogFetcher::init(
IObCDCPartTransResolver::test_checkpoint_mode_on = cfg.test_checkpoint_mode_on;
IObCDCPartTransResolver::test_mode_ignore_log_type = static_cast<IObCDCPartTransResolver::IgnoreLogType>(cfg.test_mode_ignore_log_type.get());
LOG_INFO("init fetcher succ", K_(is_loading_data_dict_baseline_data),
LOG_INFO("init fetcher succ", K_(is_loading_data_dict_baseline_data), K(enable_direct_load_inc),
"test_mode_on", IObCDCPartTransResolver::test_mode_on,
"test_mode_ignore_log_type", IObCDCPartTransResolver::test_mode_ignore_log_type,
"test_mode_ignore_redo_count", IObCDCPartTransResolver::test_mode_ignore_redo_count,
@ -441,7 +443,7 @@ int ObLogFetcher::add_ls(
}
// Push LS into ObLogLSFetchMgr
else if (OB_FAIL(ls_fetch_mgr_.add_ls(tls_id, start_parameters, is_loading_data_dict_baseline_data_,
fetching_mode_, archive_dest_))) {
enable_direct_load_inc_, fetching_mode_, archive_dest_))) {
LOG_ERROR("add partition by part fetch mgr fail", KR(ret), K(tls_id), K(start_parameters),
K(is_loading_data_dict_baseline_data_));
} else if (OB_FAIL(ls_fetch_mgr_.get_ls_fetch_ctx(tls_id, ls_fetch_ctx))) {

2
src/logservice/libobcdc/src/ob_log_fetcher.h
View File

@ -135,6 +135,7 @@ public:
int init(
const bool is_loading_data_dict_baseline_data,
const bool enable_direct_load_inc,
const ClientFetchingMode fetching_mode,
const ObBackupPathString &archive_dest,
IObLogFetcherDispatcher *dispatcher,
@ -259,6 +260,7 @@ private:
bool is_inited_;
bool is_running_;
bool is_loading_data_dict_baseline_data_;
bool enable_direct_load_inc_;
ClientFetchingMode fetching_mode_;
ObBackupPathString archive_dest_;
archive::LargeBufferPool large_buffer_pool_;

105
src/logservice/libobcdc/src/ob_log_formatter.cpp
View File

@ -453,7 +453,7 @@ int ObLogFormatter::init_binlog_record_for_dml_stmt_task_(
} else {
// select ... for update to record DF_LOCK log to prevent loss of row lock information on the
// standby machine in the event of a master/standby switchover, no synchronization required
if (ObDmlFlag::DF_LOCK == stmt_task->get_dml_flag()) {
if (stmt_task->get_dml_flag().is_lock()) {
is_ignore = true;
} else {
RecordType record_type = get_record_type(stmt_task->get_dml_flag());
@ -702,8 +702,13 @@ int ObLogFormatter::check_table_need_ignore_(
"table_id", table_schema->get_table_id(),
"table_type", ob_table_type_str(table_schema->get_table_type()));
if (OB_FAIL(parse_aux_lob_meta_table_(dml_stmt_task))) {
LOG_ERROR("parse_aux_lob_meta_table_ failed", KR(ret), K(dml_stmt_task));
// Incremental direct load doesn't support lob with outrow in phase1 and Observer will
// disallow the behavior which loads lob data with outrow.
bool need_parse_aux_lob_meta_table = true;
// need_parse_aux_lob_meta_table = !dml_stmt_task.get_redo_log_entry_task().get_redo_log_node()->is_direct_load_inc_log();
if (need_parse_aux_lob_meta_table && OB_FAIL(parse_aux_lob_meta_table_(dml_stmt_task))) {
LOG_ERROR("parse_aux_lob_meta_table_ failed", KR(ret), K(need_parse_aux_lob_meta_table),
K(dml_stmt_task));
}
// Filter sys tables that are not user tables and are not in backup mode
} else if (! table_schema->is_user_table()
@ -805,7 +810,7 @@ void ObLogFormatter::handle_non_full_columns_(
"commit_log_lsn", task.get_commit_log_lsn(),
"commit_version", task.get_trans_commit_version(),
"dml_type", dml_stmt_task.get_dml_flag(),
"dml_type_str", get_dml_str(dml_stmt_task.get_dml_flag()),
"dml_type_str", dml_stmt_task.get_dml_flag().getFlagStr(),
"table_name", table_schema.get_table_name(),
"table_id", table_schema.get_table_id(),
K(dml_stmt_task));
@ -1572,7 +1577,7 @@ int ObLogFormatter::fill_orig_default_value_(
int ObLogFormatter::set_src_category_(IBinlogRecord *br_data,
RowValue *rv,
const ObDmlFlag &dml_flag,
const ObDmlRowFlag &dml_flag,
const bool is_hbase_mode_put)
{
int ret = OB_SUCCESS;
@ -1586,7 +1591,7 @@ int ObLogFormatter::set_src_category_(IBinlogRecord *br_data,
// 1. INSERT statements are always set to full column log format
// 2. DELETE and UPDATE must be populated with old values in full column logging mode, so if they are populated with old values, they are in full column logging format
// 3. OB-HBase mode put special handling
if (ObDmlFlag::DF_INSERT == dml_flag || rv->contain_old_column_ || is_hbase_mode_put) {
if (dml_flag.is_insert() || rv->contain_old_column_ || is_hbase_mode_put) {
src_category = SRC_FULL_RECORDED;
} else {
src_category = SRC_FULL_FAKED;
@ -1603,7 +1608,7 @@ int ObLogFormatter::build_binlog_record_(
ObLogBR *br,
RowValue *rv,
const int64_t new_column_cnt,
const ObDmlFlag &dml_flag,
const ObDmlRowFlag &dml_flag,
const TABLE_SCHEMA *simple_table_schema)
{
int ret = OB_SUCCESS;
@ -1646,17 +1651,16 @@ int ObLogFormatter::build_binlog_record_(
br_data->setNewColumn(rv->new_column_array_, static_cast<int>(rv->column_num_));
br_data->setOldColumn(rv->old_column_array_, static_cast<int>(rv->column_num_));
ObDmlFlag current_dml_flag = dml_flag;
ObDmlRowFlag current_dml_flag = dml_flag;
if (is_hbase_mode_put) {
current_dml_flag = ObDmlFlag::DF_INSERT;
current_dml_flag.set_flag(DF_INSERT);
// modify record type
RecordType type = get_record_type(current_dml_flag);
if (OB_FAIL(br->setInsertRecordTypeForHBasePut(type))) {
LOG_ERROR("br setInsertRecordTypeForHBasePut fail", KR(ret), K(br),
"type", print_record_type(type),
"dml_flag", get_dml_str(dml_flag),
"current_dml_flag", get_dml_str(current_dml_flag),
K(dml_flag),
K(current_dml_flag),
"table_name", simple_table_schema->get_table_name(),
"table_id", simple_table_schema->get_table_id());
} else {
@ -1664,25 +1668,19 @@ int ObLogFormatter::build_binlog_record_(
}
}
switch (current_dml_flag) {
case ObDmlFlag::DF_DELETE: {
if (current_dml_flag.is_delete()) {
ret = format_dml_delete_(br_data, rv);
break;
}
case ObDmlFlag::DF_INSERT: {
} else if (current_dml_flag.is_delete_insert()) {
ret = format_dml_put_(br_data, rv);
} else if (current_dml_flag.is_insert()) {
ret = format_dml_insert_(br_data, rv);
break;
}
case ObDmlFlag::DF_UPDATE: {
} else if (current_dml_flag.is_update()) {
ret = format_dml_update_(br_data, rv);
break;
}
default: {
} else {
ret = OB_NOT_SUPPORTED;
LOG_ERROR("unknown DML type, not supported", K(current_dml_flag));
break;
}
}
if (OB_FAIL(ret)) {
LOG_ERROR("format dml failed", KR(ret), K(table_id), K(current_dml_flag), KPC(simple_table_schema));
}
@ -1693,7 +1691,7 @@ int ObLogFormatter::build_binlog_record_(
}
int ObLogFormatter::is_hbase_mode_put_(const uint64_t table_id,
const ObDmlFlag &dml_flag,
const ObDmlRowFlag &dml_flag,
const int64_t column_number,
const int64_t new_column_cnt,
const bool contain_old_column,
@ -1709,26 +1707,21 @@ int ObLogFormatter::is_hbase_mode_put_(const uint64_t table_id,
LOG_ERROR("hbase_util_ is null", KR(ret), K(hbase_util_));
} else if (OB_FAIL(hbase_util_->is_hbase_table(table_id, is_hbase_table))) {
LOG_ERROR("ObLogHbaseUtil is_hbase_table fail", KR(ret), K(table_id), K(is_hbase_table));
} else if (is_hbase_table && ObDmlFlag::DF_UPDATE == dml_flag && false == contain_old_column) {
} else if (is_hbase_table && dml_flag.is_update() && false == contain_old_column) {
if (column_number == new_column_cnt) {
is_hbase_mode_put = true;
} else if (skip_hbase_mode_put_column_count_not_consistency_) {
is_hbase_mode_put = true;
LOG_INFO("skip hbase mode put column count not consistency", K(table_id),
"dml_flag", get_dml_str(dml_flag),
"hbase_mode_put_column_cnt", new_column_cnt,
K(column_number));
LOG_INFO("skip hbase mode put column count not consistency", K(table_id), K(dml_flag),
"hbase_mode_put_column_cnt", new_column_cnt, K(column_number));
} else {
LOG_ERROR("hbase mode put column cnt is not consistency", K(table_id),
"dml_flag", get_dml_str(dml_flag),
"hbase_mode_put_column_cnt", new_column_cnt,
K(column_number));
LOG_ERROR("hbase mode put column cnt is not consistency", K(table_id), K(dml_flag),
"hbase_mode_put_column_cnt", new_column_cnt, K(column_number));
ret = OB_ERR_UNEXPECTED;
}
LOG_DEBUG("[HBASE] [PUT]", K(is_hbase_mode_put), K(table_id),
"dml_flag", get_dml_str(dml_flag),
LOG_DEBUG("[HBASE] [PUT]", K(is_hbase_mode_put), K(table_id), K(dml_flag),
K(column_number), K(new_column_cnt), K(contain_old_column));
} else {
// do nothing
@ -1933,6 +1926,44 @@ int ObLogFormatter::format_dml_update_(IBinlogRecord *br_data, const RowValue *r
return ret;
}
int ObLogFormatter::format_dml_put_(IBinlogRecord *br_data, const RowValue *row_value)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(br_data) || OB_ISNULL(row_value)) {
ret = OB_INVALID_ARGUMENT;
LOG_ERROR("invalid argument", KR(ret), K(br_data), K(row_value));
} else {
for (int64_t i = 0; OB_SUCCESS == ret && i < row_value->column_num_; i++) {
if (!row_value->is_changed_[i]) {
ObString *str_val = row_value->orig_default_value_[i];
if (OB_ISNULL(str_val)) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("column original default value is NULL", KR(ret), K(i),
"column_num", row_value->column_num_);
} else {
br_data->putNew(str_val->ptr(), str_val->length());
}
} else {
ObString *str_val = row_value->new_columns_[i];
if (OB_ISNULL(str_val)) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("changed column new value is NULL", KR(ret), K(i),
"column_num", row_value->column_num_);
} else {
br_data->putNew(str_val->ptr(), str_val->length());
}
}
// FIXME: No old values are populated, regardless of whether it is a full column log
}
}
return ret;
}
int ObLogFormatter::get_schema_with_online_schema_(
const int64_t version,
const uint64_t tenant_id,

9
src/logservice/libobcdc/src/ob_log_formatter.h
View File

@ -23,7 +23,7 @@
#include "lib/allocator/ob_allocator.h" // ObIAllocator
#include "lib/thread/ob_multi_fixed_queue_thread.h" // ObMQThread
#include "storage/blocksstable/ob_datum_row.h" // ObDmlFlag
#include "storage/blocksstable/ob_datum_row.h" // ObDmlRowFlag
#include "ob_log_binlog_record.h" // IBinlogRecord, ObLogBR
@ -266,7 +266,7 @@ private:
ObLogBR *br,
RowValue *rv,
const int64_t new_column_cnt,
const blocksstable::ObDmlFlag &dml_flag,
const blocksstable::ObDmlRowFlag &dml_flag,
const TABLE_SCHEMA *simple_table_schema);
// HBase mode put
// 1. hbase table
@ -274,18 +274,19 @@ private:
// 3. new value all columns, old value empty
int is_hbase_mode_put_(
const uint64_t table_id,
const blocksstable::ObDmlFlag &dml_flag,
const blocksstable::ObDmlRowFlag &dml_flag,
const int64_t column_number,
const int64_t new_column_cnt,
const bool contain_old_column,
bool &is_hbase_mode_put);
int set_src_category_(IBinlogRecord *br,
RowValue *rv,
const blocksstable::ObDmlFlag &dml_flag,
const blocksstable::ObDmlRowFlag &dml_flag,
const bool is_hbase_mode_put);
int format_dml_delete_(IBinlogRecord *binlog_record, const RowValue *row_value);
int format_dml_insert_(IBinlogRecord *binlog_record, const RowValue *row_value);
int format_dml_update_(IBinlogRecord *binlog_record, const RowValue *row_value);
int format_dml_put_(IBinlogRecord *binlog_record, const RowValue *row_value);
template<class TABLE_SCHEMA>
int fill_orig_default_value_(
RowValue *rv,

15
src/logservice/libobcdc/src/ob_log_instance.cpp
View File

@ -751,6 +751,8 @@ int ObLogInstance::init_components_(const uint64_t start_tstamp_ns)
const char *tb_black_list = TCONF.get_tb_black_list_buf() != NULL ? TCONF.get_tb_black_list_buf()
: TCONF.tb_black_list.str();
const bool enable_direct_load_inc = (1 == TCONF.enable_direct_load_inc);
if (OB_UNLIKELY(! is_working_mode_valid(working_mode))) {
ret = OB_INVALID_CONFIG;
LOG_ERROR("working_mode is not valid", KR(ret), K(working_mode_str), "working_mode", print_working_mode(working_mode));
@ -965,7 +967,8 @@ int ObLogInstance::init_components_(const uint64_t start_tstamp_ns)
}
}
INIT(trans_redo_dispatcher_, ObLogTransRedoDispatcher, redo_dispatcher_mem_limit, enable_sort_by_seq_no, *trans_stat_mgr_);
INIT(trans_redo_dispatcher_, ObLogTransRedoDispatcher, redo_dispatcher_mem_limit,
enable_sort_by_seq_no, *trans_stat_mgr_);
INIT(ddl_processor_, ObLogDDLProcessor, schema_getter_, TCONF.skip_reversed_schema_verison,
TCONF.enable_white_black_list);
@ -991,15 +994,15 @@ int ObLogInstance::init_components_(const uint64_t start_tstamp_ns)
}
}
INIT(fetcher_, ObLogFetcher, false/*is_load_data_dict_baseline_data*/, fetching_mode, archive_dest,
&dispatcher_, sys_ls_handler_, &trans_task_pool_, log_entry_task_pool_,
INIT(fetcher_, ObLogFetcher, false/*is_load_data_dict_baseline_data*/, enable_direct_load_inc, fetching_mode,
archive_dest, &dispatcher_, sys_ls_handler_, &trans_task_pool_, log_entry_task_pool_,
&mysql_proxy_.get_ob_mysql_proxy(), err_handler, cluster_info.cluster_id_, TCONF, start_seq);
if (OB_SUCC(ret)) {
if (is_data_dict_refresh_mode(refresh_mode_)) {
if (OB_FAIL(ObLogMetaDataService::get_instance().init(start_tstamp_ns, fetching_mode, archive_dest,
sys_ls_handler_, &mysql_proxy_.get_ob_mysql_proxy(), err_handler, *part_trans_parser_,
cluster_info.cluster_id_, TCONF, start_seq))) {
cluster_info.cluster_id_, TCONF, start_seq, enable_direct_load_inc))) {
LOG_ERROR("ObLogMetaDataService init failed", KR(ret), K(start_tstamp_ns));
}
}
@ -1702,7 +1705,7 @@ int ObLogInstance::verify_ob_trace_id_(IBinlogRecord *br)
} else {
int record_type = br->recordType();
if (EINSERT == record_type || EUPDATE == record_type || EDELETE == record_type) {
if (EINSERT == record_type || EUPDATE == record_type || EDELETE == record_type || EPUT == record_type) {
// only verify insert\update\delete type
const ObString ob_trace_id_config(ob_trace_id_str_);
ObLogBR *oblog_br = NULL;
@ -1815,7 +1818,7 @@ int ObLogInstance::verify_dml_unique_id_(IBinlogRecord *br)
int record_type = br->recordType();
if (EINSERT == record_type || EUPDATE == record_type || EDELETE == record_type) {
if (EINSERT == record_type || EUPDATE == record_type || EDELETE == record_type || EPUT == record_type) {
// only verify insert\update\delete type
ObLogBR *oblog_br = NULL;
ObLogEntryTask *log_entry_task = NULL;

3
src/logservice/libobcdc/src/ob_log_ls_fetch_mgr.cpp
View File

@ -117,6 +117,7 @@ int ObLogLSFetchMgr::add_ls(
const logservice::TenantLSID &tls_id,
const logfetcher::ObLogFetcherStartParameters &start_parameters,
const bool is_loading_data_dict_baseline_data,
const bool enable_direct_load_inc,
const ClientFetchingMode fetching_mode,
const ObBackupPathString &archive_dest_str)
{
@ -149,7 +150,7 @@ int ObLogLSFetchMgr::add_ls(
} else if (OB_ISNULL(part_trans_resolver)) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("invalid part_trans_resolver", K(part_trans_resolver));
} else if (OB_FAIL(part_trans_resolver->init(tls_id, start_tstamp_ns))) {
} else if (OB_FAIL(part_trans_resolver->init(tls_id, start_tstamp_ns, enable_direct_load_inc))) {
LOG_ERROR("init part trans resolver fail", KR(ret), K(tls_id), K(start_tstamp_ns));
}
// alloc a LSFetchCtx

2
src/logservice/libobcdc/src/ob_log_ls_fetch_mgr.h
View File

@ -45,6 +45,7 @@ public:
const logservice::TenantLSID &tls_id,
const logfetcher::ObLogFetcherStartParameters &start_parameters,
const bool is_loading_data_dict_baseline_data,
const bool enable_direct_load_inc,
const ClientFetchingMode fetching_mode,
const ObBackupPathString &archive_dest_str) = 0;
@ -104,6 +105,7 @@ public:
const logservice::TenantLSID &tls_id,
const logfetcher::ObLogFetcherStartParameters &start_parameters,
const bool is_loading_data_dict_baseline_data,
const bool enable_direct_load_inc,
const ClientFetchingMode fetching_mode,
const ObBackupPathString &archive_dest_str);
virtual int recycle_ls(const logservice::TenantLSID &tls_id);

5
src/logservice/libobcdc/src/ob_log_meta_data_fetcher.cpp
View File

@ -70,7 +70,8 @@ int ObLogMetaDataFetcher::init(
IObLogErrHandler *err_handler,
const int64_t cluster_id,
const ObLogConfig &cfg,
const int64_t start_seq)
const int64_t start_seq,
const bool enable_direct_load_inc)
{
int ret = OB_SUCCESS;
@ -93,7 +94,7 @@ int ObLogMetaDataFetcher::init(
} else {
trans_task_pool_alloc_.set_label("DictConFAlloc");
INIT(log_fetcher_, ObLogFetcher, true/*is_loading_data_dict_baseline_data*/,
fetching_mode, archive_dest, fetcher_dispatcher,
enable_direct_load_inc, fetching_mode, archive_dest, fetcher_dispatcher,
sys_ls_handler, &trans_task_pool_, &log_entry_task_pool_, proxy, err_handler,
cluster_id, cfg, start_seq);

3
src/logservice/libobcdc/src/ob_log_meta_data_fetcher.h
View File

@ -43,7 +43,8 @@ public:
IObLogErrHandler *err_handler,
const int64_t cluster_id,
const ObLogConfig &cfg,
const int64_t start_seq);
const int64_t start_seq,
const bool enable_direct_load_inc);
int start();
void stop();
void destroy();

5
src/logservice/libobcdc/src/ob_log_meta_data_service.cpp
View File

@ -63,7 +63,8 @@ int ObLogMetaDataService::init(
IObLogPartTransParser &part_trans_parser,
const int64_t cluster_id,
const ObLogConfig &cfg,
const int64_t start_seq)
const int64_t start_seq,
const bool enable_direct_load_inc)
{
int ret = OB_SUCCESS;
@ -77,7 +78,7 @@ int ObLogMetaDataService::init(
} else if (OB_FAIL(fetcher_dispatcher_.init(&incremental_replayer_, start_seq))) {
LOG_ERROR("ObLogMetaDataFetcherDispatcher init fail", KR(ret));
} else if (OB_FAIL(fetcher_.init(fetching_mode, archive_dest, &fetcher_dispatcher_, sys_ls_handler,
proxy, err_handler, cluster_id, cfg, start_seq))) {
proxy, err_handler, cluster_id, cfg, start_seq, enable_direct_load_inc))) {
LOG_ERROR("ObLogMetaDataFetcher init fail", KR(ret),
K(fetching_mode), "fetching_log_mode", print_fetching_mode(fetching_mode), K(archive_dest));
} else if (OB_FAIL(fetcher_.start())) {

3
src/logservice/libobcdc/src/ob_log_meta_data_service.h
View File

@ -56,7 +56,8 @@ public:
IObLogPartTransParser &part_trans_parser,
const int64_t cluster_id,
const ObLogConfig &cfg,
const int64_t start_seq);
const int64_t start_seq,
const bool enable_direct_load_inc);
void destroy();
static ObLogMetaDataService &get_instance();

361
src/logservice/libobcdc/src/ob_log_part_trans_parser.cpp
View File

@ -23,11 +23,13 @@
#include "ob_log_instance.h" // TCTX
#include "ob_log_config.h" // TCONF
#include "ob_cdc_lob_aux_table_parse.h" // parse_aux_lob_meta_table_row
#include "src/storage/ddl/ob_ddl_redo_log_row_iterator.h"
using namespace oceanbase::common;
using namespace oceanbase::share::schema;
using namespace oceanbase::transaction;
using namespace oceanbase::memtable;
using namespace oceanbase::storage;
namespace oceanbase
{
@ -183,13 +185,21 @@ int ObLogPartTransParser::parse(ObLogEntryTask &task, volatile bool &stop_flag)
} else if (OB_UNLIKELY(! redo_node->check_data_integrity())) {
ret = OB_INVALID_DATA;
LOG_ERROR("redo data is not valid", KR(ret), KPC(redo_node));
} else if (OB_FAIL(parse_stmts_(tenant, *redo_node, is_build_baseline,
task, *part_trans_task, row_index, stop_flag))) {
LOG_ERROR("parse_stmts_ fail", KR(ret), K(tenant), KPC(redo_node),
K(is_build_baseline), K(task), K(row_index));
} else if (redo_node->is_direct_load_inc_log()) {
if (OB_FAIL(parse_direct_load_inc_stmts_(tenant, *redo_node, task, *part_trans_task, row_index, stop_flag))) {
LOG_ERROR("parse_ddl_stmts_ fail", KR(ret), K(tenant), KPC(redo_node), K(task), K(row_index));
} else {
ATOMIC_AAF(&total_log_size_, redo_node->get_data_len());
LOG_DEBUG("[PARSE] LogEntryTask parse ddl redo log succ", K(task));
}
} else {
ATOMIC_AAF(&total_log_size_, redo_node->get_data_len());
LOG_DEBUG("[PARSE] LogEntryTask parse succ", K(task));
if (OB_FAIL(parse_stmts_(tenant, *redo_node, is_build_baseline, task, *part_trans_task,
row_index, stop_flag))) {
LOG_ERROR("parse_stmts_ fail", KR(ret), K(tenant), KPC(redo_node), K(task), K(row_index));
} else {
ATOMIC_AAF(&total_log_size_, redo_node->get_data_len());
LOG_DEBUG("[PARSE] LogEntryTask parse redo log succ", K(task));
}
}
}
}
@ -295,10 +305,10 @@ int ObLogPartTransParser::parse_stmts_(
}
} else if (MutatorType::MUTATOR_ROW == mutator_type) {
bool is_ignored = false;
MutatorRow *row = NULL;
MemtableMutatorRow *row = NULL;
ObCDCTableInfo table_info;
if (OB_FAIL(parse_mutator_row_(
if (OB_FAIL(parse_memtable_mutator_row_(
tenant,
tablet_id,
redo_data,
@ -310,7 +320,7 @@ int ObLogPartTransParser::parse_stmts_(
row,
table_info,
is_ignored))) {
LOG_ERROR("parse_mutator_row_ failed", KR(ret),
LOG_ERROR("parse_memtable_mutator_row_ failed", KR(ret),
"tls_id", task.get_tls_id(),
"trans_id", task.get_trans_id(),
K(tablet_id), K(redo_log_entry_task), K(is_build_baseline), K(row_index));
@ -374,6 +384,74 @@ int ObLogPartTransParser::parse_stmts_(
return ret;
}
int ObLogPartTransParser::parse_direct_load_inc_stmts_(
ObLogTenant *tenant,
const RedoLogMetaNode &redo_log_node,
ObLogEntryTask &redo_log_entry_task,
PartTransTask &task,
uint64_t &row_index,
volatile bool &stop_flag)
{
int ret = OB_SUCCESS;
const char *ddl_redo_data = redo_log_node.get_data();
const int64_t ddl_redo_data_len = redo_log_node.get_data_len();
if (OB_ISNULL(tenant) || OB_ISNULL(ddl_redo_data) || OB_UNLIKELY(ddl_redo_data_len <= 0)) {
ret = OB_INVALID_ARGUMENT;
LOG_ERROR("invalid argument", KR(ret), KPC(tenant), K(ddl_redo_data), K(ddl_redo_data_len), K(task), K(redo_log_entry_task));
} else {
ObDDLRedoLog ddl_redo_log;
int64_t pos = 0;
if (OB_FAIL(ddl_redo_log.deserialize(ddl_redo_data, ddl_redo_data_len, pos))) {
LOG_ERROR("ObDDLRedoLog deserialize failed", KR(ret), K(ddl_redo_log), K(ddl_redo_data_len), K(pos));
} else {
const common::ObTabletID tablet_id = ddl_redo_log.get_redo_info().table_key_.get_tablet_id();
ObArenaAllocator allocator;
ObDDLRedoLogRowIterator ddl_redo_log_row_iterator(allocator, tenant->get_tenant_id());
if (OB_FAIL(ddl_redo_log_row_iterator.init(ddl_redo_log.get_redo_info().data_buffer_))){
LOG_ERROR("direct_load_log_row_iterator init failed", KR(ret), K(ddl_redo_log));
} else {
while (OB_SUCCESS == ret) {
bool is_ignored = false;
MacroBlockMutatorRow *row = nullptr;
const blocksstable::ObDatumRow *datum_row = nullptr;
const ObStoreRowkey *row_key = nullptr;
transaction::ObTxSEQ seq;
blocksstable::ObDmlRowFlag dml_flag;
ObCDCTableInfo table_info;
if (OB_FAIL(ddl_redo_log_row_iterator.get_next_row(datum_row, row_key, seq, dml_flag))) {
if (OB_ITER_END != ret) {
LOG_ERROR("get next macroblock row failed", KR(ret));
}
} else if (OB_ISNULL(datum_row) || OB_ISNULL(row_key)) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("datum_row or row_key is nullptr", K(datum_row), K(row_key));
} else if (OB_FAIL(alloc_macroblock_mutator_row_(task, redo_log_entry_task, datum_row, row_key, seq, dml_flag, row))) {
LOG_ERROR("alloc macroblock mutator row failed", KR(ret), K(task), K(redo_log_entry_task), KPC(datum_row),
KPC(row_key), K(seq), K(dml_flag), K(row));
} else if (OB_FAIL(parse_macroblock_mutator_row_(tenant, tablet_id, task, redo_log_entry_task, row, table_info, is_ignored))) {
LOG_ERROR("parse macroblock mutator row failed", K(tenant), K(tablet_id), K(task), K(redo_log_entry_task),
KPC(row), K(table_info), K(is_ignored), K(datum_row), K(row_key), K(seq), K(dml_flag), K(ddl_redo_log));
} else if (!is_ignored && OB_FAIL(parse_dml_stmts_(table_info.get_table_id(), row_index, *row,
redo_log_entry_task, task))) {
LOG_ERROR("parse_dml_stmts failed", KR(ret), K(table_info), K(row_index), KPC(row), K(redo_log_entry_task), K(task),
K(datum_row), K(row_key), K(seq), K(dml_flag), K(ddl_redo_log));
} else {
++row_index;
}
} // for each row
if (OB_ITER_END == ret) {
ret = OB_SUCCESS;
}
}
}
}
return ret;
}
int ObLogPartTransParser::parse_mutator_header_(
const char *buf,
const int64_t buf_len,
@ -423,7 +501,7 @@ int ObLogPartTransParser::filter_mutator_table_lock_(const char *buf, const int6
return ret;
}
int ObLogPartTransParser::parse_mutator_row_(
int ObLogPartTransParser::parse_memtable_mutator_row_(
ObLogTenant *tenant,
const ObTabletID &tablet_id,
const char *redo_data,
@ -432,76 +510,24 @@ int ObLogPartTransParser::parse_mutator_row_(
int64_t &pos,
PartTransTask &part_trans_task,
ObLogEntryTask &redo_log_entry_task,
MutatorRow *&row,
MemtableMutatorRow *&row,
ObCDCTableInfo &table_info,
bool &is_ignored)
{
int ret = OB_SUCCESS;
is_ignored = false;
row = NULL;
bool need_rollback = false;
bool need_filter = false;
bool is_in_table_id_cache = false;
const char *filter_reason = NULL;
if (OB_FAIL(alloc_mutator_row_(part_trans_task, redo_log_entry_task, row))) {
LOG_ERROR("alloc_mutator_row_ failed", KR(ret), K(part_trans_task), K(redo_log_entry_task));
if (OB_FAIL(alloc_memtable_mutator_row_(part_trans_task, redo_log_entry_task, row))) {
LOG_ERROR("alloc_memtable_mutator_row_ failed", KR(ret), K(part_trans_task), K(redo_log_entry_task));
} else if (OB_FAIL(row->deserialize(redo_data, redo_data_len, pos))) {
LOG_ERROR("deserialize mutator row fail", KR(ret), KPC(row), K(redo_data_len), K(pos));
} else if (OB_FAIL(check_row_need_rollback_(part_trans_task, *row, need_rollback))) {
LOG_ERROR("check_row_need_rollback_ failed", KR(ret), K(part_trans_task), K(redo_log_entry_task), KPC(row));
} else if (need_rollback) {
LOG_DEBUG("rollback row by RollbackToSavepoint",
"tls_id", part_trans_task.get_tls_id(),
"trans_id", part_trans_task.get_trans_id(),
"row_seq_no", row->seq_no_);
} else if (OB_FAIL(get_table_info_of_tablet_(tenant, part_trans_task, tablet_id, table_info))) {
LOG_ERROR("get_table_info_of_tablet_ failed", KR(ret), K(part_trans_task), K(redo_log_entry_task), KPC(row));
} else if (table_info.is_index_table()) {
need_filter = true;
filter_reason = "INDEX_TABLE";
// DDL Table and LOB_AUX TABLE for DDL Table should not ignore
} else if (part_trans_task.is_ddl_trans()
&& ! (is_ddl_tablet(part_trans_task.get_ls_id(), tablet_id)
|| is_all_ddl_operation_lob_aux_tablet(part_trans_task.get_ls_id(), tablet_id))) {
need_filter = true;
filter_reason = "NON_DDL_RELATED_TABLE";
} else if (part_trans_task.is_ddl_trans() && is_build_baseline
&& is_all_ddl_operation_lob_aux_tablet(part_trans_task.get_ls_id(), tablet_id)) {
need_filter = true;
filter_reason = "DDL_OPERATION_LOB_AUX_TABLE_IN_BUILD_BASELINE";
} else if (part_trans_task.is_ddl_trans()) {
// do nothing, ddl trans should not be filtered
} else {
IObLogPartMgr &part_mgr = tenant->get_part_mgr();
if (OB_FAIL(part_mgr.is_exist_table_id_cache(table_info.get_table_id(), is_in_table_id_cache))) {
LOG_ERROR("check is_exist_table_id_cache failed", KR(ret),
"tls_id", part_trans_task.get_tls_id(),
"trans_id", part_trans_task.get_trans_id(),
K(tablet_id), K(table_info));
} else {
need_filter = ! is_in_table_id_cache;
filter_reason = "NOT_EXIST_IN_TB_ID_CACHE";
}
}
if (need_filter) {
LOG_DEBUG("filter mutator row",
"tls_id", part_trans_task.get_tls_id(),
"trans_id", part_trans_task.get_trans_id(),
K(tablet_id),
K(table_info),
K(filter_reason),
K(is_build_baseline));
}
if (OB_SUCC(ret)) {
is_ignored = need_rollback || need_filter;
} else if (OB_FAIL(check_row_need_ignore_(is_build_baseline, tenant, part_trans_task, redo_log_entry_task,
*row, tablet_id, table_info, is_ignored))) {
LOG_ERROR("check_row_need_ignore_ failed", KR(ret), K(is_build_baseline), K(part_trans_task),
K(redo_log_entry_task), KPC(row));
}
if (OB_FAIL(ret) || is_ignored) {
free_mutator_row_(part_trans_task, redo_log_entry_task, row);
free_memtable_mutator_row_(part_trans_task, redo_log_entry_task, row);
} else if (OB_ISNULL(row)) {
ret = OB_INVALID_DATA;
}
@ -509,10 +535,10 @@ int ObLogPartTransParser::parse_mutator_row_(
return ret;
}
int ObLogPartTransParser::alloc_mutator_row_(
int ObLogPartTransParser::alloc_memtable_mutator_row_(
PartTransTask &part_trans_task,
ObLogEntryTask &redo_log_entry_task,
MutatorRow *&row)
MemtableMutatorRow *&row)
{
int ret = OB_SUCCESS;
void *mutator_row_buf = NULL;
@ -520,35 +546,35 @@ int ObLogPartTransParser::alloc_mutator_row_(
const bool is_ddl_trans = part_trans_task.is_ddl_trans();
if (is_ddl_trans) {
mutator_row_buf = part_trans_task.alloc(sizeof(MutatorRow));
mutator_row_buf = part_trans_task.alloc(sizeof(MemtableMutatorRow));
} else {
mutator_row_buf = redo_log_entry_task.alloc(sizeof(MutatorRow));
mutator_row_buf = redo_log_entry_task.alloc(sizeof(MemtableMutatorRow));
}
if (OB_ISNULL(row = static_cast<MutatorRow *>(mutator_row_buf))) {
LOG_ERROR("alloc memory for MutatorRow fail", K(sizeof(MutatorRow)));
if (OB_ISNULL(row = static_cast<MemtableMutatorRow *>(mutator_row_buf))) {
LOG_ERROR("alloc memory for MemtableMutatorRow fail", K(sizeof(MemtableMutatorRow)));
ret = OB_ALLOCATE_MEMORY_FAILED;
} else {
// FIXME: Destroy MutatorRow from regular channels and free memory
// FIXME: Destroy MemtableMutatorRow from regular channels and free memory
// Currently destroyed in DmlStmtTask and DdlStmtTask, but no memory is freed
// Since this memory is allocated by the Allocator of the PartTransTask, it is guaranteed not to leak
if (is_ddl_trans) {
new (row) MutatorRow(part_trans_task.get_allocator());
new (row) MemtableMutatorRow(part_trans_task.get_allocator());
} else {
new (row) MutatorRow(redo_log_entry_task.get_allocator());
new (row) MemtableMutatorRow(redo_log_entry_task.get_allocator());
}
}
return ret;
}
void ObLogPartTransParser::free_mutator_row_(
void ObLogPartTransParser::free_memtable_mutator_row_(
PartTransTask &part_trans_task,
ObLogEntryTask &redo_log_entry_task,
MutatorRow *&row)
MemtableMutatorRow *&row)
{
if (OB_NOT_NULL(row)) {
row->~MutatorRow();
row->~MemtableMutatorRow();
if (part_trans_task.is_ddl_trans()) {
part_trans_task.free(row);
} else {
@ -592,7 +618,7 @@ int ObLogPartTransParser::check_row_need_rollback_(
int ret = OB_SUCCESS;
need_rollback = false;
const RollbackList &rollback_list = part_trans_task.get_rollback_list();
const auto &row_seq_no = row.seq_no_;
const auto &row_seq_no = row.get_seq_no();
const RollbackNode *rollback_node = rollback_list.head_;
while (OB_SUCC(ret) && OB_NOT_NULL(rollback_node) && ! need_rollback) {
@ -632,7 +658,7 @@ int ObLogPartTransParser::filter_row_data_(
const uint64_t tenant_id = task.get_tenant_id();
// Temporary row data structure to avoid allocation of row data memory
// TODO allocator
MutatorRow row(task.get_allocator());
MemtableMutatorRow row(task.get_allocator());
if (OB_ISNULL(redo_data) || OB_UNLIKELY(redo_data_len <= 0) || OB_UNLIKELY(cur_pos < 0) || OB_UNLIKELY(! table_info.is_valid())) {
LOG_ERROR("invalid argument", K(redo_data), K(task), K(redo_data_len), K(cur_pos), K(table_info));
@ -840,12 +866,15 @@ int ObLogPartTransParser::parse_ddl_lob_aux_stmts_(
LOG_ERROR("allocate memory for DmlStmtTask fail", KR(ret), "Dmlsize", sizeof(DmlStmtTask));
} else {
new (stmt_task) DmlStmtTask(part_trans_task, invalid_log_entry_task, row);
stmt_task->set_table_id(table_id);
ColValueList *rowkey_cols = nullptr;
ColValueList *new_cols = nullptr;
ColValueList *old_cols = nullptr;
if (OB_FAIL(part_trans_task.add_ddl_lob_aux_stmt(row_index, stmt_task))) {
if (OB_ISNULL(stmt_task)) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("stmt_task is nullptr", KR(ret), K(stmt_task));
} else if (FALSE_IT(stmt_task->set_table_id(table_id))) {
} else if (OB_FAIL(part_trans_task.add_ddl_lob_aux_stmt(row_index, stmt_task))) {
LOG_ERROR("add_ddl_lob_aux_stmt into trans task fail", KR(ret), K(part_trans_task),
K(row_index), "stmt_task", *stmt_task);
} else if (OB_FAIL(stmt_task->parse_aux_meta_table_cols(lob_aux_table_schema_info))) {
@ -908,7 +937,10 @@ int ObLogPartTransParser::parse_dml_stmts_(
new (stmt_task) DmlStmtTask(task, redo_log_entry_task, row);
stmt_task->set_table_id(table_id);
if (OB_FAIL(redo_log_entry_task.add_stmt(row_index, stmt_task))) {
if (OB_ISNULL(stmt_task)) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("stmt_task is nullptr", KR(ret), K(stmt_task));
} else if (OB_FAIL(redo_log_entry_task.add_stmt(row_index, stmt_task))) {
LOG_ERROR("add stmt into trans task fail", KR(ret), K(task), K(row_index), "stmt_task", *stmt_task);
} else {
// Update the Local Schema version of PartTransTask
@ -929,10 +961,147 @@ int ObLogPartTransParser::parse_dml_stmts_(
return ret;
}
const transaction::ObTxSEQ &ObLogPartTransParser::get_row_seq_(PartTransTask &task, MutatorRow &row) const
const transaction::ObTxSEQ ObLogPartTransParser::get_row_seq_(PartTransTask &task, MutatorRow &row) const
{
//return task.is_cluster_version_before_320() ? row.sql_no_ : row.seq_no_;
return row.seq_no_;
return row.get_seq_no();
}
int ObLogPartTransParser::alloc_macroblock_mutator_row_(
PartTransTask &part_trans_task,
ObLogEntryTask &redo_log_entry_task,
const blocksstable::ObDatumRow *datum_row,
const common::ObStoreRowkey *row_key,
transaction::ObTxSEQ &seq_no,
blocksstable::ObDmlRowFlag &dml_flag,
MacroBlockMutatorRow *&row)
{
int ret = OB_SUCCESS;
row = NULL;
if (OB_ISNULL(row = static_cast<MacroBlockMutatorRow *>(redo_log_entry_task.alloc(sizeof(MacroBlockMutatorRow))))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("alloc memory for MacroBlockMutatorRow fail", KR(ret), K(sizeof(MacroBlockMutatorRow)));
} else if (FALSE_IT(new (row) MacroBlockMutatorRow(redo_log_entry_task.get_allocator(), dml_flag, seq_no))) {
} else if (OB_FAIL(row->init(redo_log_entry_task.get_allocator(), *datum_row, *row_key))) {
LOG_ERROR("macroblock row init fail", KR(ret), KPC(datum_row), KPC(row_key));
} else {
// succ
}
return ret;
}
void ObLogPartTransParser::free_macroblock_mutator_row_(
PartTransTask &part_trans_task,
ObLogEntryTask &redo_log_entry_task,
MacroBlockMutatorRow *&row)
{
if (OB_NOT_NULL(row)) {
row->~MacroBlockMutatorRow();
redo_log_entry_task.free(row);
row = NULL;
}
}
int ObLogPartTransParser::parse_macroblock_mutator_row_(
ObLogTenant *tenant,
const ObTabletID &tablet_id,
PartTransTask &part_trans_task,
ObLogEntryTask &redo_log_entry_task,
MacroBlockMutatorRow *&row,
ObCDCTableInfo &table_info,
bool &is_ignored)
{
int ret = OB_SUCCESS;
if (OB_FAIL(check_row_need_ignore_(false /* is_build_baseline */, tenant, part_trans_task,
redo_log_entry_task, *row, tablet_id, table_info, is_ignored))) {
LOG_ERROR("check_row_need_ignore_ failed", KR(ret), K(tablet_id), K(part_trans_task),
K(redo_log_entry_task), KPC(row));
}
if (OB_FAIL(ret) || is_ignored) {
free_macroblock_mutator_row_(part_trans_task, redo_log_entry_task, row);
} else if (OB_ISNULL(row)) {
ret = OB_INVALID_DATA;
} else {
row->set_table_id(table_info.get_table_id());
}
return ret;
}
int ObLogPartTransParser::check_row_need_ignore_(
const bool is_build_baseline,
ObLogTenant *tenant,
PartTransTask &part_trans_task,
ObLogEntryTask &redo_log_entry_task,
MutatorRow &row,
const ObTabletID &tablet_id,
ObCDCTableInfo &table_info,
bool &is_ignored)
{
int ret = OB_SUCCESS;
IObLogPartMgr &part_mgr = tenant->get_part_mgr();
is_ignored = false;
bool need_rollback = false;
bool need_filter = false;
bool is_in_table_id_cache = false;
const char *filter_reason = NULL;
if (OB_FAIL(check_row_need_rollback_(part_trans_task, row, need_rollback))) {
LOG_ERROR("check_row_need_rollback_ failed", KR(ret), K(part_trans_task), K(redo_log_entry_task), K(row));
} else if (need_rollback) {
LOG_DEBUG("rollback row by RollbackToSavepoint",
"tls_id", part_trans_task.get_tls_id(),
"trans_id", part_trans_task.get_trans_id(),
"row_seq_no", row.get_seq_no());
} else if (OB_FAIL(get_table_info_of_tablet_(tenant, part_trans_task, tablet_id, table_info))) {
LOG_ERROR("get_table_info_of_tablet_ failed", KR(ret), K(tablet_id), K(part_trans_task), K(redo_log_entry_task), K(row));
} else if (table_info.is_index_table()) {
need_filter = true;
filter_reason = "INDEX_TABLE";
// DDL Table and LOB_AUX TABLE for DDL Table should not ignore
} else if (part_trans_task.is_ddl_trans()
&& ! (is_ddl_tablet(part_trans_task.get_ls_id(), tablet_id)
|| is_all_ddl_operation_lob_aux_tablet(part_trans_task.get_ls_id(), tablet_id))) {
need_filter = true;
filter_reason = "NON_DDL_RELATED_TABLE";
} else if (part_trans_task.is_ddl_trans() && is_build_baseline
&& is_all_ddl_operation_lob_aux_tablet(part_trans_task.get_ls_id(), tablet_id)) {
need_filter = true;
filter_reason = "DDL_OPERATION_LOB_AUX_TABLE_IN_BUILD_BASELINE";
} else if (part_trans_task.is_ddl_trans()) {
// do nothing, ddl trans should not be filtered
} else {
IObLogPartMgr &part_mgr = tenant->get_part_mgr();
if (OB_FAIL(part_mgr.is_exist_table_id_cache(table_info.get_table_id(), is_in_table_id_cache))) {
LOG_ERROR("check is_exist_table_id_cache failed", KR(ret),
"tls_id", part_trans_task.get_tls_id(),
"trans_id", part_trans_task.get_trans_id(),
K(tablet_id), K(table_info));
} else {
need_filter = ! is_in_table_id_cache;
filter_reason = "NOT_EXIST_IN_TB_ID_CACHE";
}
}
if (need_filter) {
LOG_DEBUG("filter mutator row",
"tls_id", part_trans_task.get_tls_id(),
"trans_id", part_trans_task.get_trans_id(),
K(tablet_id),
K(table_info),
K(filter_reason),
K(is_build_baseline));
}
if (OB_SUCC(ret)) {
is_ignored = need_rollback || need_filter;
}
return ret;
}
int ObLogPartTransParser::parse_ext_info_log_mutator_row_(
@ -942,7 +1111,7 @@ int ObLogPartTransParser::parse_ext_info_log_mutator_row_(
int64_t &pos,
PartTransTask &part_trans_task,
ObLogEntryTask &redo_log_entry_task,
MutatorRow *&row,
MemtableMutatorRow *&row,
bool &is_ignored)
{
int ret = OB_SUCCESS;
@ -950,8 +1119,8 @@ int ObLogPartTransParser::parse_ext_info_log_mutator_row_(
row = nullptr;
bool need_rollback = false;
if (OB_FAIL(alloc_mutator_row_(part_trans_task, redo_log_entry_task, row))) {
LOG_ERROR("alloc_mutator_row_ failed", KR(ret), K(part_trans_task), K(redo_log_entry_task));
if (OB_FAIL(alloc_memtable_mutator_row_(part_trans_task, redo_log_entry_task, row))) {
LOG_ERROR("alloc_memtable_mutator_row_ failed", KR(ret), K(part_trans_task), K(redo_log_entry_task));
} else if (OB_FAIL(row->deserialize(redo_data, redo_data_len, pos))) {
LOG_ERROR("deserialize mutator row fail", KR(ret), KPC(row), K(redo_data_len), K(pos));
} else if (OB_FAIL(check_row_need_rollback_(part_trans_task, *row, need_rollback))) {
@ -971,7 +1140,7 @@ int ObLogPartTransParser::parse_ext_info_log_mutator_row_(
}
if (OB_FAIL(ret) || is_ignored) {
free_mutator_row_(part_trans_task, redo_log_entry_task, row);
free_memtable_mutator_row_(part_trans_task, redo_log_entry_task, row);
row = nullptr;
} else if (OB_ISNULL(row)) {
ret = OB_INVALID_DATA;
@ -991,7 +1160,7 @@ int ObLogPartTransParser::handle_mutator_ext_info_log_(
{
int ret = OB_SUCCESS;
bool is_ignored = false;
MutatorRow *row = nullptr;
MemtableMutatorRow *row = nullptr;
if (OB_FAIL(parse_ext_info_log_mutator_row_(
tenant,
redo_data,
@ -1011,7 +1180,7 @@ int ObLogPartTransParser::handle_mutator_ext_info_log_(
const transaction::ObTransID &trans_id = part_trans_task.get_trans_id();
const uint64_t table_id = 0;
ObLobId lob_id; // empty
transaction::ObTxSEQ row_seq_no = row->seq_no_;
transaction::ObTxSEQ row_seq_no = row->get_seq_no();
ObString ext_info_log;
ObCDCLobAuxMetaStorager &lob_aux_meta_storager = TCTX.lob_aux_meta_storager_;
LobAuxMetaKey lob_aux_meta_key(commit_version, tenant_id, trans_id, table_id, lob_id, row_seq_no);
@ -1029,7 +1198,7 @@ int ObLogPartTransParser::handle_mutator_ext_info_log_(
"trans_id", part_trans_task.get_trans_id());
}
if (OB_NOT_NULL(row)) {
free_mutator_row_(part_trans_task, redo_log_entry_task, row);
free_memtable_mutator_row_(part_trans_task, redo_log_entry_task, row);
}
return ret;
}

53
src/logservice/libobcdc/src/ob_log_part_trans_parser.h
View File

@ -82,6 +82,14 @@ private:
PartTransTask &task,
uint64_t &row_index,
volatile bool &stop_flag);
int parse_direct_load_inc_stmts_(
ObLogTenant *tenant,
const RedoLogMetaNode &redo_log_node,
ObLogEntryTask &redo_log_entry_task,
PartTransTask &task,
uint64_t &row_index,
volatile bool &stop_flag);
// try parse mutator_header to get mutator type(support if ob_version >= 320)
// and move forward cur_pos to skip header if header is supported
//
@ -138,16 +146,16 @@ private:
MutatorRow &row,
ObLogEntryTask &redo_log_entry_task,
PartTransTask &part_trans_task);
const transaction::ObTxSEQ &get_row_seq_(PartTransTask &task, MutatorRow &row) const;
int alloc_mutator_row_(
const transaction::ObTxSEQ get_row_seq_(PartTransTask &task, MutatorRow &row) const;
int alloc_memtable_mutator_row_(
PartTransTask &part_trans_task,
ObLogEntryTask &redo_log_entry_task,
MutatorRow *&row);
void free_mutator_row_(
MemtableMutatorRow *&row);
void free_memtable_mutator_row_(
PartTransTask &part_trans_task,
ObLogEntryTask &redo_log_entry_task,
MutatorRow *&row);
int parse_mutator_row_(
MemtableMutatorRow *&row);
int parse_memtable_mutator_row_(
ObLogTenant *tenant,
const ObTabletID &tablet_id,
const char *redo_data,
@ -156,7 +164,36 @@ private:
int64_t &pos,
PartTransTask &part_trans_task,
ObLogEntryTask &redo_log_entry_task,
MutatorRow *&row,
MemtableMutatorRow *&row,
ObCDCTableInfo &table_info,
bool &is_ignored);
int alloc_macroblock_mutator_row_(
PartTransTask &part_trans_task,
ObLogEntryTask &redo_log_entry_task,
const blocksstable::ObDatumRow *datum_row,
const common::ObStoreRowkey *row_key,
transaction::ObTxSEQ &seq_no,
blocksstable::ObDmlRowFlag &dml_flag,
MacroBlockMutatorRow *&row);
void free_macroblock_mutator_row_(
PartTransTask &part_trans_task,
ObLogEntryTask &redo_log_entry_task,
MacroBlockMutatorRow *&row);
int parse_macroblock_mutator_row_(
ObLogTenant *tenant,
const ObTabletID &tablet_id,
PartTransTask &part_trans_task,
ObLogEntryTask &redo_log_entry_task,
MacroBlockMutatorRow *&row,
ObCDCTableInfo &table_info,
bool &is_ignored);
int check_row_need_ignore_(
const bool is_build_baseline,
ObLogTenant *tenant,
PartTransTask &part_trans_task,
ObLogEntryTask &redo_log_entry_task,
MutatorRow &row,
const ObTabletID &tablet_id,
ObCDCTableInfo &table_info,
bool &is_ignored);
int handle_mutator_ext_info_log_(
@ -174,7 +211,7 @@ private:
int64_t &pos,
PartTransTask &part_trans_task,
ObLogEntryTask &redo_log_entry_task,
MutatorRow *&row,
MemtableMutatorRow *&row,
bool &is_ignored);
private:

844
src/logservice/libobcdc/src/ob_log_part_trans_task.cpp
View File

File diff suppressed because it is too large Load Diff

226
src/logservice/libobcdc/src/ob_log_part_trans_task.h
View File

@ -205,81 +205,56 @@ struct ColValue
K_(is_col_nop));
};
///////////////////////////////////////////////////////////////////////////////////
typedef LightyList<ColValue> ColValueList;
class ObObj2strHelper;
// row value
class MutatorRow : public memtable::ObMemtableMutatorRow
//////////////////////////////////////// MutatorRow ///////////////////////////////////////////////
class MutatorRow
{
public:
explicit MutatorRow(common::ObIAllocator &allocator);
virtual ~MutatorRow();
public:
common::ObIAllocator &get_allocator() { return allocator_; }
// Deserialize a row
virtual int deserialize(const char *buf, const int64_t data_len, int64_t &pos);
// Support for filtering table data within PG
// Deserialize some fields: first step to get row_size, table_id
int deserialize_first(
const char *buf,
const int64_t data_len,
int64_t &pos,
int32_t &row_size);
// Deserialize some fields: Step 2 continues the parsing to get the table_version
int deserialize_second(
const char *buf,
const int64_t data_len,
int64_t &pos,
int64_t &table_version);
void reset();
// Parse the column data
// If obj2str_helper is empty, do not convert obj to string
int parse_cols(
virtual int parse_cols(
ObObj2strHelper *obj2str_helper = NULL,
const uint64_t tenant_id = OB_INVALID_TENANT_ID,
const uint64_t table_id = OB_INVALID_ID,
const TableSchemaInfo *tb_schema_info = NULL,
const ObTimeZoneInfoWrap *tz_info_wrap = nullptr,
const bool enable_output_hidden_primary_key = false,
const ObLogAllDdlOperationSchemaInfo *all_ddl_operation_table_schema_info = NULL);
const ObLogAllDdlOperationSchemaInfo *all_ddl_operation_table_schema_info = NULL) = 0;
// Parse the column data based on ObTableSchema
template<class CDC_INNER_TABLE_SCHEMA>
int parse_cols(
const CDC_INNER_TABLE_SCHEMA &lob_aux_table_schema_info);
virtual int parse_cols( const ObCDCLobAuxTableSchemaInfo &lob_aux_table_schema_info) = 0;
virtual int parse_ext_info_log(ObString &ext_info_log) = 0;
int get_cols(
ColValueList **rowkey_cols,
ColValueList **new_cols,
ColValueList **old_cols,
ObLobDataOutRowCtxList **new_lob_ctx_cols);
ObLobDataOutRowCtxList &get_new_lob_ctx_cols() { return new_lob_ctx_cols_; }
int parse_ext_info_log(ObString &ext_info_log);
virtual uint64_t hash(const uint64_t hash) const = 0;
virtual uint64_t get_table_id() const = 0;
virtual blocksstable::ObDmlRowFlag get_dml_flag() const = 0;
virtual const transaction::ObTxSEQ &get_seq_no() const = 0;
public:
TO_STRING_KV(
"Row", static_cast<const memtable::ObMemtableMutatorRow &>(*this),
K_(deserialized),
K_(cols_parsed),
K_(new_cols),
K_(old_cols),
K_(rowkey_cols));
K_(deserialized),
K_(cols_parsed),
K_(new_cols),
K_(old_cols),
K_(rowkey_cols));
private:
protected:
int parse_columns_(
const bool is_parse_new_col,
const char *col_data,
const int64_t col_data_size,
const blocksstable::ObDatumRow &datum_row,
const blocksstable::ObDmlRowFlag &dml_flag,
const int64_t rowkey_cnt,
ObObj2strHelper *obj2str_helper,
const uint64_t tenant_id,
const uint64_t table_id,
@ -330,8 +305,7 @@ private:
template<class CDC_INNER_TABLE_SCHEMA>
int parse_columns_(
const bool is_parse_new_col,
const char *col_data,
const int64_t col_data_size,
const blocksstable::ObDatumRow &datum_row,
const CDC_INNER_TABLE_SCHEMA &inner_table_schema,
ColValueList &cols);
template<class TABLE_SCHEMA>
@ -351,22 +325,129 @@ private:
ObObjMeta &obj_meta,
ObObj &obj);
private:
common::ObIAllocator &allocator_;
bool deserialized_;
bool cols_parsed_;
ColValueList new_cols_; // A list of new values for the columns, currently no primary key values are stored, only normal columns
ColValueList old_cols_; // A list of old values for the columns, currently no primary key values are stored, only normal columns
ColValueList rowkey_cols_; // rowkey column
ObLobDataOutRowCtxList new_lob_ctx_cols_;
protected:
common::ObIAllocator &allocator_;
bool deserialized_;
bool cols_parsed_;
ColValueList new_cols_; // A list of new values for the columns, currently no primary key values are stored, only normal columns
ColValueList old_cols_; // A list of old values for the columns, currently no primary key values are stored, only normal columns
ColValueList rowkey_cols_; // rowkey column
ObLobDataOutRowCtxList new_lob_ctx_cols_;
private:
DISALLOW_COPY_AND_ASSIGN(MutatorRow);
};
//////////////////////////////////////// MutatorRow ///////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////// MacroBlockMutatorRow ///////////////////////////////////////////////
class MacroBlockMutatorRow : public MutatorRow
{
public:
explicit MacroBlockMutatorRow(
common::ObIAllocator &allocator,
const blocksstable::ObDmlRowFlag &dml_flag,
const transaction::ObTxSEQ &seq_no);
virtual ~MacroBlockMutatorRow();
public:
int init(ObIAllocator &allocator, const blocksstable::ObDatumRow &src_datum_row, const common::ObStoreRowkey &src_row_key);
void reset();
// Parse the column data
// If obj2str_helper is empty, do not convert obj to string
int parse_cols(
ObObj2strHelper *obj2str_helper = NULL,
const uint64_t tenant_id = OB_INVALID_TENANT_ID,
const uint64_t table_id = OB_INVALID_ID,
const TableSchemaInfo *tb_schema_info = NULL,
const ObTimeZoneInfoWrap *tz_info_wrap = nullptr,
const bool enable_output_hidden_primary_key = false,
const ObLogAllDdlOperationSchemaInfo *all_ddl_operation_table_schema_info = NULL);
// Parse the column data based on ObTableSchema
int parse_cols(const ObCDCLobAuxTableSchemaInfo &lob_aux_table_schema_info);
int parse_ext_info_log(ObString &ext_info_log);
uint64_t hash(const uint64_t hash) const { return row_key_.murmurhash(hash); }
void set_table_id(const uint64_t table_id) { table_id_ = table_id; }
uint64_t get_table_id() const { return table_id_; }
blocksstable::ObDmlRowFlag get_dml_flag() const { return dml_flag_; }
const transaction::ObTxSEQ &get_seq_no() const { return seq_no_; }
public:
TO_STRING_KV(
K_(is_inited),
K_(table_id),
K_(dml_flag),
K_(seq_no),
K_(row),
K_(row_key));
private:
bool is_inited_;
uint64_t table_id_;
blocksstable::ObDmlRowFlag dml_flag_;
transaction::ObTxSEQ seq_no_;
blocksstable::ObDatumRow row_;
common::ObStoreRowkey row_key_;
private:
DISALLOW_COPY_AND_ASSIGN(MacroBlockMutatorRow);
};
//////////////////////////////////////// MacroBlockMutatorRow ///////////////////////////////////////////////
//////////////////////////////////////// MemtableMutatorRow ///////////////////////////////////////////////
class MemtableMutatorRow : public memtable::ObMemtableMutatorRow, public MutatorRow
{
public:
explicit MemtableMutatorRow(common::ObIAllocator &allocator);
virtual ~MemtableMutatorRow();
public:
// Deserialize a row
int deserialize(const char *buf, const int64_t data_len, int64_t &pos);
// Support for filtering table data within PG
// Deserialize some fields: first step to get row_size, table_id
int deserialize_first(
const char *buf,
const int64_t data_len,
int64_t &pos,
int32_t &row_size);
// Deserialize some fields: Step 2 continues the parsing to get the table_version
int deserialize_second(
const char *buf,
const int64_t data_len,
int64_t &pos,
int64_t &table_version);
void reset();
// Parse the column data
// If obj2str_helper is empty, do not convert obj to string
int parse_cols(
ObObj2strHelper *obj2str_helper = NULL,
const uint64_t tenant_id = OB_INVALID_TENANT_ID,
const uint64_t table_id = OB_INVALID_ID,
const TableSchemaInfo *tb_schema_info = NULL,
const ObTimeZoneInfoWrap *tz_info_wrap = nullptr,
const bool enable_output_hidden_primary_key = false,
const ObLogAllDdlOperationSchemaInfo *all_ddl_operation_table_schema_info = NULL);
// Parse the column data based on ObTableSchema
int parse_cols(const ObCDCLobAuxTableSchemaInfo &lob_aux_table_schema_info);
int parse_ext_info_log(ObString &ext_info_log);
uint64_t hash(const uint64_t hash) const { return rowkey_.murmurhash(hash); }
uint64_t get_table_id() const { return table_id_; }
blocksstable::ObDmlRowFlag get_dml_flag() const { return dml_flag_; }
const transaction::ObTxSEQ &get_seq_no() const { return seq_no_; }
public:
TO_STRING_KV(
"MutatorRow", static_cast<const MutatorRow &>(*this),
"MemtableMutatorRow", static_cast<const memtable::ObMemtableMutatorRow &>(*this));
private:
DISALLOW_COPY_AND_ASSIGN(MemtableMutatorRow);
};
//////////////////////////////////////// MemtableMutatorRow ///////////////////////////////////////////////
#define DELIMITER_STR "_"
// The DML unique ID is part_trans_id(tenant_id + ls_id + trans_id) + redo_log_lsn + row_index, the separator is `_`
@ -429,15 +510,17 @@ public:
void reset();
int64_t get_global_schema_version() const;
int64_t get_table_version() const { return row_.table_version_; }
int64_t get_table_version() const { return 0; }
uint64_t get_table_id() const { return table_id_; }
void set_table_id(const uint64_t table_id) { table_id_ = table_id; }
const logservice::TenantLSID &get_tls_id() const;
const common::ObStoreRowkey &get_rowkey() const { return row_.rowkey_; }
blocksstable::ObDmlFlag get_dml_flag() const { return row_.dml_flag_; }
bool is_insert() const { return blocksstable::ObDmlFlag::DF_INSERT == row_.dml_flag_; }
bool is_update() const { return blocksstable::ObDmlFlag::DF_UPDATE == row_.dml_flag_; }
bool is_delete() const { return blocksstable::ObDmlFlag::DF_DELETE == row_.dml_flag_; }
// TODO(fankun.fan)
// const common::ObStoreRowkey &get_rowkey() const {}
blocksstable::ObDmlRowFlag get_dml_flag() const { return row_.get_dml_flag(); }
bool is_insert() const { return row_.get_dml_flag().is_insert(); }
bool is_update() const { return row_.get_dml_flag().is_update(); }
bool is_delete() const { return row_.get_dml_flag().is_delete(); }
bool is_put() const { return row_.get_dml_flag().is_delete_insert(); }
// Parse the column data
// If obj2str_helper is empty, then no conversion of obj to string
@ -474,7 +557,7 @@ public:
ObLogEntryTask &get_redo_log_entry_task() { return log_entry_task_; }
const transaction::ObTxSEQ get_row_seq_no() const { return row_.seq_no_; }
const transaction::ObTxSEQ &get_row_seq_no() const { return row_.get_seq_no(); }
bool is_callback() const { return 1 == is_callback_; }
void mark_callback() { is_callback_ = 1; }
@ -559,7 +642,7 @@ public:
uint64_t get_op_tablegroup_id() const { return ddl_op_tablegroup_id_; }
int64_t get_op_schema_version() const { return ddl_op_schema_version_; }
uint64_t get_exec_tenant_id() const { return ddl_exec_tenant_id_; }
const transaction::ObTxSEQ &get_row_seq_no() const { return row_.seq_no_; }
const transaction::ObTxSEQ &get_row_seq_no() const { return row_.get_seq_no(); }
public:
// tennat_id(UINT64_MAX: 20) + schema_version(INT64_MAX:19)
@ -646,7 +729,7 @@ typedef LightyList<IStmtTask> StmtList;
class ObLogEntryTask
{
public:
ObLogEntryTask(PartTransTask &host);
ObLogEntryTask(PartTransTask &host, const bool is_direct_load_inc_log = false);
virtual ~ObLogEntryTask();
void reset();
bool is_valid() const;
@ -805,6 +888,13 @@ public:
const char *buf,
const int64_t buf_len);
int push_direct_load_inc_log(
const transaction::ObTransID &trans_id,
const palf::LSN &log_lsn,
const int64_t tstamp,
const char *buf,
const int64_t buf_len);
/**
* @brief: push rollback_to info:
* 1. lsn of rollback_to should push into all_recorded_lsns(ObTxRollbackToLog should has independent LogEntry)
@ -1234,7 +1324,7 @@ private:
const int64_t mutator_row_size);
int push_dml_redo_on_row_start_(
const bool need_store_data,
const memtable::ObMemtableMutatorMeta &meta,
const bool is_direct_load_inc_log,
const palf::LSN &log_lsn,
const char *redo_data,
const int64_t redo_data_size,

1
src/logservice/libobcdc/src/ob_log_resource_collector.cpp
View File

@ -317,7 +317,6 @@ int ObLogResourceCollector::revert_log_entry_task_(ObLogEntryTask *log_entry_tas
}
}
}
log_entry_task_pool->free(log_entry_task);
}

6
src/logservice/libobcdc/src/ob_log_trans_log.h
View File

@ -229,6 +229,11 @@ public:
// Retrieve the last digit of reserve_field_
bool is_stored() const { return reserve_field_ & 0x01; }
// Retrieve the third digit of reserve_field_
bool is_direct_load_inc_log() const { return reserve_field_ & 0x04; }
// Set teh third digit of reserve_field_ to 1
void set_direct_load_inc_log() { reserve_field_ |= 0x04; }
int set_data_info(char *data, int64_t data_len);
bool is_readed() const { return ATOMIC_LOAD(&is_readed_); }
void set_readed() { ATOMIC_SET(&is_readed_, true); }
@ -267,6 +272,7 @@ private:
// An 8 bit reserved field:
// The lowest bit represents the data is stored
// The second low bit represents the data is parsed when contain rollback to savepoint
// The third low bit represents the log is direct load inc log
int8_t reserve_field_; // reserved field
};

6
src/logservice/libobcdc/src/ob_log_trans_redo_dispatcher.cpp
View File

@ -39,7 +39,9 @@ ObLogTransRedoDispatcher::~ObLogTransRedoDispatcher()
destroy();
}
int ObLogTransRedoDispatcher::init(const int64_t redo_dispatcher_memory_limit, const bool enable_sort_by_seq_no, IObLogTransStatMgr &trans_stat_mgr)
int ObLogTransRedoDispatcher::init(const int64_t redo_dispatcher_memory_limit,
const bool enable_sort_by_seq_no,
IObLogTransStatMgr &trans_stat_mgr)
{
int ret = OB_SUCCESS;
@ -417,7 +419,7 @@ int ObLogTransRedoDispatcher::alloc_task_for_redo_(PartTransTask &part_task,
if (OB_UNLIKELY(OB_ISNULL(TCTX.log_entry_task_pool_))) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("log_entry_task_pool is null!", KR(ret));
} else if (OB_FAIL(TCTX.log_entry_task_pool_->alloc(log_entry_task, part_task))) {
} else if (OB_FAIL(TCTX.log_entry_task_pool_->alloc(redo_node.is_direct_load_inc_log(), log_entry_task, part_task))) {
LOG_ERROR("log_entry_task_pool_ alloc fail", KR(ret), KPC(log_entry_task), K(part_task));
} else if (OB_ISNULL(log_entry_task)) {
ret = OB_ERR_UNEXPECTED;

5
src/logservice/libobcdc/src/ob_log_trans_redo_dispatcher.h
View File

@ -55,7 +55,9 @@ class ObLogTransRedoDispatcher: public IObLogTransRedoDispatcher
public:
ObLogTransRedoDispatcher();
virtual ~ObLogTransRedoDispatcher();
int init(const int64_t redo_dispatcher_memory_limit, const bool enable_sort_by_seq_no, IObLogTransStatMgr &trans_stat_mgr);
int init(const int64_t redo_dispatcher_memory_limit,
const bool enable_sort_by_seq_no,
IObLogTransStatMgr &trans_stat_mgr);
void destroy();
public:
@ -118,7 +120,6 @@ private:
int64_t cur_dispatched_redo_memory_; // can dynamicly modify
IObLogTransStatMgr *trans_stat_mgr_;
bool enable_sort_by_seq_no_; // config by init and can't change unless progress restart
private:
DISALLOW_COPY_AND_ASSIGN(ObLogTransRedoDispatcher);
};

67
src/logservice/libobcdc/src/ob_log_utils.cpp
View File

@ -189,57 +189,44 @@ int get_local_ip(ObString &local_ip)
return ret;
}
RecordType get_record_type(const ObDmlFlag &dml_flag)
RecordType get_record_type(const ObDmlRowFlag &dml_flag)
{
RecordType record_type = EUNKNOWN;
// Set record type
// Note: The REPLACE type is not handled, it does not exist in Redo
switch (dml_flag) {
case ObDmlFlag::DF_INSERT:
record_type = EINSERT;
break;
case ObDmlFlag::DF_UPDATE:
record_type = EUPDATE;
break;
case ObDmlFlag::DF_DELETE:
record_type = EDELETE;
break;
default:
record_type = EUNKNOWN;
break;
// Note: must judge is_delete_insert first because PUT is also is_insert, but it's flag_type is DF_TYPE_INSERT_DELETE
if (OB_UNLIKELY(dml_flag.is_delete_insert())) {
record_type = EPUT;
} else if (dml_flag.is_insert()) {
record_type = EINSERT;
} else if (dml_flag.is_update()) {
record_type = EUPDATE;
} else if (dml_flag.is_delete()) {
record_type = EDELETE;
} else {
record_type = EUNKNOWN;
}
return record_type;
}
const char *print_dml_flag(const blocksstable::ObDmlFlag &dml_flag)
const char *print_dml_flag(const blocksstable::ObDmlRowFlag &dml_flag)
{
const char *str = "UNKNOWN";
switch (dml_flag) {
case ObDmlFlag::DF_INSERT:
str = "insert";
break;
case ObDmlFlag::DF_UPDATE:
str = "update";
break;
case ObDmlFlag::DF_DELETE:
str = "delete";
break;
case ObDmlFlag::DF_LOCK:
str = "lock";
break;
default:
str = "UNKNOWN";
break;
if (dml_flag.is_delete_insert()) {
str = "put";
} else if (dml_flag.is_insert()) {
str = "insert";
} else if (dml_flag.is_update()) {
str = "update";
} else if (dml_flag.is_delete()) {
str = "delete";
} else if (dml_flag.is_lock()) {
str = "lock";
} else {
str = "UNKNOWN";
}
return str;
@ -312,6 +299,10 @@ const char *print_record_type(int type)
str = "EDML";
break;
case EPUT:
str = "EPUT";
break;
default:
str = "UNKNOWN";
break;

4
src/logservice/libobcdc/src/ob_log_utils.h
View File

@ -236,8 +236,8 @@ private:
int get_local_ip(common::ObString &local_ip);
RecordType get_record_type(const blocksstable::ObDmlFlag &dml_flag);
const char *print_dml_flag(const blocksstable::ObDmlFlag &dml_flag);
RecordType get_record_type(const blocksstable::ObDmlRowFlag &dml_flag);
const char *print_dml_flag(const blocksstable::ObDmlRowFlag &dml_flag);
const char *print_record_type(int type);
const char *print_src_category(int src_category);
const char *print_record_src_type(int type);

4
src/logservice/libobcdc/tests/ob_binlog_record_printer.cpp
View File

@ -386,7 +386,7 @@ int ObBinlogRecordPrinter::output_data_file(IBinlogRecord *br,
ROW_PRINTF(ptr, size, pos, ri, "unique_id:[%.*s](%d)", unique_id.length(), unique_id.ptr(), unique_id.length());
}
}
} else if ((EINSERT == record_type || EUPDATE == record_type || EDELETE == record_type) && ! only_print_dml_tx_checksum) {
} else if ((EINSERT == record_type || EUPDATE == record_type || EDELETE == record_type || EPUT == record_type) && ! only_print_dml_tx_checksum) {
ri++;
ITableMeta *table_meta = br->getTableMeta();
int64_t column_count = table_meta ? table_meta->getColCount() : -1;
@ -915,7 +915,7 @@ void ObBinlogRecordPrinter::do_br_statistic_(IBinlogRecord &br)
} else if (EDDL == record_type) {
total_tx_count_++;
total_br_count_++;
} else if (EINSERT == record_type || EUPDATE == record_type || EDELETE == record_type) {
} else if (EINSERT == record_type || EUPDATE == record_type || EDELETE == record_type || EPUT == record_type) {
dml_tx_br_count_++;
total_br_count_++;
binlogBuf *new_cols = br.newCols((unsigned int &)new_cols_count);

42
src/observer/ob_rpc_processor_simple.cpp
View File

@ -26,6 +26,7 @@
#include "share/cache/ob_cache_name_define.h"
#include "share/ob_share_util.h"
#include "storage/ddl/ob_ddl_redo_log_writer.h"
#include "storage/ddl/ob_ddl_inc_redo_log_writer.h"
#include "storage/memtable/ob_memtable.h"
#include "storage/blocksstable/ob_block_manager.h"
#include "rootserver/ob_root_service.h"
@ -2305,7 +2306,7 @@ int ObRpcRemoteWriteDDLRedoLogP::process()
LOG_WARN("fail to wait macro block io finish", K(ret));
} else if (OB_FAIL(sstable_redo_writer.init(arg_.ls_id_, arg_.redo_info_.table_key_.tablet_id_))) {
LOG_WARN("init sstable redo writer", K(ret), K_(arg));
} else if (OB_FAIL(sstable_redo_writer.write_macro_block_log(arg_.redo_info_, macro_handle.get_macro_id(), false, arg_.task_id_))) {
} else if (OB_FAIL(sstable_redo_writer.write_macro_block_log(arg_.redo_info_, macro_handle.get_macro_id(), false/*allow_remote_write*/, arg_.task_id_))) {
LOG_WARN("fail to write macro redo", K(ret), K_(arg));
} else if (OB_FAIL(sstable_redo_writer.wait_macro_block_log_finish(arg_.redo_info_,
macro_handle.get_macro_id()))) {
@ -2402,6 +2403,45 @@ int ObRpcRemoteWriteDDLCommitLogP::process()
return ret;
}
int ObRpcRemoteWriteDDLIncCommitLogP::process()
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!arg_.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(ret), K_(arg));
} else {
MTL_SWITCH(arg_.tenant_id_) {
ObRole role = INVALID_ROLE;
ObDDLIncRedoLogWriter sstable_redo_writer;
ObLSService *ls_service = MTL(ObLSService*);
ObTransService *trans_service = MTL(ObTransService *);
ObLSHandle ls_handle;
ObLS *ls = nullptr;
if (OB_FAIL(ls_service->get_ls(arg_.ls_id_, ls_handle, ObLSGetMod::OBSERVER_MOD))) {
LOG_WARN("get ls failed", K(ret), K(arg_));
} else if (OB_ISNULL(ls = ls_handle.get_ls())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected error", K(ret), K(MTL_ID()), K(arg_.ls_id_));
} else if (OB_FAIL(ls->get_ls_role(role))) {
LOG_WARN("get role failed", K(ret), K(MTL_ID()), K(arg_.ls_id_));
} else if (ObRole::LEADER != role) {
ret = OB_NOT_MASTER;
LOG_INFO("leader may not have finished replaying clog, caller retry", K(ret), K(MTL_ID()), K(arg_.ls_id_));
} else if (OB_FAIL(sstable_redo_writer.init(arg_.ls_id_, arg_.tablet_id_))) {
LOG_WARN("init sstable redo writer", K(ret), K(arg_.tablet_id_));
} else if (OB_FAIL(sstable_redo_writer.write_inc_commit_log_with_retry(false/*allow_remote_write*/,
arg_.lob_meta_tablet_id_,
arg_.tx_desc_))) {
LOG_WARN("fail to write inc commit log", K(ret), K(arg_));
} else if (OB_FAIL(trans_service->get_tx_exec_result(*arg_.tx_desc_, result_.tx_result_))) {
LOG_WARN("fail to get_tx_exec_result", K(ret), K(arg_));
}
}
}
return ret;
}
int ObCleanSequenceCacheP::process()
{
int ret = OB_SUCCESS;

1
src/observer/ob_rpc_processor_simple.h
View File

@ -228,6 +228,7 @@ OB_DEFINE_PROCESSOR_S(Srv, OB_DELETE_ARB, ObRpcDeleteArbP);
OB_DEFINE_PROCESSOR_S(Srv, OB_BATCH_BROADCAST_SCHEMA, ObBatchBroadcastSchemaP);
OB_DEFINE_PROCESSOR_S(Srv, OB_REMOTE_WRITE_DDL_REDO_LOG, ObRpcRemoteWriteDDLRedoLogP);
OB_DEFINE_PROCESSOR_S(Srv, OB_REMOTE_WRITE_DDL_COMMIT_LOG, ObRpcRemoteWriteDDLCommitLogP);
OB_DEFINE_PROCESSOR_S(Srv, OB_REMOTE_WRITE_DDL_INC_COMMIT_LOG, ObRpcRemoteWriteDDLIncCommitLogP);
OB_DEFINE_PROCESSOR_S(Srv, OB_CHECK_LS_CAN_OFFLINE, ObRpcCheckLSCanOfflineP);
OB_DEFINE_PROCESSOR_S(Srv, OB_CLEAN_SEQUENCE_CACHE, ObCleanSequenceCacheP);
OB_DEFINE_PROCESSOR_S(Srv, OB_REGISTER_TX_DATA, ObRegisterTxDataP);

1
src/observer/ob_server.cpp
View File

@ -177,7 +177,6 @@ ObServer::ObServer()
lst_operator_(), tablet_operator_(),
server_tracer_(),
location_service_(),
partition_cfy_(),
bandwidth_throttle_(),
sys_bkgd_net_percentage_(0),
ethernet_speed_(0),

3
src/observer/ob_server.h
View File

@ -38,7 +38,6 @@
#include "storage/tx/wrs/ob_weak_read_service.h" // ObWeakReadService
#include "storage/tx/wrs/ob_black_list.h"
#include "storage/ob_partition_component_factory.h"
#include "rootserver/ob_root_service.h"
@ -64,7 +63,6 @@
#include "observer/ob_startup_accel_task_handler.h"
#include "share/ls/ob_ls_table_operator.h" // for ObLSTableOperator
#include "storage/ob_locality_manager.h"
#include "storage/ob_partition_component_factory.h"
#include "storage/ddl/ob_ddl_heart_beat_task.h"
#include "storage/ob_disk_usage_reporter.h"
@ -409,7 +407,6 @@ private:
share::ObLocationService location_service_;
// storage related
storage::ObPartitionComponentFactory partition_cfy_;
common::ObInOutBandwidthThrottle bandwidth_throttle_;
int64_t sys_bkgd_net_percentage_;
int64_t ethernet_speed_;

1
src/observer/ob_service.cpp
View File

@ -40,7 +40,6 @@
#include "share/ob_tablet_replica_checksum_operator.h" // ObTabletReplicaChecksumItem
#include "share/rc/ob_tenant_base.h"
#include "storage/ob_partition_component_factory.h"
#include "storage/ob_i_table.h"
#include "storage/tx/ob_trans_service.h"
#include "sql/optimizer/ob_storage_estimator.h"

1
src/observer/ob_srv_xlator_storage.cpp
View File

@ -104,6 +104,7 @@ void oceanbase::observer::init_srv_xlator_for_storage(ObSrvRpcXlator *xlator) {
RPC_PROCESSOR(ObRpcClearTabletAutoincSeqCacheP, gctx_);
RPC_PROCESSOR(ObRpcRemoteWriteDDLRedoLogP, gctx_);
RPC_PROCESSOR(ObRpcRemoteWriteDDLCommitLogP, gctx_);
RPC_PROCESSOR(ObRpcRemoteWriteDDLIncCommitLogP, gctx_);
RPC_PROCESSOR(ObRpcLSMigrateReplicaP, gctx_);
RPC_PROCESSOR(ObRpcLSAddReplicaP, gctx_);
RPC_PROCESSOR(ObRpcLSTypeTransformP, gctx_);

4
src/observer/table_load/control/ob_table_load_control_rpc_executor.cpp
View File

@ -271,7 +271,9 @@ int ObDirectLoadControlCommitExecutor::process()
ObTableLoadStore store(table_ctx);
if (OB_FAIL(store.init())) {
LOG_WARN("fail to init store", KR(ret));
} else if (OB_FAIL(store.commit(res_.result_info_))) {
} else if (OB_FAIL(store.commit(res_.result_info_,
res_.sql_statistics_,
res_.trans_result_))) {
LOG_WARN("fail to store commit", KR(ret));
} else if (OB_FAIL(ObTableLoadService::remove_ctx(table_ctx))) {
LOG_WARN("fail to remove table ctx", KR(ret), K(key));

1
src/observer/table_load/control/ob_table_load_control_rpc_struct.cpp
View File

@ -20,6 +20,7 @@ namespace oceanbase
namespace observer
{
using namespace sql;
using namespace storage;
using namespace table;
OB_SERIALIZE_MEMBER(ObDirectLoadControlRequest,

1
src/observer/table_load/control/ob_table_load_control_rpc_struct.h
View File

@ -20,7 +20,6 @@
#include "share/table/ob_table_load_define.h"
#include "share/table/ob_table_load_sql_statistics.h"
#include "sql/session/ob_sql_session_mgr.h"
#include "observer/table_load/ob_table_load_struct.h"
#include "storage/direct_load/ob_direct_load_struct.h"
#include "storage/tx/ob_trans_define_v4.h"

6
src/observer/table_load/ob_table_load_client_task.cpp
View File

@ -29,6 +29,7 @@ namespace observer
{
using namespace common;
using namespace sql;
using namespace storage;
using namespace table;
/**
@ -583,7 +584,6 @@ int ObTableLoadClientTask::init_instance()
load_param.session_count_ = load_param.parallel_;
load_param.batch_size_ = 100;
load_param.max_error_row_count_ = param_.get_max_error_row_count();
// load_param.sql_mode_ = 0; // TODO(suzhi.yt) 自增列会用到这个参数
load_param.column_count_ = column_idxs.count();
load_param.need_sort_ = true;
load_param.px_mode_ = false;
@ -613,8 +613,10 @@ int ObTableLoadClientTask::commit_instance()
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadClientTask not init", KR(ret));
} else {
if (OB_FAIL(instance_.commit(result_info_))) {
if (OB_FAIL(instance_.commit())) {
LOG_WARN("fail to commit instance", KR(ret));
} else {
result_info_ = instance_.get_result_info();
}
}
return ret;

249
src/observer/table_load/ob_table_load_coordinator.cpp
View File

@ -28,6 +28,8 @@
#include "observer/table_load/ob_table_load_utils.h"
#include "share/ob_share_util.h"
#include "share/stat/ob_incremental_stat_estimator.h"
#include "share/stat/ob_dbms_stats_utils.h"
#include "observer/table_load/ob_table_load_index_long_wait.h"
#include "observer/omt/ob_tenant.h"
namespace oceanbase
@ -38,6 +40,8 @@ using namespace common;
using namespace table;
using namespace share;
using namespace sql;
using namespace storage;
using namespace transaction;
using namespace lib;
using namespace omt;
@ -111,10 +115,6 @@ void ObTableLoadCoordinator::abort_ctx(ObTableLoadTableCtx *ctx)
if (OB_FAIL(abort_peers_ctx(ctx))) {
LOG_WARN("fail to abort peers ctx", KR(ret));
}
// 5. abort redef table, release table lock
if (OB_FAIL(abort_redef_table(ctx))) {
LOG_WARN("fail to abort redef table", KR(ret));
}
}
}
@ -173,6 +173,7 @@ int ObTableLoadCoordinator::abort_peers_ctx(ObTableLoadTableCtx *ctx)
LOG_WARN("fail to push back", KR(ret), K(addr));
}
}
ObTableLoadIndexLongWait wait_obj(10 * 1000, WAIT_INTERVAL_US);
while (!curr_round->empty() && tries < max_retry_times) {
ret = OB_SUCCESS;
++round;
@ -218,24 +219,12 @@ int ObTableLoadCoordinator::abort_peers_ctx(ObTableLoadTableCtx *ctx)
}
std::swap(curr_round, next_round);
next_round->reuse();
wait_obj.wait();
}
}
return ret;
}
int ObTableLoadCoordinator::abort_redef_table(ObTableLoadTableCtx *ctx)
{
int ret = OB_SUCCESS;
ObTableLoadRedefTableAbortArg arg;
arg.tenant_id_ = ctx->param_.tenant_id_;
arg.task_id_ = ctx->ddl_param_.task_id_;
if (OB_FAIL(
ObTableLoadRedefTable::abort(arg, *ctx->coordinator_ctx_->exec_ctx_->get_session_info()))) {
LOG_WARN("fail to abort redef table", KR(ret), K(arg));
}
return ret;
}
int ObTableLoadCoordinator::init()
{
int ret = OB_SUCCESS;
@ -780,6 +769,7 @@ public:
if (OB_FAIL(coordinator.init())) {
LOG_WARN("fail to init coordinator", KR(ret));
}
ObTableLoadIndexLongWait wait_obj(10 * 1000, WAIT_INTERVAL_US);
while (OB_SUCC(ret)) {
// 确认状态
if (OB_FAIL(ctx_->coordinator_ctx_->check_status(ObTableLoadStatusType::MERGING))) {
@ -789,7 +779,7 @@ public:
else if (OB_FAIL(coordinator.check_peers_merge_result(is_merge_finish))) {
LOG_WARN("fail to check peers merge result", KR(ret));
} else if (!is_merge_finish) {
usleep(WAIT_INTERVAL_US); // 等待1s后重试
wait_obj.wait();
} else {
break;
}
@ -866,11 +856,15 @@ int ObTableLoadCoordinator::add_check_merge_result_task()
* commit
*/
int ObTableLoadCoordinator::commit_peers()
int ObTableLoadCoordinator::commit_peers(ObTableLoadSqlStatistics &sql_statistics)
{
int ret = OB_SUCCESS;
ObTransService *txs = nullptr;
ObTableLoadArray<ObAddr> all_addr_array;
if (OB_FAIL(coordinator_ctx_->partition_location_.get_all_leader(all_addr_array))) {
if (OB_ISNULL(MTL(ObTransService *))) {
ret = OB_ERR_SYS;
LOG_WARN("trans service is null", KR(ret));
} else if (OB_FAIL(coordinator_ctx_->partition_location_.get_all_leader(all_addr_array))) {
LOG_WARN("fail to get all addr", KR(ret));
} else {
LOG_INFO("route_commit_peer_request begin", K(all_addr_array.count()));
@ -884,7 +878,9 @@ int ObTableLoadCoordinator::commit_peers()
ObTableLoadStore store(ctx_);
if (OB_FAIL(store.init())) {
LOG_WARN("fail to init store", KR(ret));
} else if (OB_FAIL(store.commit(res.result_info_))) {
} else if (OB_FAIL(store.commit(res.result_info_,
res.sql_statistics_,
res.trans_result_))) {
LOG_WARN("fail to commit store", KR(ret));
}
} else { // 远端, 发送rpc
@ -895,30 +891,127 @@ int ObTableLoadCoordinator::commit_peers()
ATOMIC_AAF(&coordinator_ctx_->result_info_.deleted_, res.result_info_.deleted_);
ATOMIC_AAF(&coordinator_ctx_->result_info_.skipped_, res.result_info_.skipped_);
ATOMIC_AAF(&coordinator_ctx_->result_info_.warnings_, res.result_info_.warnings_);
if (OB_FAIL(sql_statistics.merge(res.sql_statistics_))) {
LOG_WARN("fail to add result sql stats", KR(ret), K(addr), K(res));
} else if (ObDirectLoadMethod::is_incremental(param_.method_) &&
txs->add_tx_exec_result(*ctx_->session_info_->get_tx_desc(), res.trans_result_)) {
LOG_WARN("fail to add tx exec result", KR(ret));
}
}
}
}
return ret;
}
int ObTableLoadCoordinator::commit_redef_table()
int ObTableLoadCoordinator::write_sql_stat(ObTableLoadSqlStatistics &sql_statistics)
{
int ret = OB_SUCCESS;
ObTableLoadRedefTableFinishArg arg;
arg.tenant_id_ = param_.tenant_id_;
arg.table_id_ = param_.table_id_;
arg.dest_table_id_ = ctx_->ddl_param_.dest_table_id_;
arg.task_id_ = ctx_->ddl_param_.task_id_;
arg.schema_version_ = ctx_->ddl_param_.schema_version_;
if (OB_FAIL(
ObTableLoadRedefTable::finish(arg, *coordinator_ctx_->exec_ctx_->get_session_info()))) {
LOG_WARN("fail to finish redef table", KR(ret), K(arg));
const uint64_t tenant_id = MTL_ID();
const uint64_t table_id = ctx_->ddl_param_.dest_table_id_;
ObArenaAllocator allocator("TLD_Temp");
ObArray<ObOptColumnStat *> global_column_stats;
ObArray<ObOptTableStat *> global_table_stats;
allocator.set_tenant_id(MTL_ID());
global_column_stats.set_tenant_id(MTL_ID());
global_table_stats.set_tenant_id(MTL_ID());
if (OB_UNLIKELY(sql_statistics.is_empty())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(sql_statistics));
} else if (ObDirectLoadMethod::is_full(ctx_->param_.method_)) {
if (OB_FAIL(sql_statistics.get_table_stat_array(global_table_stats))) {
LOG_WARN("fail to get table stat array", KR(ret));
} else if (OB_FAIL(sql_statistics.get_col_stat_array(global_column_stats))) {
LOG_WARN("fail to get column stat array", KR(ret));
}
} else {
ObTableStatParam param;
ObMySQLTransaction trans; // for acquire inner conn
sqlclient::ObISQLConnection *conn = nullptr;
ObArray<ObOptTableStat *> base_table_stats;
ObArray<ObOptColumnStat *> base_column_stats;
TabStatIndMap inc_table_stats;
ColStatIndMap inc_column_stats;
const int64_t map_size = ctx_->param_.column_count_;
base_table_stats.set_tenant_id(MTL_ID());
base_column_stats.set_tenant_id(MTL_ID());
param.tenant_id_ = tenant_id;
param.table_id_ = table_id;
param.part_level_ = ctx_->schema_.part_level_;
param.allocator_ = &allocator;
param.global_stat_param_.need_modify_ = true;
param.part_stat_param_.need_modify_ = false;
param.subpart_stat_param_.need_modify_ = false;
if (!ctx_->schema_.is_partitioned_table_) {
param.global_part_id_ = table_id;
param.global_tablet_id_ = table_id;
}
for (int64_t i = 0; OB_SUCC(ret) && i < ctx_->schema_.column_descs_.count(); ++i) {
const ObColDesc &col_desc = ctx_->schema_.column_descs_.at(i);
ObColumnStatParam col_param;
col_param.column_id_ = col_desc.col_id_;
col_param.cs_type_ = col_desc.col_type_.get_collation_type();
if (0 == i && ctx_->schema_.is_heap_table_) {
// skip hidden pk
} else if (OB_FAIL(param.column_params_.push_back(col_param))) {
LOG_WARN("fail to push back column param", KR(ret));
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(trans.start(GCTX.sql_proxy_, tenant_id))) {
LOG_WARN("fail to start transaction", KR(ret));
} else if (OB_ISNULL(conn = trans.get_connection())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected conn is null", KR(ret));
} else if (OB_FAIL(inc_table_stats.create(map_size,
"TLD_TabStatBkt",
"TLD_TabStatNode",
tenant_id))) {
LOG_WARN("fail to create table stats map", KR(ret));
} else if (OB_FAIL(inc_column_stats.create(map_size,
"TLD_ColStatBkt",
"TLD_ColStatNode",
tenant_id))) {
LOG_WARN("fail to create column stats map", KR(ret));
} else if (OB_FAIL(ObDbmsStatsUtils::get_current_opt_stats(allocator,
conn,
param,
base_table_stats,
base_column_stats))) {
LOG_WARN("failed to get current opt stats", KR(ret));
} else if (OB_FAIL(sql_statistics.get_table_stats(inc_table_stats))) {
LOG_WARN("fail to get table stat array", KR(ret));
} else if (OB_FAIL(sql_statistics.get_col_stats(inc_column_stats))) {
LOG_WARN("fail to get column stat array", KR(ret));
} else if (OB_FAIL(ObDbmsStatsUtils::merge_tab_stats(param,
inc_table_stats,
base_table_stats,
global_table_stats))) {
LOG_WARN("fail to merge tab stats", KR(ret), K(base_table_stats));
} else if (OB_FAIL(ObDbmsStatsUtils::merge_col_stats(param,
inc_column_stats,
base_column_stats,
global_column_stats))) {
LOG_WARN("fail to merge col stats", KR(ret), K(base_column_stats));
}
}
if (OB_SUCC(ret)) {
ObSchemaGetterGuard schema_guard;
if (OB_UNLIKELY(global_table_stats.empty() || global_column_stats.empty())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected empty sql stats", KR(ret), K(global_table_stats), K(global_column_stats));
} else if (OB_FAIL(ObTableLoadSchema::get_schema_guard(tenant_id, schema_guard))) {
LOG_WARN("fail to get schema guard", KR(ret), K(tenant_id));
} else if (OB_FAIL(ObDbmsStatsUtils::split_batch_write(&schema_guard,
ctx_->session_info_,
GCTX.sql_proxy_,
global_table_stats,
global_column_stats))) {
LOG_WARN("fail to split batch write", KR(ret), K(sql_statistics), K(global_table_stats), K(global_column_stats));
}
}
return ret;
}
// commit() = px_commit_data() + px_commit_ddl()
// used in non px_mode
int ObTableLoadCoordinator::commit(ObTableLoadResultInfo &result_info)
{
int ret = OB_SUCCESS;
@ -931,15 +1024,11 @@ int ObTableLoadCoordinator::commit(ObTableLoadResultInfo &result_info)
ObTableLoadSqlStatistics sql_statistics;
if (OB_FAIL(coordinator_ctx_->check_status(ObTableLoadStatusType::MERGED))) {
LOG_WARN("fail to check coordinator status", KR(ret));
} else if (OB_FAIL(commit_peers())) {
} else if (OB_FAIL(commit_peers(sql_statistics))) {
LOG_WARN("fail to commit peers", KR(ret));
} else if (FALSE_IT(coordinator_ctx_->set_enable_heart_beat(false))) {
} else if (param_.online_opt_stat_gather_ &&
OB_FAIL(
drive_sql_stat(coordinator_ctx_->exec_ctx_->get_exec_ctx()))) {
LOG_WARN("fail to drive sql stat", KR(ret));
} else if (OB_FAIL(commit_redef_table())) {
LOG_WARN("fail to commit redef table", KR(ret));
} else if (param_.online_opt_stat_gather_ && OB_FAIL(write_sql_stat(sql_statistics))) {
LOG_WARN("fail to write sql stat", KR(ret));
} else if (OB_FAIL(coordinator_ctx_->set_status_commit())) {
LOG_WARN("fail to set coordinator status commit", KR(ret));
} else {
@ -949,83 +1038,6 @@ int ObTableLoadCoordinator::commit(ObTableLoadResultInfo &result_info)
return ret;
}
// used in insert /*+ append */ into select clause
// commit data loaded
int ObTableLoadCoordinator::px_commit_data()
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadCoordinator not init", KR(ret), KP(this));
} else {
LOG_INFO("coordinator px_commit_data");
ObMutexGuard guard(coordinator_ctx_->get_op_lock());
ObTableLoadSqlStatistics sql_statistics;
if (OB_FAIL(coordinator_ctx_->check_status(ObTableLoadStatusType::MERGED))) {
LOG_WARN("fail to check coordinator status", KR(ret));
} else if (OB_FAIL(commit_peers())) {
LOG_WARN("fail to commit peers", KR(ret));
} else if (FALSE_IT(coordinator_ctx_->set_enable_heart_beat(false))) {
} else if (param_.online_opt_stat_gather_ &&
OB_FAIL(
drive_sql_stat(coordinator_ctx_->exec_ctx_->get_exec_ctx()))) {
LOG_WARN("fail to drive sql stat", KR(ret));
}
}
return ret;
}
// commit ddl procedure
int ObTableLoadCoordinator::px_commit_ddl()
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadCoordinator not init", KR(ret), KP(this));
} else {
LOG_INFO("coordinator px_commit_ddl");
ObMutexGuard guard(coordinator_ctx_->get_op_lock());
if (OB_FAIL(coordinator_ctx_->check_status(ObTableLoadStatusType::MERGED))) {
LOG_WARN("fail to check coordinator status", KR(ret));
} else if (OB_FAIL(commit_redef_table())) {
LOG_WARN("fail to commit redef table", KR(ret));
} else if (OB_FAIL(coordinator_ctx_->set_status_commit())) {
LOG_WARN("fail to set coordinator status commit", KR(ret));
}
}
return ret;
}
int ObTableLoadCoordinator::drive_sql_stat(ObExecContext *ctx)
{
int ret = OB_SUCCESS;
const uint64_t tenant_id = MTL_ID();
const uint64_t table_id = ctx_->ddl_param_.dest_table_id_;
ObSchemaGetterGuard schema_guard;
ObSchemaGetterGuard *tmp_schema_guard = nullptr;
const ObTableSchema *table_schema = nullptr;
if (OB_UNLIKELY(nullptr == ctx)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), KPC(ctx));
} else if (OB_FAIL(ObTableLoadSchema::get_table_schema(tenant_id, table_id, schema_guard,
table_schema))) {
LOG_WARN("fail to get table schema", KR(ret), K(tenant_id), K(table_id));
} else {
tmp_schema_guard = ctx->get_virtual_table_ctx().schema_guard_;
ctx->get_sql_ctx()->schema_guard_ = &schema_guard;
ctx->get_das_ctx().set_sql_ctx(ctx->get_sql_ctx());
}
if (OB_SUCC(ret)) {
if (OB_FAIL(ObIncrementalStatEstimator::derive_global_stat_by_direct_load(
*ctx, table_id))) {
LOG_WARN("fail to drive global stat by direct load", KR(ret));
}
ctx->get_sql_ctx()->schema_guard_ = tmp_schema_guard;
ctx->get_das_ctx().set_sql_ctx(ctx->get_sql_ctx());
}
return ret;
}
/**
* get status
*/
@ -1361,6 +1373,7 @@ public:
if (OB_FAIL(coordinator.init())) {
LOG_WARN("fail to init coordinator", KR(ret));
}
ObTableLoadIndexLongWait wait_obj(10 * 1000, WAIT_INTERVAL_US);
while (OB_SUCC(ret)) {
// 确认trans状态为frozen
if (OB_FAIL(trans_->check_trans_status(ObTableLoadTransStatusType::FROZEN))) {
@ -1370,7 +1383,7 @@ public:
else if (OB_FAIL(coordinator.check_peers_trans_commit(trans_, is_peers_commit))) {
LOG_WARN("fail to check peers trans commit", KR(ret));
} else if (!is_peers_commit) {
usleep(WAIT_INTERVAL_US); // 等待1s后重试
wait_obj.wait();
} else {
break;
}

10
src/observer/table_load/ob_table_load_coordinator.h
View File

@ -34,7 +34,7 @@ class ObTableLoadCoordinatorTrans;
class ObTableLoadCoordinator
{
static const int64_t WAIT_INTERVAL_US = 1LL * 1000 * 1000; // 1s
static const int64_t WAIT_INTERVAL_US = 3 * 1000 * 1000; // 3s
static const int64_t DEFAULT_TIMEOUT_US = 10LL * 1000 * 1000; // 10s
static const int64_t HEART_BEAT_RPC_TIMEOUT_US = 1LL * 1000 * 1000; // 1s
// 申请和释放资源失败等待间隔时间
@ -52,15 +52,12 @@ public:
private:
static int abort_active_trans(ObTableLoadTableCtx *ctx);
static int abort_peers_ctx(ObTableLoadTableCtx *ctx);
static int abort_redef_table(ObTableLoadTableCtx *ctx);
// table load ctrl interface
public:
int begin();
int finish();
int commit(table::ObTableLoadResultInfo &result_info);
int px_commit_data();
int px_commit_ddl();
int get_status(table::ObTableLoadStatusType &status, int &error_code);
int heart_beat();
private:
@ -69,9 +66,8 @@ private:
int confirm_begin_peers();
int pre_merge_peers();
int start_merge_peers();
int commit_peers();
int commit_redef_table();
int drive_sql_stat(sql::ObExecContext *ctx);
int commit_peers(table::ObTableLoadSqlStatistics &sql_statistics);
int write_sql_stat(table::ObTableLoadSqlStatistics &sql_statistics);
int heart_beat_peer();
private:
int add_check_merge_result_task();

52
src/observer/table_load/ob_table_load_index_long_wait.h Normal file
View File

@ -0,0 +1,52 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OB_TABLE_LOAD_INDEX_LONG_WAIT_H_
#define OB_TABLE_LOAD_INDEX_LONG_WAIT_H_
#include "lib/utility/utility.h"
namespace oceanbase
{
namespace observer
{
class ObTableLoadIndexLongWait
{
public:
ObTableLoadIndexLongWait(int64_t wait_us, int64_t max_wait_us)
: wait_us_(wait_us), max_wait_us_(max_wait_us) {}
virtual ~ObTableLoadIndexLongWait() {}
void wait()
{
if (wait_us_ < max_wait_us_) {
ob_usleep(wait_us_);
wait_us_ = 2 * wait_us_;
} else {
ob_usleep(max_wait_us_);
}
}
private:
DISALLOW_COPY_AND_ASSIGN(ObTableLoadIndexLongWait);
private:
// data members
int64_t wait_us_;
int64_t max_wait_us_;
};
}
}
#endif /* OB_TABLE_LOAD_INDEX_LONG_WAIT_H_ */

728
src/observer/table_load/ob_table_load_instance.cpp
View File

@ -15,11 +15,16 @@
#include "observer/table_load/ob_table_load_instance.h"
#include "observer/table_load/ob_table_load_coordinator.h"
#include "observer/table_load/ob_table_load_exec_ctx.h"
#include "observer/table_load/ob_table_load_index_long_wait.h"
#include "observer/table_load/ob_table_load_redef_table.h"
#include "observer/table_load/ob_table_load_service.h"
#include "observer/table_load/ob_table_load_table_ctx.h"
#include "share/ls/ob_ls_operator.h"
#include "share/table/ob_table_load_define.h"
#include "sql/engine/ob_exec_context.h"
#include "storage/ob_common_id_utils.h"
#include "storage/tablelock/ob_table_lock_common.h"
#include "storage/tablelock/ob_table_lock_service.h"
namespace oceanbase
{
@ -28,13 +33,14 @@ namespace observer
using namespace sql;
using namespace storage;
using namespace table;
using namespace transaction;
using namespace transaction::tablelock;
ObTableLoadInstance::ObTableLoadInstance()
: execute_ctx_(nullptr),
allocator_(nullptr),
table_ctx_(nullptr),
job_stat_(nullptr),
is_committed_(false),
is_inited_(false)
{
}
@ -46,16 +52,14 @@ void ObTableLoadInstance::destroy()
int ret = OB_SUCCESS;
trans_ctx_.reset();
if (nullptr != table_ctx_) {
if (!is_committed_) {
// must abort here, abort redef table need exec_ctx session_info
ObTableLoadCoordinator::abort_ctx(table_ctx_);
if (OB_FAIL(end_direct_load(false /*commit*/))) {
LOG_WARN("fail to end direct load", KR(ret));
}
if (OB_FAIL(ObTableLoadService::remove_ctx(table_ctx_))) {
LOG_WARN("table ctx may remove by service", KR(ret), KP(table_ctx_));
}
if (stmt_ctx_.is_started()) {
if (OB_FAIL(end_stmt(false /*commit*/))) {
LOG_WARN("fail to end stmt", KR(ret));
}
ObTableLoadService::put_ctx(table_ctx_);
table_ctx_ = nullptr;
job_stat_ = nullptr;
}
}
@ -86,13 +90,13 @@ int ObTableLoadInstance::init(ObTableLoadParam &param, const ObIArray<int64_t> &
param.insert_mode_))) {
LOG_WARN("fail to check support direct load", KR(ret), K(param));
}
// create table ctx
else if (OB_FAIL(create_table_ctx(param, idx_array))) {
LOG_WARN("fail to create table ctx", KR(ret));
// start stmt
else if (OB_FAIL(start_stmt(param))) {
LOG_WARN("fail to start stmt", KR(ret), K(param));
}
// begin
else if (OB_FAIL(begin())) {
LOG_WARN("fail to begin", KR(ret));
// start direct load
else if (OB_FAIL(start_direct_load(param, idx_array))) {
LOG_WARN("fail to start direct load", KR(ret));
}
// start trans
else if (!param.px_mode_ && OB_FAIL(start_trans())) {
@ -108,21 +112,382 @@ int ObTableLoadInstance::init(ObTableLoadParam &param, const ObIArray<int64_t> &
return ret;
}
int ObTableLoadInstance::create_table_ctx(ObTableLoadParam &param,
const ObIArray<int64_t> &idx_array)
int ObTableLoadInstance::write(int32_t session_id, const table::ObTableLoadObjRowArray &obj_rows)
{
int ret = OB_SUCCESS;
ObTableLoadTableCtx *table_ctx = nullptr;
ObSQLSessionInfo *session_info = execute_ctx_->get_session_info();
ObTableLoadDDLParam ddl_param;
// start redef table
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadInstance not init", KR(ret), KP(this));
} else {
if (OB_FAIL(write_trans(session_id, obj_rows))) {
LOG_WARN("fail to write trans", KR(ret));
}
}
return ret;
}
// commit() = px_commit_data() + px_commit_ddl()
// used in non px_mode
int ObTableLoadInstance::commit()
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadInstance not init", KR(ret), KP(this));
} else {
if (OB_FAIL(commit_trans())) {
LOG_WARN("fail to commit trans", KR(ret));
}
// end direct load
else if (OB_FAIL(end_direct_load(true /*commit*/))) {
LOG_WARN("fail to end direct load", KR(ret));
}
// end stmt
else if (OB_FAIL(end_stmt(true /*commit*/))) {
LOG_WARN("fail to end stmt", KR(ret));
}
}
return ret;
}
// used in insert /*+ append */ into select clause
int ObTableLoadInstance::px_commit_data()
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadInstance not init", KR(ret), KP(this));
} else {
if (OB_FAIL(end_direct_load(true /*commit*/))) {
LOG_WARN("fail to end direct load", KR(ret));
}
}
return ret;
}
int ObTableLoadInstance::px_commit_ddl()
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadInstance not init", KR(ret), KP(this));
} else {
if (OB_FAIL(end_stmt(true /*commit*/))) {
LOG_WARN("fail to end stmt", KR(ret));
}
}
return ret;
}
int ObTableLoadInstance::start_stmt(const ObTableLoadParam &param)
{
int ret = OB_SUCCESS;
stmt_ctx_.reset();
stmt_ctx_.tenant_id_ = param.tenant_id_;
stmt_ctx_.table_id_ = param.table_id_;
stmt_ctx_.session_info_ = execute_ctx_->get_session_info();
stmt_ctx_.is_incremental_ = ObDirectLoadMethod::is_incremental(param.method_);
stmt_ctx_.use_insert_into_select_tx_ = param.px_mode_;
if (stmt_ctx_.is_incremental_) { // incremental direct-load
bool end_sql_tx_if_fail = false;
bool rollback_savepoint_if_fail = false;
if (OB_FAIL(build_tx_param())) {
LOG_WARN("fail to build tx param", KR(ret), K(stmt_ctx_));
} else if (OB_FAIL(start_sql_tx())) {
LOG_WARN("fail to start sql tx", KR(ret), K(stmt_ctx_));
} else if (FALSE_IT(end_sql_tx_if_fail = true)) {
} else if (OB_FAIL(create_implicit_savepoint())) {
LOG_WARN("fail to create implicit savepoint", KR(ret), K(stmt_ctx_));
} else if (OB_FAIL(lock_table_in_tx())) {
LOG_WARN("fail to lock table in tx", KR(ret), K(stmt_ctx_));
} else if (FALSE_IT(rollback_savepoint_if_fail = true)) {
} else if (OB_FAIL(init_ddl_param_for_inc_direct_load())) {
LOG_WARN("fail to init ddl param for inc direct load", KR(ret), K(stmt_ctx_));
}
if (OB_FAIL(ret)) {
int tmp_ret = OB_SUCCESS;
if (rollback_savepoint_if_fail && OB_TMP_FAIL(rollback_to_implicit_savepoint())) {
LOG_WARN("fail to rollback to implicit savepoint", KR(tmp_ret));
}
if (end_sql_tx_if_fail && OB_TMP_FAIL(end_sql_tx(false /*commit*/))) {
LOG_WARN("fail to end sql tx", KR(tmp_ret));
}
}
} else {
if (OB_FAIL(start_redef_table(param))) {
LOG_WARN("fail to start redef table", KR(ret), K(param));
}
}
if (OB_SUCC(ret)) {
stmt_ctx_.is_started_ = true;
LOG_INFO("start stmt succeed", KR(ret), K(stmt_ctx_));
}
return ret;
}
int ObTableLoadInstance::end_stmt(const bool commit)
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
if (stmt_ctx_.is_incremental_) {
// rollback in fail only, ignore ret
if (!commit && OB_TMP_FAIL(rollback_to_implicit_savepoint())) {
LOG_WARN("fail to rollback to implicit savepoint", KR(tmp_ret));
}
if (OB_FAIL(end_sql_tx(commit))) {
LOG_WARN("fail to end sql tx", KR(ret));
}
} else {
if (commit && OB_FAIL(commit_redef_table())) {
LOG_WARN("fail to commit redef table", KR(ret));
}
if (OB_FAIL(ret) || !commit) {
if (OB_TMP_FAIL(abort_redef_table())) {
LOG_WARN("fail to abort redef table", KR(tmp_ret));
ret = COVER_SUCC(tmp_ret);
}
}
}
stmt_ctx_.is_started_ = false;
LOG_INFO("end stmt succeed", KR(ret));
return ret;
}
int64_t ObTableLoadInstance::get_stmt_expire_ts(ObSQLSessionInfo *session_info)
{
int ret = OB_SUCCESS;
const int64_t default_query_timeout = 1000 * 1000; // default timeout 1s
if (OB_ISNULL(session_info)) {
return ObTimeUtil::current_time() + default_query_timeout;
} else {
int64_t query_timeout = 0;
if (OB_FAIL(session_info->get_query_timeout(query_timeout))) {
LOG_WARN("fail to get query timeout", KR(ret));
query_timeout = default_query_timeout;
}
return session_info->get_query_start_time() + query_timeout;
}
}
int ObTableLoadInstance::build_tx_param()
{
int ret = OB_SUCCESS;
ObSQLSessionInfo *session_info = stmt_ctx_.session_info_;
ObTxParam &tx_param = stmt_ctx_.tx_param_;
int64_t timeout_us = 0;
if (OB_FAIL(session_info->get_tx_timeout(timeout_us))) {
LOG_WARN("failed to get tx timeout", KR(ret));
} else {
tx_param.timeout_us_ = timeout_us;
tx_param.lock_timeout_us_ = session_info->get_trx_lock_timeout();
tx_param.access_mode_ = ObTxAccessMode::RW;
tx_param.isolation_ = session_info->get_tx_isolation();
tx_param.cluster_id_ = GCONF.cluster_id;
}
return ret;
}
int ObTableLoadInstance::start_sql_tx()
{
int ret = OB_SUCCESS;
ObTransService *txs = MTL(ObTransService *);
ObSQLSessionInfo *session_info = stmt_ctx_.session_info_;
ObTxDesc *&tx_desc = session_info->get_tx_desc();
if (stmt_ctx_.use_insert_into_select_tx_) { // insert into select path, tx_desc should be available
if (OB_ISNULL(tx_desc)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tx_desc of insert into select should not be null", KR(ret), KP(tx_desc));
} else if (OB_UNLIKELY(!tx_desc->is_valid())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tx_desc of insert into select should be valid", KR(ret), KPC(tx_desc));
} else {
stmt_ctx_.tx_desc_ = tx_desc;
LOG_INFO("use insert into select tx", KPC(tx_desc));
}
} else { // other path, tx_desc could be null, tx_param needs to be set manually
if (OB_UNLIKELY(nullptr != tx_desc && tx_desc->is_in_tx())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("trans already exist", KR(ret), KPC(tx_desc));
} else if (OB_ISNULL(tx_desc) && OB_FAIL(txs->acquire_tx(tx_desc,
session_info->get_sessid(),
session_info->get_data_version()))) {
LOG_WARN("failed to acquire tx", KR(ret), K(session_info->get_sessid()),
K(session_info->get_data_version()));
} else if (OB_FAIL(txs->start_tx(*tx_desc, stmt_ctx_.tx_param_))) {
LOG_WARN("failed to start tx", KR(ret), K(stmt_ctx_));
int tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(txs->release_tx(*tx_desc))) {
LOG_WARN("failed to release tx", KR(tmp_ret), KPC(tx_desc));
} else {
tx_desc = nullptr;
}
} else {
stmt_ctx_.tx_desc_ = tx_desc;
LOG_INFO("start tx succeed", KPC(tx_desc));
}
}
return ret;
}
int ObTableLoadInstance::end_sql_tx(const bool commit)
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
ObTransService *txs = MTL(ObTransService *);
ObSQLSessionInfo *session_info = stmt_ctx_.session_info_;
ObTxDesc *tx_desc = stmt_ctx_.tx_desc_;
if (stmt_ctx_.use_insert_into_select_tx_) {
// do nothing
} else {
if (nullptr == tx_desc) {
// do nothing
} else {
if (commit) {
const int64_t stmt_timeout_ts = get_stmt_expire_ts(session_info);
if (OB_FAIL(txs->commit_tx(*tx_desc, stmt_timeout_ts))) {
LOG_WARN("failed to commit tx", KR(ret), KPC(tx_desc));
} else {
LOG_INFO("commit tx succeed", KPC(tx_desc));
}
} else {
if (OB_FAIL(txs->rollback_tx(*tx_desc))) {
LOG_WARN("failed to rollback tx", KR(ret), KPC(tx_desc));
} else {
LOG_INFO("rollback tx succeed", KPC(tx_desc));
}
}
if (OB_TMP_FAIL(txs->release_tx(*tx_desc))) {
LOG_ERROR("failed to release tx", KR(tmp_ret), KPC(tx_desc));
}
// reset session tx_desc
session_info->get_tx_desc() = nullptr;
}
}
stmt_ctx_.tx_desc_ = nullptr;
return ret;
}
int ObTableLoadInstance::create_implicit_savepoint()
{
int ret = OB_SUCCESS;
ObTransService *txs = MTL(ObTransService *);
ObTxDesc *tx_desc = stmt_ctx_.tx_desc_;
if (stmt_ctx_.use_insert_into_select_tx_) {
// do nothing
} else {
ObTxSEQ savepoint;
if (OB_FAIL(txs->create_implicit_savepoint(*tx_desc, stmt_ctx_.tx_param_, savepoint))) {
LOG_WARN("fail to create implicit savepoint", KR(ret));
} else {
stmt_ctx_.savepoint_ = savepoint;
LOG_INFO("create implicit savepoint succeed", KPC(tx_desc), K(savepoint));
}
}
return ret;
}
int ObTableLoadInstance::rollback_to_implicit_savepoint()
{
int ret = OB_SUCCESS;
ObTransService *txs = MTL(ObTransService *);
ObSQLSessionInfo *session_info = stmt_ctx_.session_info_;
ObTxDesc *tx_desc = stmt_ctx_.tx_desc_;
if (stmt_ctx_.use_insert_into_select_tx_) {
// do nothing
} else {
if (!stmt_ctx_.savepoint_.is_valid()) {
// do nothing
} else {
const int64_t stmt_timeout_ts = get_stmt_expire_ts(session_info);
const ObTxSEQ savepoint = stmt_ctx_.savepoint_;
if (OB_FAIL(txs->rollback_to_implicit_savepoint(*tx_desc, savepoint, stmt_timeout_ts, nullptr))) {
LOG_WARN("failed to rollback to implicit savepoint", KR(ret), KPC(tx_desc));
} else {
stmt_ctx_.savepoint_.reset();
LOG_INFO("rollback to implicit savepoint succeed", KPC(tx_desc), K(savepoint));
}
}
}
return ret;
}
int ObTableLoadInstance::lock_table_in_tx()
{
int ret = OB_SUCCESS;
ObTableLockService *table_lock_service = MTL(ObTableLockService *);
const uint64_t table_id = stmt_ctx_.table_id_;
ObTxDesc *tx_desc = stmt_ctx_.tx_desc_;
ObLockTableRequest lock_table_arg;
lock_table_arg.owner_id_.set_default();
lock_table_arg.lock_mode_ = tablelock::EXCLUSIVE;
lock_table_arg.op_type_ = ObTableLockOpType::IN_TRANS_DML_LOCK;
lock_table_arg.timeout_us_ = 0; // try lock
lock_table_arg.table_id_ = table_id;
bool lock_succeed = false;
int64_t sleep_time = 100 * 1000L; // 100ms
while (OB_SUCC(ret) && !lock_succeed) {
if (OB_FAIL(execute_ctx_->get_exec_ctx()->check_status())) {
LOG_WARN("failed to check status", KR(ret));
} else if (OB_FAIL(table_lock_service->lock_table(*tx_desc, stmt_ctx_.tx_param_, lock_table_arg))) {
if (OB_EAGAIN == ret) {
ob_usleep(sleep_time);
ret = OB_SUCCESS;
} else {
LOG_WARN("failed to lock table", KR(ret), K(lock_table_arg));
}
} else {
lock_succeed = true;
LOG_INFO("lock table in tx succeed", K(table_id), KPC(tx_desc));
}
}
return ret;
}
int ObTableLoadInstance::init_ddl_param_for_inc_direct_load()
{
int ret = OB_SUCCESS;
ObSchemaGetterGuard schema_guard;
ObCommonID raw_id;
share::SCN current_scn;
int64_t schema_version = 0;
uint64_t tenant_data_version = 0;
const uint64_t tenant_id = stmt_ctx_.tenant_id_;
const uint64_t table_id = stmt_ctx_.table_id_;
ObTableLoadDDLParam &ddl_param = stmt_ctx_.ddl_param_;
if (OB_FAIL(ObMultiVersionSchemaService::get_instance().get_tenant_schema_guard(tenant_id,
schema_guard))) {
LOG_WARN("failed to get schema guard", K(ret), K(tenant_id));
} else if (OB_FAIL(schema_guard.get_schema_version(tenant_id, schema_version))) {
LOG_WARN("failed to get tenant schema version", K(ret), K(tenant_id));
} else if (OB_FAIL(ObCommonIDUtils::gen_unique_id_by_rpc(tenant_id, raw_id))) {
LOG_WARN("failed to gen unique id by rpc", KR(ret), K(tenant_id));
} else if (OB_FAIL(share::ObLSAttrOperator::get_tenant_gts(tenant_id, current_scn))) {
LOG_WARN("failed to get gts", KR(ret), K(tenant_id));
} else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, tenant_data_version))) {
LOG_WARN("failed to get min data version", KR(ret), K(tenant_id));
} else {
ddl_param.schema_version_ = schema_version;
ddl_param.task_id_ = raw_id.id();
ddl_param.snapshot_version_ = current_scn.convert_to_ts();
ddl_param.data_version_ = tenant_data_version;
ddl_param.dest_table_id_ = table_id;
ddl_param.cluster_version_ = GET_MIN_CLUSTER_VERSION();
LOG_INFO("init ddl param for inc direct load succeed", K(ddl_param));
}
return ret;
}
int ObTableLoadInstance::start_redef_table(const ObTableLoadParam &param)
{
int ret = OB_SUCCESS;
ObTableLoadDDLParam &ddl_param = stmt_ctx_.ddl_param_;
ObTableLoadRedefTableStartArg start_arg;
ObTableLoadRedefTableStartRes start_res;
start_arg.tenant_id_ = param.tenant_id_;
start_arg.table_id_ = param.table_id_;
start_arg.parallelism_ = param.parallel_;
start_arg.is_load_data_ = !param.px_mode_;
if (OB_FAIL(ObTableLoadRedefTable::start(start_arg, start_res, *session_info))) {
if (OB_FAIL(ObTableLoadRedefTable::start(start_arg, start_res, *stmt_ctx_.session_info_))) {
LOG_WARN("fail to start redef table", KR(ret), K(start_arg));
} else {
ddl_param.dest_table_id_ = start_res.dest_table_id_;
@ -131,47 +496,171 @@ int ObTableLoadInstance::create_table_ctx(ObTableLoadParam &param,
ddl_param.snapshot_version_ = start_res.snapshot_version_;
ddl_param.data_version_ = start_res.data_format_version_;
ddl_param.cluster_version_ = GET_MIN_CLUSTER_VERSION();
LOG_INFO("start redef table succeed", K(ddl_param));
}
if (OB_SUCC(ret)) {
if (OB_ISNULL(table_ctx = ObTableLoadService::alloc_ctx())) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to alloc table ctx", KR(ret), K(param));
} else if (OB_FAIL(table_ctx->init(param, ddl_param, session_info))) {
LOG_WARN("fail to init table ctx", KR(ret));
} else if (OB_FAIL(ObTableLoadCoordinator::init_ctx(table_ctx, idx_array, execute_ctx_))) {
LOG_WARN("fail to coordinator init ctx", KR(ret));
} else if (OB_FAIL(ObTableLoadService::add_ctx(table_ctx))) {
LOG_WARN("fail to add ctx", KR(ret));
} else {
table_ctx_ = table_ctx;
}
return ret;
}
int ObTableLoadInstance::commit_redef_table()
{
int ret = OB_SUCCESS;
ObTableLoadRedefTableFinishArg arg;
arg.tenant_id_ = stmt_ctx_.tenant_id_;
arg.table_id_ = stmt_ctx_.table_id_;
arg.dest_table_id_ = stmt_ctx_.ddl_param_.dest_table_id_;
arg.task_id_ = stmt_ctx_.ddl_param_.task_id_;
arg.schema_version_ = stmt_ctx_.ddl_param_.schema_version_;
if (OB_FAIL(ObTableLoadRedefTable::finish(arg, *stmt_ctx_.session_info_))) {
LOG_WARN("fail to finish redef table", KR(ret), K(arg));
} else {
LOG_INFO("commit redef table succeed");
}
return ret;
}
int ObTableLoadInstance::abort_redef_table()
{
int ret = OB_SUCCESS;
ObTableLoadRedefTableAbortArg arg;
arg.tenant_id_ = stmt_ctx_.tenant_id_;
arg.task_id_ = stmt_ctx_.ddl_param_.task_id_;
if (OB_FAIL(ObTableLoadRedefTable::abort(arg, *stmt_ctx_.session_info_))) {
LOG_WARN("fail to abort redef table", KR(ret), K(arg));
} else {
LOG_INFO("abort redef table succeed");
}
return ret;
}
int ObTableLoadInstance::start_direct_load(const ObTableLoadParam &param,
const ObIArray<int64_t> &idx_array)
{
int ret = OB_SUCCESS;
ObTableLoadTableCtx *table_ctx = nullptr;
ObSQLSessionInfo *session_info = execute_ctx_->get_session_info();
if (OB_UNLIKELY(nullptr != table_ctx_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected table ctx is not null", KR(ret));
} else if (OB_ISNULL(table_ctx = ObTableLoadService::alloc_ctx())) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to alloc table ctx", KR(ret), K(param));
} else if (OB_FAIL(table_ctx->init(param, stmt_ctx_.ddl_param_, session_info))) {
LOG_WARN("fail to init table ctx", KR(ret));
} else if (OB_FAIL(ObTableLoadCoordinator::init_ctx(table_ctx, idx_array, execute_ctx_))) {
LOG_WARN("fail to coordinator init ctx", KR(ret));
} else if (OB_FAIL(ObTableLoadService::add_ctx(table_ctx))) {
LOG_WARN("fail to add ctx", KR(ret));
} else {
table_ctx_ = table_ctx;
}
if (OB_FAIL(ret)) {
int tmp_ret = OB_SUCCESS;
if (ddl_param.is_valid()) {
ObTableLoadRedefTableAbortArg abort_arg;
abort_arg.tenant_id_ = param.tenant_id_;
abort_arg.task_id_ = ddl_param.task_id_;
if (OB_TMP_FAIL(ObTableLoadRedefTable::abort(abort_arg, *session_info))) {
LOG_WARN("fail to abort redef table", KR(tmp_ret), K(abort_arg));
}
}
if (nullptr != table_ctx) {
ObTableLoadService::free_ctx(table_ctx);
table_ctx = nullptr;
}
}
if (OB_SUCC(ret)) {
ObTableLoadCoordinator coordinator(table_ctx_);
if (OB_FAIL(coordinator.init())) {
LOG_WARN("fail to init coordinator", KR(ret));
} else if (OB_FAIL(coordinator.begin())) {
LOG_WARN("fail to coodrinator begin", KR(ret));
}
}
return ret;
}
int ObTableLoadInstance::begin()
int ObTableLoadInstance::end_direct_load(const bool commit)
{
int ret = OB_SUCCESS;
ObTableLoadCoordinator coordinator(table_ctx_);
if (OB_FAIL(coordinator.init())) {
LOG_WARN("fail to init coordinator", KR(ret));
} else if (OB_FAIL(coordinator.begin())) {
LOG_WARN("fail to coodrinator begin", KR(ret));
int tmp_ret = OB_SUCCESS;
if (OB_UNLIKELY(nullptr == table_ctx_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected table ctx is null", KR(ret));
} else {
bool need_abort = !commit;
if (commit) {
ObTableLoadStatusType status = ObTableLoadStatusType::NONE;
int error_code = OB_SUCCESS;
ObTableLoadCoordinator coordinator(table_ctx_);
if (OB_FAIL(coordinator.init())) {
LOG_WARN("fail to init coordinator", KR(ret));
}
// finish
else if (OB_FAIL(coordinator.finish())) {
LOG_WARN("fail to finish", KR(ret));
}
ObTableLoadIndexLongWait wait_obj(10 * 1000, WAIT_INTERVAL_US);
while (OB_SUCC(ret) && ObTableLoadStatusType::MERGED != status &&
OB_SUCC(execute_ctx_->check_status())) {
if (OB_FAIL(coordinator.get_status(status, error_code))) {
LOG_WARN("fail to coordinator get status", KR(ret));
} else {
switch (status) {
case ObTableLoadStatusType::FROZEN:
case ObTableLoadStatusType::MERGING:
wait_obj.wait();
break;
case ObTableLoadStatusType::MERGED:
break;
case ObTableLoadStatusType::ERROR:
ret = error_code;
LOG_WARN("table load has error", KR(ret));
break;
default:
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected status", KR(ret), K(status));
break;
}
}
}
if (OB_SUCC(ret)) {
// commit
if (OB_FAIL(coordinator.commit(result_info_))) {
LOG_WARN("fail to commit", KR(ret));
}
}
if (OB_FAIL(ret)) {
need_abort = true;
}
}
if (need_abort) {
// must abort here, abort redef table need exec_ctx session_info
ObTableLoadCoordinator::abort_ctx(table_ctx_);
}
if (OB_TMP_FAIL(add_tx_result_to_user_session())) {
LOG_WARN("fail to add tx result to user session", KR(tmp_ret));
ret = COVER_SUCC(tmp_ret);
}
if (OB_TMP_FAIL(ObTableLoadService::remove_ctx(table_ctx_))) {
LOG_WARN("table ctx may remove by service", KR(tmp_ret), KP(table_ctx_));
}
ObTableLoadService::put_ctx(table_ctx_);
table_ctx_ = nullptr;
job_stat_ = nullptr;
}
return ret;
}
int ObTableLoadInstance::add_tx_result_to_user_session()
{
int ret = OB_SUCCESS;
ObTransService *txs = MTL(ObTransService *);
ObSQLSessionInfo *session_info = stmt_ctx_.session_info_;
ObTxDesc *tx_desc = session_info->get_tx_desc();
if (OB_ISNULL(table_ctx_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected table ctx is null", KR(ret));
} else if (stmt_ctx_.is_incremental_ && !stmt_ctx_.has_added_tx_result_ && OB_NOT_NULL(tx_desc)) {
ObTxExecResult exec_result;
if (OB_FAIL(txs->get_tx_exec_result(*table_ctx_->session_info_->get_tx_desc(), exec_result))) {
LOG_WARN("failed to get tx exec result", KR(ret));
} else if (OB_FAIL(txs->add_tx_exec_result(*tx_desc, exec_result))) {
LOG_WARN("failed to add tx exec result", KR(ret), K(exec_result));
} else {
stmt_ctx_.has_added_tx_result_ = true;
LOG_INFO("add tx result to user session succeed");
}
}
return ret;
}
@ -196,14 +685,11 @@ int ObTableLoadInstance::start_trans()
return ret;
}
int ObTableLoadInstance::write(int32_t session_id, const table::ObTableLoadObjRowArray &obj_rows)
int ObTableLoadInstance::write_trans(int32_t session_id, const ObTableLoadObjRowArray &obj_rows)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadInstance not init", KR(ret), KP(this));
} else if (OB_UNLIKELY(session_id < 0 || session_id > table_ctx_->param_.write_session_count_ ||
obj_rows.empty())) {
if (OB_UNLIKELY(session_id < 0 || session_id > table_ctx_->param_.write_session_count_ ||
obj_rows.empty())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(session_id), K(obj_rows.count()));
} else {
@ -220,7 +706,7 @@ int ObTableLoadInstance::write(int32_t session_id, const table::ObTableLoadObjRo
return ret;
}
int ObTableLoadInstance::check_trans_committed()
int ObTableLoadInstance::commit_trans()
{
int ret = OB_SUCCESS;
ObTableLoadTransStatusType trans_status = ObTableLoadTransStatusType::NONE;
@ -229,6 +715,11 @@ int ObTableLoadInstance::check_trans_committed()
if (OB_FAIL(coordinator.init())) {
LOG_WARN("fail to init coordinator", KR(ret));
}
// finish trans
else if (OB_FAIL(coordinator.finish_trans(trans_ctx_.trans_id_))) {
LOG_WARN("fail to finish trans", KR(ret));
}
ObTableLoadIndexLongWait wait_obj(10 * 1000, WAIT_INTERVAL_US);
while (OB_SUCC(ret) && ObTableLoadTransStatusType::COMMIT != trans_status &&
OB_SUCC(execute_ctx_->check_status())) {
if (OB_FAIL(coordinator.get_trans_status(trans_ctx_.trans_id_, trans_status, error_code))) {
@ -236,7 +727,7 @@ int ObTableLoadInstance::check_trans_committed()
} else {
switch (trans_status) {
case ObTableLoadTransStatusType::FROZEN:
usleep(WAIT_INTERVAL_US);
wait_obj.wait();
break;
case ObTableLoadTransStatusType::COMMIT:
break;
@ -254,126 +745,5 @@ int ObTableLoadInstance::check_trans_committed()
return ret;
}
int ObTableLoadInstance::check_merged()
{
int ret = OB_SUCCESS;
ObTableLoadStatusType status = ObTableLoadStatusType::NONE;
int error_code = OB_SUCCESS;
ObTableLoadCoordinator coordinator(table_ctx_);
if (OB_FAIL(coordinator.init())) {
LOG_WARN("fail to init coordinator", KR(ret));
}
while (OB_SUCC(ret) && ObTableLoadStatusType::MERGED != status &&
OB_SUCC(execute_ctx_->check_status())) {
if (OB_FAIL(coordinator.get_status(status, error_code))) {
LOG_WARN("fail to coordinator get status", KR(ret));
} else {
switch (status) {
case ObTableLoadStatusType::FROZEN:
case ObTableLoadStatusType::MERGING:
usleep(WAIT_INTERVAL_US);
break;
case ObTableLoadStatusType::MERGED:
break;
case ObTableLoadStatusType::ERROR:
ret = error_code;
LOG_WARN("table load has error", KR(ret));
break;
default:
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected status", KR(ret), K(status));
break;
}
}
}
return ret;
}
// commit() = px_commit_data() + px_commit_ddl()
// used in non px_mode
int ObTableLoadInstance::commit(ObTableLoadResultInfo &result_info)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadInstance not init", KR(ret), KP(this));
} else {
ObTableLoadCoordinator coordinator(table_ctx_);
if (OB_FAIL(coordinator.init())) {
LOG_WARN("fail to init coordinator", KR(ret));
}
// finish trans
else if (OB_FAIL(coordinator.finish_trans(trans_ctx_.trans_id_))) {
LOG_WARN("fail to finish trans", KR(ret));
}
// wait trans commit
else if (OB_FAIL(check_trans_committed())) {
LOG_WARN("fail to check trans committed", KR(ret));
}
// finish
else if (OB_FAIL(coordinator.finish())) {
LOG_WARN("fail to finish", KR(ret));
}
// wait merge
else if (OB_FAIL(check_merged())) {
LOG_WARN("fail to check merged", KR(ret));
}
// commit
else if (OB_FAIL(coordinator.commit(result_info))) {
LOG_WARN("fail to commit", KR(ret));
} else {
is_committed_ = true;
}
}
return ret;
}
// used in insert /*+ append */ into select clause
int ObTableLoadInstance::px_commit_data()
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadInstance not init", KR(ret), KP(this));
} else {
ObTableLoadCoordinator coordinator(table_ctx_);
if (OB_FAIL(coordinator.init())) {
LOG_WARN("fail to init coordinator", KR(ret));
}
// finish
else if (OB_FAIL(coordinator.finish())) {
LOG_WARN("fail to finish", KR(ret));
}
// wait merge
else if (OB_FAIL(check_merged())) {
LOG_WARN("fail to check merged", KR(ret));
}
// commit
else if (OB_FAIL(coordinator.px_commit_data())) {
LOG_WARN("fail to do px_commit_data", KR(ret));
}
}
return ret;
}
int ObTableLoadInstance::px_commit_ddl()
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadInstance not init", KR(ret), KP(this));
} else {
ObTableLoadCoordinator coordinator(table_ctx_);
if (OB_FAIL(coordinator.init())) {
LOG_WARN("fail to init coordinator", KR(ret));
} else if (OB_FAIL(coordinator.px_commit_ddl())) {
LOG_WARN("fail to do px_commit_ddl", KR(ret));
} else {
is_committed_ = true;
}
}
return ret;
}
} // namespace observer
} // namespace oceanbase

106
src/observer/table_load/ob_table_load_instance.h
View File

@ -18,15 +18,21 @@
namespace oceanbase
{
namespace transaction
{
class ObTxDesc;
struct ObTxParam;
}
namespace observer
{
class ObTableLoadParam;
class ObTableLoadTableCtx;
class ObTableLoadExecCtx;
class ObTableLoadDDLParam;
class ObTableLoadInstance
{
static const int64_t WAIT_INTERVAL_US = 1LL * 1000 * 1000; // 1s
static const int64_t WAIT_INTERVAL_US = 3 * 1000 * 1000; // 3s
public:
ObTableLoadInstance();
~ObTableLoadInstance();
@ -34,26 +40,91 @@ public:
int init(ObTableLoadParam &param, const common::ObIArray<int64_t> &idx_array,
ObTableLoadExecCtx *execute_ctx);
int write(int32_t session_id, const table::ObTableLoadObjRowArray &obj_rows);
int commit(table::ObTableLoadResultInfo &result_info);
int commit();
int px_commit_data();
int px_commit_ddl();
sql::ObLoadDataStat *get_job_stat() const { return job_stat_; }
void update_job_stat_parsed_rows(int64_t parsed_rows)
{
ATOMIC_AAF(&job_stat_->parsed_rows_, parsed_rows);
}
void update_job_stat_parsed_bytes(int64_t parsed_bytes)
{
ATOMIC_AAF(&job_stat_->parsed_bytes_, parsed_bytes);
}
const ObTableLoadTableCtx* get_table_ctx() { return table_ctx_; }
sql::ObLoadDataStat *get_job_stat() const { return job_stat_; }
const table::ObTableLoadResultInfo &get_result_info() const { return result_info_; }
private:
int create_table_ctx(ObTableLoadParam &param, const common::ObIArray<int64_t> &idx_array);
int begin();
int start_stmt(const ObTableLoadParam &param);
int end_stmt(const bool commit);
// incremental
static int64_t get_stmt_expire_ts(sql::ObSQLSessionInfo *session_info);
int build_tx_param();
int start_sql_tx();
int end_sql_tx(const bool commit);
// abort tx is async, use rollback savepoint to sync release table lock
int create_implicit_savepoint();
int rollback_to_implicit_savepoint();
int lock_table_in_tx();
int init_ddl_param_for_inc_direct_load();
// full
int start_redef_table(const ObTableLoadParam &param);
int commit_redef_table();
int abort_redef_table();
private:
// direct load
int start_direct_load(const ObTableLoadParam &param, const common::ObIArray<int64_t> &idx_array);
int end_direct_load(const bool commit);
int add_tx_result_to_user_session();
int start_trans();
int check_trans_committed();
int check_merged();
int write_trans(int32_t session_id, const table::ObTableLoadObjRowArray &obj_rows);
int commit_trans();
private:
struct StmtCtx
{
public:
StmtCtx()
: tenant_id_(OB_INVALID_TENANT_ID),
table_id_(OB_INVALID_ID),
session_info_(nullptr),
tx_desc_(nullptr),
is_incremental_(false),
use_insert_into_select_tx_(false),
is_started_(false),
has_added_tx_result_(false)
{
}
void reset()
{
tenant_id_ = OB_INVALID_TENANT_ID;
table_id_ = OB_INVALID_ID;
ddl_param_.reset();
session_info_ = nullptr;
tx_desc_ = nullptr;
// tx_param_.reset();
savepoint_.reset();
is_incremental_ = false;
use_insert_into_select_tx_ = false;
is_started_ = false;
has_added_tx_result_ = false;
}
bool is_started() const { return is_started_; }
TO_STRING_KV(K_(tenant_id),
K_(table_id),
K_(ddl_param),
KP_(session_info),
KPC_(tx_desc),
K_(tx_param),
K_(savepoint),
KP_(is_incremental),
KP_(use_insert_into_select_tx),
KP_(is_started),
KP_(has_added_tx_result));
public:
uint64_t tenant_id_;
uint64_t table_id_;
ObTableLoadDDLParam ddl_param_;
sql::ObSQLSessionInfo *session_info_;
transaction::ObTxDesc *tx_desc_;
transaction::ObTxParam tx_param_;
transaction::ObTxSEQ savepoint_;
bool is_incremental_;
bool use_insert_into_select_tx_; // whether use the transaction of insert into select
bool is_started_;
bool has_added_tx_result_;
};
struct TransCtx
{
public:
@ -72,11 +143,12 @@ private:
common::ObIAllocator *allocator_;
ObTableLoadTableCtx *table_ctx_;
sql::ObLoadDataStat *job_stat_;
StmtCtx stmt_ctx_;
TransCtx trans_ctx_;
bool is_committed_;
table::ObTableLoadResultInfo result_info_;
bool is_inited_;
DISALLOW_COPY_AND_ASSIGN(ObTableLoadInstance);
};
} // namespace observer
} // namespace oceanbase
} // namespace oceanbase

120
src/observer/table_load/ob_table_load_merger.cpp
View File

@ -272,18 +272,13 @@ int ObTableLoadMerger::build_merge_ctx()
merge_param.target_table_id_ = store_ctx_->ctx_->ddl_param_.dest_table_id_;
merge_param.rowkey_column_num_ = store_ctx_->ctx_->schema_.rowkey_column_count_;
merge_param.store_column_count_ = store_ctx_->ctx_->schema_.store_column_count_;
merge_param.snapshot_version_ = store_ctx_->ctx_->ddl_param_.snapshot_version_;
merge_param.fill_cg_thread_cnt_ = param_.session_count_;
merge_param.table_data_desc_ = store_ctx_->table_data_desc_;
merge_param.datum_utils_ = &(store_ctx_->ctx_->schema_.datum_utils_);
merge_param.col_descs_ = &(store_ctx_->ctx_->schema_.column_descs_);
merge_param.lob_column_cnt_ = store_ctx_->ctx_->schema_.lob_column_cnt_;
merge_param.cmp_funcs_ = &(store_ctx_->ctx_->schema_.cmp_funcs_);
merge_param.is_heap_table_ = store_ctx_->ctx_->schema_.is_heap_table_;
merge_param.is_fast_heap_table_ = store_ctx_->is_fast_heap_table_;
merge_param.online_opt_stat_gather_ = param_.online_opt_stat_gather_;
merge_param.is_column_store_ = store_ctx_->ctx_->schema_.is_column_store_;
merge_param.fill_cg_thread_cnt_ = param_.session_count_;
merge_param.px_mode_ = param_.px_mode_;
merge_param.is_incremental_ = ObDirectLoadMethod::is_incremental(param_.method_);
merge_param.insert_mode_ = param_.insert_mode_;
merge_param.insert_table_ctx_ = store_ctx_->insert_table_ctx_;
merge_param.dml_row_handler_ = store_ctx_->error_row_handler_;
@ -398,115 +393,6 @@ int ObTableLoadMerger::build_merge_ctx()
return ret;
}
int ObTableLoadMerger::collect_dml_stat(ObTableLoadDmlStat &dml_stats)
{
int ret = OB_SUCCESS;
if (store_ctx_->is_fast_heap_table_) {
ObDirectLoadMultiMap<ObTabletID, ObDirectLoadFastHeapTable *> tables;
ObArray<ObTableLoadTransStore *> trans_store_array;
trans_store_array.set_tenant_id(MTL_ID());
if (OB_FAIL(tables.init())) {
LOG_WARN("fail to init table", KR(ret));
} else if (OB_FAIL(store_ctx_->get_committed_trans_stores(trans_store_array))) {
LOG_WARN("fail to get trans store", KR(ret));
} else {
for (int i = 0; OB_SUCC(ret) && i < trans_store_array.count(); ++i) {
ObTableLoadTransStore *trans_store = trans_store_array.at(i);
for (int j = 0; OB_SUCC(ret) && j < trans_store->session_store_array_.count(); ++j) {
ObTableLoadTransStore::SessionStore * session_store = trans_store->session_store_array_.at(j);
for (int k = 0 ; OB_SUCC(ret) && k < session_store->partition_table_array_.count(); ++k) {
ObIDirectLoadPartitionTable *table = session_store->partition_table_array_.at(k);
ObDirectLoadFastHeapTable *sstable = nullptr;
if (OB_ISNULL(sstable = dynamic_cast<ObDirectLoadFastHeapTable *>(table))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected not heap sstable", KR(ret), KPC(table));
} else {
const ObTabletID &tablet_id = sstable->get_tablet_id();
if (OB_FAIL(tables.add(tablet_id, sstable))) {
LOG_WARN("fail to add tables", KR(ret), KPC(sstable));
}
}
}
}
}
for (int i = 0; OB_SUCC(ret) && i < merge_ctx_.get_tablet_merge_ctxs().count(); ++i) {
ObDirectLoadTabletMergeCtx *tablet_ctx = merge_ctx_.get_tablet_merge_ctxs().at(i);
ObArray<ObDirectLoadFastHeapTable *> heap_table_array;
heap_table_array.set_tenant_id(MTL_ID());
if (OB_FAIL(tables.get(tablet_ctx->get_tablet_id(), heap_table_array))) {
LOG_WARN("get heap sstable failed", KR(ret));
} else if (OB_FAIL(tablet_ctx->collect_dml_stat(heap_table_array, dml_stats))) {
LOG_WARN("fail to collect sql statics", KR(ret));
}
}
}
} else {
for (int i = 0; OB_SUCC(ret) && i < merge_ctx_.get_tablet_merge_ctxs().count(); ++i) {
ObDirectLoadTabletMergeCtx *tablet_ctx = merge_ctx_.get_tablet_merge_ctxs().at(i);
ObArray<ObDirectLoadFastHeapTable *> empty_heap_table_array;
if (OB_FAIL(tablet_ctx->collect_dml_stat(empty_heap_table_array, dml_stats))) {
LOG_WARN("fail to collect sql statics", KR(ret));
}
}
}
return ret;
}
int ObTableLoadMerger::collect_sql_statistics(ObTableLoadSqlStatistics &sql_statistics)
{
int ret = OB_SUCCESS;
if (store_ctx_->is_fast_heap_table_) {
ObDirectLoadMultiMap<ObTabletID, ObDirectLoadFastHeapTable *> tables;
ObArray<ObTableLoadTransStore *> trans_store_array;
trans_store_array.set_tenant_id(MTL_ID());
if (OB_FAIL(tables.init())) {
LOG_WARN("fail to init table", KR(ret));
} else if (OB_FAIL(store_ctx_->get_committed_trans_stores(trans_store_array))) {
LOG_WARN("fail to get trans store", KR(ret));
} else {
for (int i = 0; OB_SUCC(ret) && i < trans_store_array.count(); ++i) {
ObTableLoadTransStore *trans_store = trans_store_array.at(i);
for (int j = 0; OB_SUCC(ret) && j < trans_store->session_store_array_.count(); ++j) {
ObTableLoadTransStore::SessionStore * session_store = trans_store->session_store_array_.at(j);
for (int k = 0 ; OB_SUCC(ret) && k < session_store->partition_table_array_.count(); ++k) {
ObIDirectLoadPartitionTable *table = session_store->partition_table_array_.at(k);
ObDirectLoadFastHeapTable *sstable = nullptr;
if (OB_ISNULL(sstable = dynamic_cast<ObDirectLoadFastHeapTable *>(table))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected not heap sstable", KR(ret), KPC(table));
} else {
const ObTabletID &tablet_id = sstable->get_tablet_id();
if (OB_FAIL(tables.add(tablet_id, sstable))) {
LOG_WARN("fail to add tables", KR(ret), KPC(sstable));
}
}
}
}
}
for (int i = 0; OB_SUCC(ret) && i < merge_ctx_.get_tablet_merge_ctxs().count(); ++i) {
ObDirectLoadTabletMergeCtx *tablet_ctx = merge_ctx_.get_tablet_merge_ctxs().at(i);
ObArray<ObDirectLoadFastHeapTable *> heap_table_array;
heap_table_array.set_tenant_id(MTL_ID());
if (OB_FAIL(tables.get(tablet_ctx->get_tablet_id(), heap_table_array))) {
LOG_WARN("get heap sstable failed", KR(ret));
} else if (OB_FAIL(tablet_ctx->collect_sql_statistics(heap_table_array, sql_statistics))) {
LOG_WARN("fail to collect sql statics", KR(ret));
}
}
}
} else {
for (int i = 0; OB_SUCC(ret) && i < merge_ctx_.get_tablet_merge_ctxs().count(); ++i) {
ObDirectLoadTabletMergeCtx *tablet_ctx = merge_ctx_.get_tablet_merge_ctxs().at(i);
ObArray<ObDirectLoadFastHeapTable *> heap_table_array;
heap_table_array.set_tenant_id(MTL_ID());
if (OB_FAIL(tablet_ctx->collect_sql_statistics(heap_table_array, sql_statistics))) {
LOG_WARN("fail to collect sql statics", KR(ret));
}
}
}
return ret;
}
int ObTableLoadMerger::build_rescan_ctx()
{
int ret = OB_SUCCESS;
@ -675,7 +561,7 @@ int ObTableLoadMerger::handle_merge_thread_finish(int ret_code)
if (!store_ctx_->is_fast_heap_table_) {
table_compact_ctx_.result_.release_all_table_data();
}
if (store_ctx_->ctx_->schema_.is_column_store_) {
if (store_ctx_->insert_table_ctx_->need_rescan()) {
if (OB_FAIL(build_rescan_ctx())) {
LOG_WARN("fail to build rescan ctx", KR(ret));
} else if (OB_FAIL(start_rescan())) {

2
src/observer/table_load/ob_table_load_merger.h
View File

@ -41,8 +41,6 @@ public:
int start();
void stop();
int handle_table_compact_success();
int collect_sql_statistics(table::ObTableLoadSqlStatistics &sql_statistics);
int collect_dml_stat(table::ObTableLoadDmlStat &dml_stats);
private:
int build_merge_ctx();
int build_rescan_ctx();

26
src/observer/table_load/ob_table_load_schema.cpp
View File

@ -253,6 +253,29 @@ int ObTableLoadSchema::get_tenant_optimizer_gather_stats_on_load(const uint64_t
return ret;
}
int ObTableLoadSchema::get_lob_meta_tid(
const uint64_t tenant_id,
const uint64_t data_table_id,
uint64_t &lob_meta_table_id)
{
int ret = OB_SUCCESS;
ObSchemaGetterGuard schema_guard;
const ObTableSchema *data_table_schema = nullptr;
lob_meta_table_id = OB_INVALID_ID;
if (OB_FAIL(ObTableLoadSchema::get_table_schema(
tenant_id, data_table_id, schema_guard, data_table_schema))) {
LOG_WARN("failed to get table schema", KR(ret), K(tenant_id), K(data_table_id));
} else if (OB_ISNULL(data_table_schema)) {
ret = OB_TABLE_NOT_EXIST;
LOG_WARN("data table not exist", KR(ret), K(tenant_id), K(data_table_id));
} else if (!data_table_schema->has_lob_aux_table()) {
// bypass
} else {
lob_meta_table_id = data_table_schema->get_aux_lob_meta_tid();
}
return ret;
}
ObTableLoadSchema::ObTableLoadSchema()
: allocator_("TLD_Schema"),
is_partitioned_table_(false),
@ -264,6 +287,7 @@ ObTableLoadSchema::ObTableLoadSchema()
store_column_count_(0),
lob_column_cnt_(0),
collation_type_(CS_TYPE_INVALID),
part_level_(PARTITION_LEVEL_ZERO),
schema_version_(0),
is_inited_(false)
{
@ -289,6 +313,7 @@ void ObTableLoadSchema::reset()
store_column_count_ = 0;
lob_column_cnt_ = 0;
collation_type_ = CS_TYPE_INVALID;
part_level_ = PARTITION_LEVEL_ZERO;
schema_version_ = 0;
column_descs_.reset();
multi_version_column_descs_.reset();
@ -332,6 +357,7 @@ int ObTableLoadSchema::init_table_schema(const ObTableSchema *table_schema)
has_autoinc_column_ = (table_schema->get_autoinc_column_id() != 0);
rowkey_column_count_ = table_schema->get_rowkey_column_num();
collation_type_ = table_schema->get_collation_type();
part_level_ = table_schema->get_part_level();
schema_version_ = table_schema->get_schema_version();
if (OB_FAIL(ObTableLoadUtils::deep_copy(table_schema->get_table_name_str(), table_name_,
allocator_))) {

4
src/observer/table_load/ob_table_load_schema.h
View File

@ -47,6 +47,9 @@ public:
common::ObIArray<int64_t> &column_idxs);
static int check_has_udt_column(const share::schema::ObTableSchema *table_schema, bool &bret);
static int get_tenant_optimizer_gather_stats_on_load(const uint64_t tenant_id, bool &value);
static int get_lob_meta_tid(const uint64_t tenant_id,
const uint64_t data_table_id,
uint64_t &lob_meta_table_id);
public:
ObTableLoadSchema();
~ObTableLoadSchema();
@ -78,6 +81,7 @@ public:
int64_t store_column_count_;
int64_t lob_column_cnt_;
common::ObCollationType collation_type_;
share::schema::ObPartitionLevel part_level_;
int64_t schema_version_;
// if it is a heap table, it contains hidden primary key column
// does not contain virtual generated columns

34
src/observer/table_load/ob_table_load_service.cpp
View File

@ -32,6 +32,7 @@ namespace observer
using namespace common;
using namespace lib;
using namespace share::schema;
using namespace storage;
using namespace table;
using namespace omt;
@ -476,14 +477,31 @@ int ObTableLoadService::check_support_direct_load(
ret = OB_NOT_SUPPORTED;
LOG_WARN("direct-load does not support table with materialized view log", KR(ret));
FORWARD_USER_ERROR_MSG(ret, "direct-load does not support table with materialized view log");
} else if (OB_UNLIKELY(ObDirectLoadMethod::is_incremental(method))) { // incremental direct-load
ret = OB_NOT_SUPPORTED;
LOG_WARN("incremental direct-load is not supported", KR(ret));
FORWARD_USER_ERROR_MSG(ret, "incremental direct-load is not supported");
} else if (ObDirectLoadMethod::is_full(method)) { // full direct-load
if (OB_UNLIKELY(!ObDirectLoadInsertMode::is_valid_for_full_method(insert_mode))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected insert mode for full direct-load", KR(ret), K(method), K(insert_mode));
} else if (ObDirectLoadMethod::is_incremental(method)) { // incremental direct-load
uint64_t compat_version = 0;
if (OB_UNLIKELY(ObDirectLoadInsertMode::INC_REPLACE != insert_mode)) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("using incremental direct-load without inc_replace is not supported", KR(ret));
FORWARD_USER_ERROR_MSG(ret, "using incremental direct-load without inc_replace is not supported");
} else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) {
LOG_WARN("fail to get data version", KR(ret), K(tenant_id));
} else if (compat_version < DATA_VERSION_4_3_1_0) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("version lower than 4.3.1.0 does not support incremental direct-load", KR(ret));
FORWARD_USER_ERROR_MSG(ret, "version lower than 4.3.1.0 does not support incremental direct-load");
} else if (table_schema->get_index_tid_count() > 0) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("incremental direct-load does not support table with indexes", KR(ret));
FORWARD_USER_ERROR_MSG(ret, "incremental direct-load does not support table with indexes");
} else if (table_schema->get_foreign_key_infos().count() > 0) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("incremental direct-load does not support table with foreign keys", KR(ret));
FORWARD_USER_ERROR_MSG(ret, "incremental direct-load does not support table with foreign keys");
} else if (ObDirectLoadMethod::is_full(method)) { // full direct-load
if (OB_UNLIKELY(!ObDirectLoadInsertMode::is_valid_for_full_method(insert_mode))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected insert mode for full direct-load", KR(ret), K(method), K(insert_mode));
}
}
}
}

59
src/observer/table_load/ob_table_load_store.cpp
View File

@ -25,8 +25,8 @@
#include "observer/table_load/ob_table_load_utils.h"
#include "storage/direct_load/ob_direct_load_insert_table_ctx.h"
#include "share/stat/ob_opt_stat_monitor_manager.h"
#include "share/stat/ob_dbms_stats_utils.h"
#include "storage/blocksstable/ob_sstable.h"
#include "share/table/ob_table_load_dml_stat.h"
namespace oceanbase
{
@ -34,6 +34,7 @@ namespace observer
{
using namespace common;
using namespace table;
using namespace transaction;
ObTableLoadStore::ObTableLoadStore(ObTableLoadTableCtx *ctx)
: ctx_(ctx), param_(ctx->param_), store_ctx_(ctx->store_ctx_), is_inited_(false)
@ -299,31 +300,35 @@ int ObTableLoadStore::start_merge()
return ret;
}
int ObTableLoadStore::commit(ObTableLoadResultInfo &result_info)
int ObTableLoadStore::commit(ObTableLoadResultInfo &result_info,
ObTableLoadSqlStatistics &sql_statistics,
ObTxExecResult &trans_result)
{
int ret = OB_SUCCESS;
sql_statistics.reset();
trans_result.reset();
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadStore not init", KR(ret), KP(this));
} else {
LOG_INFO("store commit");
ObTransService *txs = nullptr;
ObMutexGuard guard(store_ctx_->get_op_lock());
ObTableLoadDmlStat dml_stats;
ObTableLoadSqlStatistics sql_statistics;
if (OB_FAIL(store_ctx_->check_status(ObTableLoadStatusType::MERGED))) {
if (OB_ISNULL(MTL(ObTransService *))) {
ret = OB_ERR_SYS;
LOG_WARN("trans service is null", KR(ret));
} else if (OB_FAIL(store_ctx_->check_status(ObTableLoadStatusType::MERGED))) {
LOG_WARN("fail to check store status", KR(ret));
} else if (OB_FAIL(store_ctx_->insert_table_ctx_->commit(dml_stats, sql_statistics))) {
LOG_WARN("fail to commit insert table", KR(ret));
} else if (ctx_->schema_.has_autoinc_column_ && OB_FAIL(store_ctx_->commit_autoinc_value())) {
LOG_WARN("fail to commit sync auto increment value", KR(ret));
} else if (param_.online_opt_stat_gather_ &&
OB_FAIL(store_ctx_->merger_->collect_sql_statistics(sql_statistics))) {
LOG_WARN("fail to collect sql stats", KR(ret));
} else if (param_.online_opt_stat_gather_ &&
OB_FAIL(commit_sql_statistics(sql_statistics))) {
LOG_WARN("fail to commit sql stats", KR(ret));
} else if (OB_FAIL(store_ctx_->merger_->collect_dml_stat(dml_stats))) {
LOG_WARN("fail to build dml stat", KR(ret));
} else if (OB_FAIL(ObOptStatMonitorManager::update_dml_stat_info_from_direct_load(dml_stats.dml_stat_array_))) {
LOG_WARN("fail to update dml stat info", KR(ret));
} else if (ObDirectLoadMethod::is_incremental(param_.method_) &&
txs->get_tx_exec_result(*ctx_->session_info_->get_tx_desc(), trans_result)) {
LOG_WARN("fail to get tx exec result", KR(ret));
} else if (OB_FAIL(store_ctx_->set_status_commit())) {
LOG_WARN("fail to set store status commit", KR(ret));
} else {
@ -334,36 +339,6 @@ int ObTableLoadStore::commit(ObTableLoadResultInfo &result_info)
return ret;
}
int ObTableLoadStore::commit_sql_statistics(const ObTableLoadSqlStatistics &sql_statistics)
{
int ret = OB_SUCCESS;
const uint64_t tenant_id = MTL_ID();
const uint64_t table_id = param_.table_id_;
ObSchemaGetterGuard schema_guard;
const ObTableSchema *table_schema = nullptr;
ObArray<ObOptColumnStat *> part_column_stats;
ObArray<ObOptTableStat *> part_table_stats;
part_column_stats.set_tenant_id(MTL_ID());
part_table_stats.set_tenant_id(MTL_ID());
if (sql_statistics.is_empty()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sql statistics is empty", K(ret));
} else if (OB_FAIL(ObTableLoadSchema::get_table_schema(tenant_id, table_id, schema_guard,
table_schema))) {
LOG_WARN("fail to get table schema", KR(ret), K(tenant_id), K(table_id));
} else if (OB_FAIL(sql_statistics.get_col_stat_array(part_column_stats))) {
LOG_WARN("failed to get column stat array");
} else if (OB_FAIL(sql_statistics.get_table_stat_array(part_table_stats))) {
LOG_WARN("failed to get table stat array");
} else if (OB_FAIL(ObDbmsStatsUtils::split_batch_write(
&schema_guard, store_ctx_->ctx_->session_info_, GCTX.sql_proxy_, part_table_stats,
part_column_stats))) {
LOG_WARN("failed to batch write stats", K(ret), K(sql_statistics.table_stat_array_),
K(sql_statistics.col_stat_array_));
}
return ret;
}
int ObTableLoadStore::get_status(ObTableLoadStatusType &status, int &error_code)
{
int ret = OB_SUCCESS;

11
src/observer/table_load/ob_table_load_store.h
View File

@ -17,10 +17,13 @@
#include "share/table/ob_table_load_array.h"
#include "share/table/ob_table_load_define.h"
#include "share/table/ob_table_load_row_array.h"
#include "share/table/ob_table_load_sql_statistics.h"
namespace oceanbase
{
namespace table
{
class ObTableLoadSqlStatistics;
} // namespace table
namespace observer
{
class ObTableLoadTableCtx;
@ -46,11 +49,11 @@ public:
int confirm_begin();
int pre_merge(const table::ObTableLoadArray<table::ObTableLoadTransId> &committed_trans_id_array);
int start_merge();
int commit(table::ObTableLoadResultInfo &result_info);
int commit(table::ObTableLoadResultInfo &result_info,
table::ObTableLoadSqlStatistics &sql_statistics,
transaction::ObTxExecResult &trans_result);
int get_status(table::ObTableLoadStatusType &status, int &error_code);
int heart_beat();
private:
int commit_sql_statistics(const table::ObTableLoadSqlStatistics &sql_statistics);
private:
class MergeTaskProcessor;
class MergeTaskCallback;

75
src/observer/table_load/ob_table_load_store_ctx.cpp
View File

@ -83,14 +83,6 @@ int ObTableLoadStoreCtx::init(
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(partition_id_array), K(target_partition_id_array));
} else {
ObDirectLoadInsertTableParam insert_table_param;
insert_table_param.table_id_ = ctx_->param_.table_id_;
insert_table_param.schema_version_ = ctx_->ddl_param_.schema_version_;
insert_table_param.snapshot_version_ = ctx_->ddl_param_.snapshot_version_;
insert_table_param.ddl_task_id_ = ctx_->ddl_param_.task_id_;
insert_table_param.execution_id_ = 1; //仓氐说暂时设置为1，不然后面检测过不了
insert_table_param.data_version_ = ctx_->ddl_param_.data_version_;
insert_table_param.reserved_parallel_ = ctx_->param_.session_count_;
for (int64_t i = 0; OB_SUCC(ret) && i < partition_id_array.count(); ++i) {
const ObLSID &ls_id = partition_id_array[i].ls_id_;
const ObTableLoadPartitionId &part_tablet_id = partition_id_array[i].part_tablet_id_;
@ -167,10 +159,6 @@ int ObTableLoadStoreCtx::init(
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(insert_table_param.ls_partition_ids_.assign(ls_partition_ids_))) {
LOG_WARN("fail to assign ls tablet ids", KR(ret));
} else if (OB_FAIL(insert_table_param.target_ls_partition_ids_.assign(target_ls_partition_ids_))) {
LOG_WARN("fail to assign ls tablet ids", KR(ret));
}
// init trans_allocator_
else if (OB_FAIL(trans_allocator_.init("TLD_STransPool", ctx_->param_.tenant_id_))) {
@ -208,12 +196,41 @@ int ObTableLoadStoreCtx::init(
LOG_WARN("fail to init merger", KR(ret));
}
// init insert_table_ctx_
else if (OB_ISNULL(insert_table_ctx_ =
if (OB_SUCC(ret)) {
ObDirectLoadInsertTableParam insert_table_param;
insert_table_param.table_id_ = ctx_->ddl_param_.dest_table_id_;
insert_table_param.schema_version_ = ctx_->ddl_param_.schema_version_;
insert_table_param.snapshot_version_ = ctx_->ddl_param_.snapshot_version_;
insert_table_param.ddl_task_id_ = ctx_->ddl_param_.task_id_;
insert_table_param.data_version_ = ctx_->ddl_param_.data_version_;
insert_table_param.parallel_ = ctx_->param_.session_count_;
insert_table_param.reserved_parallel_ = is_fast_heap_table_ ? ctx_->param_.session_count_ : 0;
insert_table_param.rowkey_column_count_ = ctx_->schema_.rowkey_column_count_;
insert_table_param.column_count_ = ctx_->schema_.store_column_count_;
insert_table_param.lob_column_count_ = ctx_->schema_.lob_column_cnt_;
insert_table_param.is_partitioned_table_ = ctx_->schema_.is_partitioned_table_;
insert_table_param.is_heap_table_ = ctx_->schema_.is_heap_table_;
insert_table_param.is_column_store_ = ctx_->schema_.is_column_store_;
insert_table_param.online_opt_stat_gather_ = ctx_->param_.online_opt_stat_gather_;
insert_table_param.is_incremental_ = ObDirectLoadMethod::is_incremental(ctx_->param_.method_);
insert_table_param.datum_utils_ = &(ctx_->schema_.datum_utils_);
insert_table_param.col_descs_ = &(ctx_->schema_.column_descs_);
insert_table_param.cmp_funcs_ = &(ctx_->schema_.cmp_funcs_);
if (insert_table_param.is_incremental_ && OB_FAIL(init_trans_param(insert_table_param.trans_param_))) {
LOG_WARN("fail to init trans param", KR(ret));
} else if (OB_FAIL(ObTableLoadSchema::get_lob_meta_tid(ctx_->param_.tenant_id_,
insert_table_param.table_id_, insert_table_param.lob_meta_tid_))) {
LOG_WARN("fail to get lob meta tid", KR(ret),
K(ctx_->param_.tenant_id_), K(insert_table_param.table_id_));
} else if (OB_ISNULL(insert_table_ctx_ =
OB_NEWx(ObDirectLoadInsertTableContext, (&allocator_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObDirectLoadInsertTableContext", KR(ret));
} else if (OB_FAIL(insert_table_ctx_->init(insert_table_param))) {
LOG_WARN("fail to init insert table ctx", KR(ret));
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObDirectLoadInsertTableContext", KR(ret));
} else if (OB_FAIL(insert_table_ctx_->init(insert_table_param, ls_partition_ids_, target_ls_partition_ids_))) {
LOG_WARN("fail to init insert table ctx", KR(ret));
}
}
if (OB_FAIL(ret)) {
}
// init tmp_file_mgr_
else if (OB_ISNULL(tmp_file_mgr_ = OB_NEWx(ObDirectLoadTmpFileManager, (&allocator_)))) {
@ -400,6 +417,30 @@ bool ObTableLoadStoreCtx::check_heart_beat_expired(const uint64_t expired_time_u
return ObTimeUtil::current_time() > (last_heart_beat_ts_ + expired_time_us);
}
int ObTableLoadStoreCtx::init_trans_param(ObDirectLoadTransParam &trans_param)
{
int ret = OB_SUCCESS;
ObSQLSessionInfo *session_info = nullptr;
transaction::ObTxDesc *tx_desc = nullptr;
trans_param.reset();
if (OB_ISNULL(session_info = ctx_->session_info_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null session info", KR(ret));
} else if (OB_UNLIKELY(!session_info->is_in_transaction())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected not in transaction", KR(ret));
} else if (OB_ISNULL(tx_desc = session_info->get_tx_desc())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null tx desc", KR(ret));
} else {
trans_param.tx_desc_ = tx_desc;
trans_param.tx_id_ = tx_desc->get_tx_id();
trans_param.tx_seq_ = tx_desc->inc_and_get_tx_seq(0);
LOG_INFO("init trans param", K(trans_param));
}
return ret;
}
int ObTableLoadStoreCtx::generate_autoinc_params(AutoincParam &autoinc_param)
{
int ret = OB_SUCCESS;

2
src/observer/table_load/ob_table_load_store_ctx.h
View File

@ -26,6 +26,7 @@ namespace oceanbase
{
namespace storage
{
class ObDirectLoadTransParam;
class ObDirectLoadInsertTableContext;
class ObDirectLoadTmpFileManager;
} // namespace storage
@ -130,6 +131,7 @@ private:
int alloc_trans_ctx(const table::ObTableLoadTransId &trans_id, ObTableLoadTransCtx *&trans_ctx);
int alloc_trans(const table::ObTableLoadTransId &trans_id, ObTableLoadStoreTrans *&trans);
int init_session_ctx_array();
int init_trans_param(ObDirectLoadTransParam &trans_param);
int generate_autoinc_params(share::AutoincParam &autoinc_param);
int init_sequence();
public:

2
src/observer/table_load/ob_table_load_struct.cpp
View File

@ -21,7 +21,7 @@ namespace observer
using namespace common;
using namespace table;
OB_SERIALIZE_MEMBER_SIMPLE(ObTableLoadUniqueKey, table_id_, task_id_);
OB_SERIALIZE_MEMBER(ObTableLoadUniqueKey, table_id_, task_id_);
OB_SERIALIZE_MEMBER(ObTableLoadDDLParam,
dest_table_id_,

3
src/observer/table_load/ob_table_load_struct.h
View File

@ -131,6 +131,7 @@ enum class ObTableLoadExeMode {
struct ObTableLoadParam
{
public:
ObTableLoadParam()
: tenant_id_(common::OB_INVALID_ID),
table_id_(common::OB_INVALID_ID),
@ -191,6 +192,7 @@ struct ObTableLoadParam
K_(session_count),
K_(batch_size),
K_(max_error_row_count),
K_(sql_mode),
K_(column_count),
K_(need_sort),
K_(px_mode),
@ -241,6 +243,7 @@ public:
schema_version_ = 0;
snapshot_version_ = 0;
data_version_ = 0;
cluster_version_ = 0;
}
bool is_valid() const
{

5
src/observer/table_load/ob_table_load_trans_bucket_writer.cpp
View File

@ -191,8 +191,9 @@ int ObTableLoadTransBucketWriter::write(int32_t session_id, ObTableLoadObjRowArr
}
} else {
if (coordinator_ctx_->partition_calc_.is_partition_with_autoinc_ &&
OB_FAIL(handle_partition_with_autoinc_identity(session_ctx, obj_rows,
trans_ctx_->ctx_->session_info_->get_sql_mode(), session_id))) {
OB_FAIL(handle_partition_with_autoinc_identity(
session_ctx, obj_rows, coordinator_ctx_->ctx_->session_info_->get_sql_mode(),
session_id))) {
LOG_WARN("fail to handle partition column with autoincrement or identity", KR(ret));
} else if (OB_FAIL(write_for_partitioned(session_ctx, obj_rows))) {
LOG_WARN("fail to write for partitioned", KR(ret));

39
src/observer/table_load/ob_table_load_trans_store.cpp
View File

@ -119,7 +119,10 @@ ObTableLoadTransStoreWriter::ObTableLoadTransStoreWriter(ObTableLoadTransStore *
param_(trans_ctx_->ctx_->param_),
allocator_("TLD_TSWriter"),
table_data_desc_(nullptr),
lob_inrow_threshold_(0),
ref_count_(0),
is_incremental_(false),
is_inc_replace_(false),
is_inited_(false)
{
allocator_.set_tenant_id(MTL_ID());
@ -158,16 +161,18 @@ int ObTableLoadTransStoreWriter::init()
collation_type_ = trans_ctx_->ctx_->schema_.collation_type_;
if (OB_FAIL(init_session_ctx_array())) {
LOG_WARN("fail to init session ctx array", KR(ret));
} else if (OB_FAIL(init_column_schemas())) {
LOG_WARN("fail to init column schemas", KR(ret));
} else if (OB_FAIL(init_column_schemas_and_lob_info())) {
LOG_WARN("fail to init column schemas and lob info", KR(ret));
} else {
is_incremental_ = ObDirectLoadMethod::is_incremental(store_ctx_->ctx_->param_.method_);
is_inc_replace_ = (ObDirectLoadInsertMode::INC_REPLACE == store_ctx_->ctx_->param_.insert_mode_);
is_inited_ = true;
}
}
return ret;
}
int ObTableLoadTransStoreWriter::init_column_schemas()
int ObTableLoadTransStoreWriter::init_column_schemas_and_lob_info()
{
int ret = OB_SUCCESS;
const ObIArray<ObColDesc> &column_descs = store_ctx_->ctx_->schema_.column_descs_;
@ -184,6 +189,9 @@ int ObTableLoadTransStoreWriter::init_column_schemas()
LOG_WARN("failed to push back column schema", K(ret), K(i), KPC(column_schema));
}
}
if (OB_SUCC(ret)) {
lob_inrow_threshold_ = table_schema->get_lob_inrow_threshold();
}
return ret;
}
@ -206,17 +214,11 @@ int ObTableLoadTransStoreWriter::init_session_ctx_array()
}
}
ObDirectLoadTableStoreParam param;
param.snapshot_version_ = trans_ctx_->ctx_->ddl_param_.snapshot_version_;
param.table_data_desc_ = *table_data_desc_;
param.datum_utils_ = &(trans_ctx_->ctx_->schema_.datum_utils_);
param.col_descs_ = &(trans_ctx_->ctx_->schema_.column_descs_);
param.lob_column_cnt_ = trans_ctx_->ctx_->schema_.lob_column_cnt_;
param.cmp_funcs_ = &(trans_ctx_->ctx_->schema_.cmp_funcs_);
param.file_mgr_ = trans_ctx_->ctx_->store_ctx_->tmp_file_mgr_;
param.is_multiple_mode_ = trans_ctx_->ctx_->store_ctx_->is_multiple_mode_;
param.is_fast_heap_table_ = trans_ctx_->ctx_->store_ctx_->is_fast_heap_table_;
param.online_opt_stat_gather_ = trans_ctx_->ctx_->param_.online_opt_stat_gather_;
param.px_mode_ = trans_ctx_->ctx_->param_.px_mode_;
param.insert_table_ctx_ = trans_ctx_->ctx_->store_ctx_->insert_table_ctx_;
param.dml_row_handler_ = trans_ctx_->ctx_->store_ctx_->error_row_handler_;
for (int64_t i = 0; OB_SUCC(ret) && i < session_count; ++i) {
@ -334,7 +336,9 @@ int ObTableLoadTransStoreWriter::write(int32_t session_id,
const ObNewRow &row = row_array.at(i);
for (int64_t j = 0; OB_SUCC(ret) && (j < table_data_desc_->column_count_); ++j) {
const ObObj &obj = row.cells_[j];
if (OB_FAIL(session_ctx.datum_row_.storage_datums_[j].from_obj_enhance(obj))) {
if (OB_FAIL(check_support_obj(obj))) {
LOG_WARN("failed to check support obj", KR(ret), K(obj));
} else if (OB_FAIL(session_ctx.datum_row_.storage_datums_[j].from_obj_enhance(obj))) {
LOG_WARN("fail to from obj enhance", KR(ret), K(obj));
}
}
@ -411,6 +415,8 @@ int ObTableLoadTransStoreWriter::cast_row(ObArenaAllocator &cast_allocator,
if (!is_null_autoinc && OB_FAIL(ObTableLoadObjCaster::cast_obj(cast_obj_ctx, column_schema,
row.cells_[i], out_obj))) {
LOG_WARN("fail to cast obj and check", KR(ret), K(i), K(row.cells_[i]));
} else if (OB_FAIL(check_support_obj(out_obj))) {
LOG_WARN("failed to check support obj", KR(ret), K(out_obj));
} else if (OB_FAIL(datum_row.storage_datums_[i].from_obj_enhance(out_obj))) {
LOG_WARN("fail to from obj enhance", KR(ret), K(out_obj));
} else if (column_schema->is_autoincrement() &&
@ -497,5 +503,18 @@ int ObTableLoadTransStoreWriter::write_row_to_table_store(ObDirectLoadTableStore
return ret;
}
int ObTableLoadTransStoreWriter::check_support_obj(const ObObj &obj)
{
int ret = OB_SUCCESS;
if (is_incremental_ && is_inc_replace_) {
if (obj.is_lob_storage() && OB_UNLIKELY(obj.get_data_length() > lob_inrow_threshold_)) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("incremental direct-load does not support outrow lob", KR(ret), K(obj));
FORWARD_USER_ERROR_MSG(ret, "incremental direct-load does not support outrow lob");
}
}
return ret;
}
} // namespace observer
} // namespace oceanbase

6
src/observer/table_load/ob_table_load_trans_store.h
View File

@ -85,7 +85,7 @@ public:
private:
class SessionContext;
int init_session_ctx_array();
int init_column_schemas();
int init_column_schemas_and_lob_info();
int cast_row(common::ObArenaAllocator &cast_allocator, ObDataTypeCastParams cast_params,
const common::ObNewRow &row, blocksstable::ObDatumRow &datum_row,
int32_t session_id);
@ -100,6 +100,7 @@ private:
const common::ObTabletID &tablet_id,
const table::ObTableLoadSequenceNo &seq_no,
const blocksstable::ObDatumRow &datum_row);
int check_support_obj(const common::ObObj &obj);
private:
ObTableLoadTransStore *const trans_store_;
ObTableLoadTransCtx *const trans_ctx_;
@ -126,7 +127,10 @@ private:
int64_t extra_buf_size_;
};
SessionContext *session_ctx_array_;
int64_t lob_inrow_threshold_; // for incremental direct load
int64_t ref_count_ CACHE_ALIGNED;
bool is_incremental_;
bool is_inc_replace_;
bool is_inited_;
ObSchemaGetterGuard schema_guard_;
};

26
src/observer/virtual_table/ob_all_virtual_memstore_info.cpp
View File

@ -178,7 +178,7 @@ int ObAllVirtualMemstoreInfo::get_next_memtable(memtable::ObMemtable *&mt)
return ret;
}
void ObAllVirtualMemstoreInfo::get_freeze_time_dist(const memtable::ObMtStat& mt_stat)
void ObAllVirtualMemstoreInfo::get_freeze_time_dist(const ObMtStat& mt_stat)
{
memset(freeze_time_dist_, 0, 128);
int64_t ready_for_flush_cost_time = (mt_stat.ready_for_flush_time_ - mt_stat.frozen_time_) / 1000;
@ -212,7 +212,7 @@ int ObAllVirtualMemstoreInfo::process_curr_tenant(ObNewRow *&row)
ret = OB_ERR_UNEXPECTED;
SERVER_LOG(WARN, "mt shouldn't NULL here", K(ret), K(mt));
} else {
memtable::ObMtStat& mt_stat = mt->get_mt_stat();
ObMtStat& mt_stat = mt->get_mt_stat();
const int64_t col_count = output_column_ids_.count();
for (int64_t i = 0; OB_SUCC(ret) && i < col_count; ++i) {
uint64_t col_id = output_column_ids_.at(i);
@ -315,27 +315,7 @@ int ObAllVirtualMemstoreInfo::process_curr_tenant(ObNewRow *&row)
break;
case OB_APP_MIN_COLUMN_ID + 22:
// freeze_state
switch (mt->get_freeze_state()) {
case ObMemtableFreezeState::INVALID:
cur_row_.cells_[i].set_varchar("INVALID");
break;
case ObMemtableFreezeState::NOT_READY_FOR_FLUSH:
cur_row_.cells_[i].set_varchar("NOT_READY_FOR_FLUSH");
break;
case ObMemtableFreezeState::READY_FOR_FLUSH:
cur_row_.cells_[i].set_varchar("READY_FOR_FLUSH");
break;
case ObMemtableFreezeState::FLUSHED:
cur_row_.cells_[i].set_varchar("FLUSHED");
break;
case ObMemtableFreezeState::RELEASED:
cur_row_.cells_[i].set_varchar("RELEASED");
break;
default:
ret = OB_ERR_UNEXPECTED;
SERVER_LOG(WARN, "invalid freeze state", K(ret), K(col_id));
break;
}
cur_row_.cells_[i].set_varchar(storage::TABLET_MEMTABLE_FREEZE_STATE_TO_STR(mt->get_freeze_state()));
cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
break;
case OB_APP_MIN_COLUMN_ID + 23:

2
src/observer/virtual_table/ob_all_virtual_memstore_info.h
View File

@ -50,7 +50,7 @@ private:
int get_next_ls(ObLS *&ls);
int get_next_tablet(storage::ObTabletHandle &tablet_handle);
int get_next_memtable(memtable::ObMemtable *&mt);
void get_freeze_time_dist(const memtable::ObMtStat& mt_stat);
void get_freeze_time_dist(const ObMtStat& mt_stat);
private:
common::ObAddr addr_;
int64_t ls_id_;

4
src/observer/virtual_table/ob_all_virtual_table_mgr.cpp
View File

@ -235,7 +235,7 @@ int ObAllVirtualTableMgr::process_curr_tenant(common::ObNewRow *&row)
case SIZE: {
int64_t size = 0;
if (table->is_memtable()) {
size = static_cast<memtable::ObIMemtable *>(table)->get_occupied_size();
size = static_cast<ObIMemtable *>(table)->get_occupied_size();
} else if (table->is_sstable()) {
size = static_cast<blocksstable::ObSSTable *>(table)->get_occupy_size();
}
@ -276,7 +276,7 @@ int ObAllVirtualTableMgr::process_curr_tenant(common::ObNewRow *&row)
case IS_ACTIVE: {
bool is_active = false;
if (table->is_memtable()) {
is_active = static_cast<memtable::ObIMemtable *>(table)->is_active_memtable();
is_active = static_cast<ObIMemtable *>(table)->is_active_memtable();
}
cur_row_.cells_[i].set_varchar(is_active ? "YES" : "NO");
cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));

2
src/observer/virtual_table/ob_all_virtual_tenant_memstore_allocator_info.cpp
View File

@ -38,7 +38,7 @@ public:
ObLSID ls_id;
item.protection_clock_ = handle->get_protection_clock();
item.is_active_ = handle->is_active();
item.ls_id_ = (OB_SUCCESS == mt.get_ls_id(ls_id)) ? ls_id.id() : ObLSID::INVALID_LS_ID;
item.ls_id_ = mt.get_ls_id().id();
item.tablet_id_ = mt.get_key().tablet_id_.id();
item.scn_range_ = mt.get_scn_range();
item.mt_addr_ = &mt;

2
src/share/inner_table/ob_inner_table_schema.21101_21150.cpp
View File

@ -60,7 +60,7 @@ int ObInnerTableSchema::gv_ob_sstables_schema(ObTableSchema &table_schema)
table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
if (OB_SUCC(ret)) {
if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT M.SVR_IP, M.SVR_PORT, (case M.TABLE_TYPE when 0 then 'MEMTABLE' when 1 then 'TX_DATA_MEMTABLE' when 2 then 'TX_CTX_MEMTABLE' when 3 then 'LOCK_MEMTABLE' when 10 then 'MAJOR' when 11 then 'MINOR' when 12 then 'MINI' when 13 then 'META' when 14 then 'DDL_DUMP' when 15 then 'REMOTE_LOGICAL_MINOR' when 16 then 'DDL_MEM' when 17 then 'CO_MAJOR' when 18 then 'NORMAL_CG' when 19 then 'ROWKEY_CG' when 20 then 'DDL_MERGE' else 'INVALID' end) as TABLE_TYPE, M.TENANT_ID, M.LS_ID, M.TABLET_ID, M.CG_IDX, M.START_LOG_SCN, M.END_LOG_SCN, M.DATA_CHECKSUM, M.SIZE, M.REF, M.UPPER_TRANS_VERSION, M.IS_ACTIVE, M.CONTAIN_UNCOMMITTED_ROW FROM oceanbase.__all_virtual_table_mgr M )__"))) {
if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT M.SVR_IP, M.SVR_PORT, (case M.TABLE_TYPE when 0 then 'MEMTABLE' when 1 then 'TX_DATA_MEMTABLE' when 2 then 'TX_CTX_MEMTABLE' when 3 then 'LOCK_MEMTABLE' when 4 then 'DIRECT_LOAD_MEMTABLE' when 10 then 'MAJOR' when 11 then 'MINOR' when 12 then 'MINI' when 13 then 'META' when 14 then 'DDL_DUMP' when 15 then 'REMOTE_LOGICAL_MINOR' when 16 then 'DDL_MEM' when 17 then 'CO_MAJOR' when 18 then 'NORMAL_CG' when 19 then 'ROWKEY_CG' when 20 then 'DDL_MERGE' else 'INVALID' end) as TABLE_TYPE, M.TENANT_ID, M.LS_ID, M.TABLET_ID, M.CG_IDX, M.START_LOG_SCN, M.END_LOG_SCN, M.DATA_CHECKSUM, M.SIZE, M.REF, M.UPPER_TRANS_VERSION, M.IS_ACTIVE, M.CONTAIN_UNCOMMITTED_ROW FROM oceanbase.__all_virtual_table_mgr M )__"))) {
LOG_ERROR("fail to set view_definition", K(ret));
}
}

2
src/share/inner_table/ob_inner_table_schema.28051_28100.cpp
View File

@ -310,7 +310,7 @@ int ObInnerTableSchema::gv_ob_sstables_ora_schema(ObTableSchema &table_schema)
table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
if (OB_SUCC(ret)) {
if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT M.SVR_IP, M.SVR_PORT, (case M.TABLE_TYPE when 0 then 'MEMTABLE' when 1 then 'TX_DATA_MEMTABLE' when 2 then 'TX_CTX_MEMTABLE' when 3 then 'LOCK_MEMTABLE' when 10 then 'MAJOR' when 11 then 'MINOR' when 12 then 'MINI' when 13 then 'META' when 14 then 'DDL_DUMP' when 15 then 'REMOTE_LOGICAL_MINOR' when 16 then 'IMC_SEGMENT' when 20 then 'DDL_MERGE' else 'INVALID' end) as TABLE_TYPE, M.LS_ID, M.TABLET_ID, M.START_LOG_SCN, M.END_LOG_SCN, M."SIZE", M.REF, M.UPPER_TRANS_VERSION, M.IS_ACTIVE, M.CONTAIN_UNCOMMITTED_ROW FROM SYS.ALL_VIRTUAL_TABLE_MGR M )__"))) {
if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT M.SVR_IP, M.SVR_PORT, (case M.TABLE_TYPE when 0 then 'MEMTABLE' when 1 then 'TX_DATA_MEMTABLE' when 2 then 'TX_CTX_MEMTABLE' when 3 then 'LOCK_MEMTABLE' when 4 then 'DIRECT_LOAD_MEMTABLE' when 10 then 'MAJOR' when 11 then 'MINOR' when 12 then 'MINI' when 13 then 'META' when 14 then 'DDL_DUMP' when 15 then 'REMOTE_LOGICAL_MINOR' when 16 then 'IMC_SEGMENT' when 20 then 'DDL_MERGE' else 'INVALID' end) as TABLE_TYPE, M.LS_ID, M.TABLET_ID, M.START_LOG_SCN, M.END_LOG_SCN, M."SIZE", M.REF, M.UPPER_TRANS_VERSION, M.IS_ACTIVE, M.CONTAIN_UNCOMMITTED_ROW FROM SYS.ALL_VIRTUAL_TABLE_MGR M )__"))) {
LOG_ERROR("fail to set view_definition", K(ret));
}
}

4
src/share/inner_table/ob_inner_table_schema_def.py
View File

@ -17771,7 +17771,7 @@ SELECT
M.SVR_PORT,
(case M.TABLE_TYPE
when 0 then 'MEMTABLE' when 1 then 'TX_DATA_MEMTABLE' when 2 then 'TX_CTX_MEMTABLE'
when 3 then 'LOCK_MEMTABLE' when 10 then 'MAJOR' when 11 then 'MINOR'
when 3 then 'LOCK_MEMTABLE' when 4 then 'DIRECT_LOAD_MEMTABLE' when 10 then 'MAJOR' when 11 then 'MINOR'
when 12 then 'MINI' when 13 then 'META'
when 14 then 'DDL_DUMP' when 15 then 'REMOTE_LOGICAL_MINOR' when 16 then 'DDL_MEM'
when 17 then 'CO_MAJOR' when 18 then 'NORMAL_CG' when 19 then 'ROWKEY_CG' when 20 then 'DDL_MERGE'
@ -55756,7 +55756,7 @@ SELECT
M.SVR_PORT,
(case M.TABLE_TYPE
when 0 then 'MEMTABLE' when 1 then 'TX_DATA_MEMTABLE' when 2 then 'TX_CTX_MEMTABLE'
when 3 then 'LOCK_MEMTABLE' when 10 then 'MAJOR' when 11 then 'MINOR'
when 3 then 'LOCK_MEMTABLE' when 4 then 'DIRECT_LOAD_MEMTABLE' when 10 then 'MAJOR' when 11 then 'MINOR'
when 12 then 'MINI' when 13 then 'META'
when 14 then 'DDL_DUMP' when 15 then 'REMOTE_LOGICAL_MINOR' when 16 then 'IMC_SEGMENT' when 20 then 'DDL_MERGE'
else 'INVALID'

1
src/share/ob_ddl_common.h
View File

@ -317,6 +317,7 @@ static inline bool is_direct_load_retry_err(const int ret)
|| ret == OB_NOT_MASTER
|| ret == OB_TASK_EXPIRED
|| ret == OB_REPLICA_NOT_READABLE
|| ret == OB_TRANS_CTX_NOT_EXIST
;
}

102
src/share/ob_rpc_struct.cpp
View File

@ -9872,7 +9872,7 @@ ObRpcRemoteWriteDDLRedoLogArg::ObRpcRemoteWriteDDLRedoLogArg()
int ObRpcRemoteWriteDDLRedoLogArg::init(const uint64_t tenant_id,
const share::ObLSID &ls_id,
const blocksstable::ObDDLMacroBlockRedoInfo &redo_info,
const storage::ObDDLMacroBlockRedoInfo &redo_info,
const int64_t task_id)
{
int ret = OB_SUCCESS;
@ -9916,6 +9916,106 @@ int ObRpcRemoteWriteDDLCommitLogArg::init(const uint64_t tenant_id,
OB_SERIALIZE_MEMBER(ObRpcRemoteWriteDDLCommitLogArg, tenant_id_, ls_id_, table_key_, start_scn_,
table_id_, execution_id_, ddl_task_id_);
ObRpcRemoteWriteDDLIncCommitLogArg::ObRpcRemoteWriteDDLIncCommitLogArg()
: tenant_id_(OB_INVALID_ID), ls_id_(), tablet_id_(), lob_meta_tablet_id_(), tx_desc_(nullptr), need_release_(false)
{}
ObRpcRemoteWriteDDLIncCommitLogArg::~ObRpcRemoteWriteDDLIncCommitLogArg()
{
int ret = OB_SUCCESS;
if (OB_FAIL(release())) {
LOG_WARN("fail to release tx_desc", K(ret));
}
}
int ObRpcRemoteWriteDDLIncCommitLogArg::init(const uint64_t tenant_id,
const share::ObLSID &ls_id,
const common::ObTabletID tablet_id,
const common::ObTabletID lob_meta_tablet_id,
transaction::ObTxDesc *tx_desc)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(tenant_id_ = OB_INVALID_ID && !ls_id.is_valid() || !tablet_id.is_valid() ||
OB_ISNULL(tx_desc) || !tx_desc->is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("tablet id is not valid", K(ret), K(tenant_id), K(ls_id), K(tablet_id), K(lob_meta_tablet_id), KPC(tx_desc));
} else if (OB_FAIL(release())) {
LOG_WARN("fail to release tx_desc", K(ret));
} else {
tenant_id_ = tenant_id;
ls_id_ = ls_id;
tablet_id_ = tablet_id;
lob_meta_tablet_id_ = lob_meta_tablet_id;
tx_desc_ = tx_desc;
}
return ret;
}
int ObRpcRemoteWriteDDLIncCommitLogArg::release()
{
int ret = OB_SUCCESS;
if (tx_desc_ != nullptr && need_release_) {
ObTransService *tx_svc = MTL_WITH_CHECK_TENANT(ObTransService *, tenant_id_);
if (OB_ISNULL(tx_svc)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("null tx service ptr", K(ret));
} else if (OB_FAIL(tx_svc->release_tx(*tx_desc_))) {
LOG_WARN("release tx fail", K(ret));
} else {
need_release_ = false;
tx_desc_ = nullptr;
}
}
return ret;
}
OB_DEF_SERIALIZE(ObRpcRemoteWriteDDLIncCommitLogArg)
{
int ret = OB_SUCCESS;
LST_DO_CODE(OB_UNIS_ENCODE, tenant_id_, ls_id_, tablet_id_, lob_meta_tablet_id_);
if (OB_SUCC(ret)) {
if (OB_ISNULL(tx_desc_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tx_desc_ is nullptr", K(ret));
} else {
LST_DO_CODE(OB_UNIS_ENCODE, *tx_desc_);
}
}
return ret;
}
OB_DEF_DESERIALIZE(ObRpcRemoteWriteDDLIncCommitLogArg)
{
int ret = OB_SUCCESS;
LST_DO_CODE(OB_UNIS_DECODE, tenant_id_, ls_id_, tablet_id_, lob_meta_tablet_id_);
if (OB_SUCC(ret)) {
ObTransService *tx_svc = nullptr;
if (OB_FAIL(release())) {
LOG_WARN("fail to release tx_desc", K(ret));
} else if (OB_ISNULL(tx_svc = MTL_WITH_CHECK_TENANT(ObTransService *, tenant_id_))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("null tx service ptr", KR(ret), K(tenant_id_));
} else if (OB_FAIL(tx_svc->acquire_tx(buf, data_len, pos, tx_desc_))) {
LOG_WARN("acquire tx by deserialize fail", K(data_len), K(pos), KR(ret));
} else {
need_release_ = true;
}
}
return ret;
}
OB_DEF_SERIALIZE_SIZE(ObRpcRemoteWriteDDLIncCommitLogArg)
{
int64_t len = 0;
LST_DO_CODE(OB_UNIS_ADD_LEN, tenant_id_, ls_id_, tablet_id_, lob_meta_tablet_id_);
if (tx_desc_ != nullptr) {
LST_DO_CODE(OB_UNIS_ADD_LEN, *tx_desc_);
}
return len;
}
OB_SERIALIZE_MEMBER(ObRpcRemoteWriteDDLIncCommitLogRes, tx_result_);
bool ObCheckLSCanOfflineArg::is_valid() const
{

47
src/share/ob_rpc_struct.h
View File

@ -56,6 +56,7 @@
#include "share/ob_tenant_info_proxy.h"//ObAllTenantInfo
#include "share/ob_alive_server_tracer.h"//ServerAddr
#include "storage/blocksstable/ob_block_sstable_struct.h"
#include "storage/ddl/ob_ddl_struct.h"
#include "storage/tx/ob_trans_define.h"
#include "share/unit/ob_unit_info.h" //ObUnit*
#include "share/backup/ob_backup_clean_struct.h"
@ -10042,16 +10043,15 @@ public:
~ObRpcRemoteWriteDDLRedoLogArg() = default;
int init(const uint64_t tenant_id,
const share::ObLSID &ls_id,
const blocksstable::ObDDLMacroBlockRedoInfo &redo_info,
const storage::ObDDLMacroBlockRedoInfo &redo_info,
const int64_t task_id);
bool is_valid() const { return tenant_id_ != OB_INVALID_ID && ls_id_.is_valid() && redo_info_.is_valid() && task_id_ != 0; }
TO_STRING_KV(K_(tenant_id), K(ls_id_), K_(redo_info), K(task_id_));
public:
uint64_t tenant_id_;
share::ObLSID ls_id_;
blocksstable::ObDDLMacroBlockRedoInfo redo_info_;
storage::ObDDLMacroBlockRedoInfo redo_info_;
int64_t task_id_;
private:
DISALLOW_COPY_AND_ASSIGN(ObRpcRemoteWriteDDLRedoLogArg);
};
@ -10084,6 +10084,47 @@ private:
DISALLOW_COPY_AND_ASSIGN(ObRpcRemoteWriteDDLCommitLogArg);
};
struct ObRpcRemoteWriteDDLIncCommitLogArg final
{
OB_UNIS_VERSION(1);
public:
ObRpcRemoteWriteDDLIncCommitLogArg();
~ObRpcRemoteWriteDDLIncCommitLogArg();
int init(const uint64_t tenant_id,
const share::ObLSID &ls_id,
const common::ObTabletID tablet_id,
const common::ObTabletID lob_meta_tablet_id,
transaction::ObTxDesc *tx_desc);
int release();
bool is_valid() const
{
return tenant_id_ != OB_INVALID_ID && ls_id_.is_valid() && tablet_id_.is_valid() &&
OB_NOT_NULL(tx_desc_) && tx_desc_->is_valid();
}
TO_STRING_KV(K_(tenant_id), K_(ls_id), K_(tablet_id), K_(lob_meta_tablet_id), KP_(tx_desc));
public:
uint64_t tenant_id_;
share::ObLSID ls_id_;
common::ObTabletID tablet_id_;
common::ObTabletID lob_meta_tablet_id_;
transaction::ObTxDesc *tx_desc_;
bool need_release_;
private:
DISALLOW_COPY_AND_ASSIGN(ObRpcRemoteWriteDDLIncCommitLogArg);
};
struct ObRpcRemoteWriteDDLIncCommitLogRes final
{
OB_UNIS_VERSION(1);
public:
ObRpcRemoteWriteDDLIncCommitLogRes() : tx_result_() {}
~ObRpcRemoteWriteDDLIncCommitLogRes() {}
TO_STRING_KV(K_(tx_result));
public:
transaction::ObTxExecResult tx_result_;
private:
DISALLOW_COPY_AND_ASSIGN(ObRpcRemoteWriteDDLIncCommitLogRes);
};
struct ObCheckLSCanOfflineArg
{

1
src/share/ob_srv_rpc_proxy.h
View File

@ -211,6 +211,7 @@ public:
RPC_S(PR4 admin_update_lock_op, OB_UPDATE_OBJ_LOCK, (transaction::tablelock::ObAdminUpdateLockOpArg));
RPC_S(PR5 remote_write_ddl_redo_log, OB_REMOTE_WRITE_DDL_REDO_LOG, (obrpc::ObRpcRemoteWriteDDLRedoLogArg));
RPC_S(PR5 remote_write_ddl_commit_log, OB_REMOTE_WRITE_DDL_COMMIT_LOG, (obrpc::ObRpcRemoteWriteDDLCommitLogArg), obrpc::Int64);
RPC_S(PR5 remote_write_ddl_inc_commit_log, OB_REMOTE_WRITE_DDL_INC_COMMIT_LOG, (obrpc::ObRpcRemoteWriteDDLIncCommitLogArg), ObRpcRemoteWriteDDLIncCommitLogRes);
RPC_S(PR5 check_ls_can_offline, OB_CHECK_LS_CAN_OFFLINE, (obrpc::ObCheckLSCanOfflineArg));
RPC_S(PR5 clean_sequence_cache, obrpc::OB_CLEAN_SEQUENCE_CACHE, (obrpc::UInt64));
RPC_S(PR5 register_tx_data, OB_REGISTER_TX_DATA, (ObRegisterTxDataArg), ObRegisterTxDataResult);

2
src/share/table/ob_table_load_define.cpp
View File

@ -48,7 +48,7 @@ OB_SERIALIZE_MEMBER(ObTableLoadResultInfo,
skipped_,
warnings_);
OB_SERIALIZE_MEMBER(ObTableLoadSequenceNo, sequence_no_);
OB_SERIALIZE_MEMBER_SIMPLE(ObTableLoadSequenceNo, sequence_no_);
} // namespace table
} // namespace oceanbase

235
src/share/table/ob_table_load_sql_statistics.cpp
View File

@ -13,6 +13,7 @@
#define USING_LOG_PREFIX CLIENT
#include "ob_table_load_sql_statistics.h"
#include "lib/oblog/ob_log_module.h"
namespace oceanbase
{
@ -38,6 +39,31 @@ void ObTableLoadSqlStatistics::reset()
allocator_.reset();
}
int ObTableLoadSqlStatistics::create(int64_t column_count)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(column_count <= 0)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invald args", KR(ret), K(column_count));
} else if (OB_UNLIKELY(!is_empty())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected not empty", KR(ret));
} else {
ObOptTableStat *table_stat = nullptr;
ObOptOSGColumnStat *osg_col_stat = nullptr;
if (OB_FAIL(allocate_table_stat(table_stat))) {
LOG_WARN("fail to allocate table stat", KR(ret));
}
for (int64_t i = 0; OB_SUCC(ret) && i < column_count; ++i) {
ObOptOSGColumnStat *osg_col_stat = nullptr;
if (OB_FAIL(allocate_col_stat(osg_col_stat))) {
LOG_WARN("fail to allocate col stat", KR(ret));
}
}
}
return ret;
}
int ObTableLoadSqlStatistics::allocate_table_stat(ObOptTableStat *&table_stat)
{
int ret = OB_SUCCESS;
@ -82,98 +108,189 @@ int ObTableLoadSqlStatistics::allocate_col_stat(ObOptOSGColumnStat *&col_stat)
return ret;
}
int ObTableLoadSqlStatistics::add(const ObTableLoadSqlStatistics& other)
int ObTableLoadSqlStatistics::merge(const ObTableLoadSqlStatistics &other)
{
int ret = OB_SUCCESS;
for (int64_t i = 0; OB_SUCC(ret)&& i < other.table_stat_array_.count(); ++i) {
ObOptTableStat *table_stat = other.table_stat_array_.at(i);
if (table_stat != nullptr) {
if (is_empty()) {
for (int64_t i = 0; OB_SUCC(ret) && i < other.table_stat_array_.count(); ++i) {
ObOptTableStat *table_stat = other.table_stat_array_.at(i);
ObOptTableStat *copied_table_stat = nullptr;
int64_t size = table_stat->size();
char *new_buf = nullptr;
if (OB_ISNULL(new_buf = static_cast<char *>(allocator_.alloc(size)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
OB_LOG(WARN, "fail to allocate buffer", KR(ret), K(size));
} else if (OB_FAIL(table_stat->deep_copy(new_buf, size, copied_table_stat))) {
OB_LOG(WARN, "fail to copy table stat", KR(ret));
} else if (OB_FAIL(table_stat_array_.push_back(copied_table_stat))) {
OB_LOG(WARN, "fail to add table stat", KR(ret));
}
if (OB_FAIL(ret)) {
if (copied_table_stat != nullptr) {
copied_table_stat->~ObOptTableStat();
copied_table_stat = nullptr;
if (OB_ISNULL(table_stat)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected table stat is null", KR(ret), K(i), K(other.table_stat_array_));
} else {
int64_t size = table_stat->size();
char *buf = nullptr;
if (OB_ISNULL(buf = static_cast<char *>(allocator_.alloc(size)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to allocate buffer", KR(ret), K(size));
} else if (OB_FAIL(table_stat->deep_copy(buf, size, copied_table_stat))) {
LOG_WARN("fail to copy table stat", KR(ret));
} else if (OB_FAIL(table_stat_array_.push_back(copied_table_stat))) {
LOG_WARN("fail to add table stat", KR(ret));
}
if(new_buf != nullptr) {
allocator_.free(new_buf);
new_buf = nullptr;
if (OB_FAIL(ret)) {
if (copied_table_stat != nullptr) {
copied_table_stat->~ObOptTableStat();
copied_table_stat = nullptr;
}
if (buf != nullptr) {
allocator_.free(buf);
buf = nullptr;
}
}
}
}
}
for (int64_t i = 0; OB_SUCC(ret)&& i < other.col_stat_array_.count(); ++i) {
ObOptOSGColumnStat *col_stat = other.col_stat_array_.at(i);
ObOptOSGColumnStat *copied_col_stat = nullptr;
if (OB_ISNULL(col_stat)) {
ret = OB_ERR_UNEXPECTED;
OB_LOG(WARN, "get unexpected null");
} else if (OB_ISNULL(copied_col_stat = ObOptOSGColumnStat::create_new_osg_col_stat(allocator_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
OB_LOG(WARN, "failed to create new col stat");
} else if (OB_FAIL(copied_col_stat->deep_copy(*col_stat))) {
OB_LOG(WARN, "fail to copy col stat", KR(ret));
} else if (OB_FAIL(col_stat_array_.push_back(copied_col_stat))) {
OB_LOG(WARN, "fail to add col stat", KR(ret));
for (int64_t i = 0; OB_SUCC(ret) && i < other.col_stat_array_.count(); ++i) {
ObOptOSGColumnStat *col_stat = other.col_stat_array_.at(i);
ObOptOSGColumnStat *copied_col_stat = nullptr;
if (OB_ISNULL(col_stat)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected col stat is null", KR(ret), K(i), K(other.col_stat_array_));
} else if (OB_FAIL(allocate_col_stat(copied_col_stat))) {
LOG_WARN("fail to allocate col stat", KR(ret));
} else if (OB_FAIL(copied_col_stat->deep_copy(*col_stat))) {
LOG_WARN("fail to copy col stat", KR(ret));
}
}
if (OB_FAIL(ret)) {
if (copied_col_stat != nullptr) {
copied_col_stat->~ObOptOSGColumnStat();
copied_col_stat = nullptr;
} else {
if (OB_UNLIKELY(other.table_stat_array_.count() != table_stat_array_.count() ||
other.col_stat_array_.count() != col_stat_array_.count())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), KPC(this), K(other));
}
for (int64_t i = 0; OB_SUCC(ret) && i < other.table_stat_array_.count(); ++i) {
ObOptTableStat *table_stat = other.table_stat_array_.at(i);
if (OB_ISNULL(table_stat)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected table stat is null", KR(ret), K(i), K(other.table_stat_array_));
} else if (OB_FAIL(table_stat_array_.at(i)->merge_table_stat(*table_stat))) {
LOG_WARN("fail to merge table stat", KR(ret));
}
}
for (int64_t i = 0; OB_SUCC(ret) && i < other.col_stat_array_.count(); ++i) {
ObOptOSGColumnStat *osg_col_stat = other.col_stat_array_.at(i);
if (OB_ISNULL(osg_col_stat)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected col stat is null", KR(ret), K(i), K(other.col_stat_array_));
}
// ObOptOSGColumnStat的序列化结果只序列化里面的ObOptColumnStat,
// 需要调用ObOptColumnStat的merge函数
else if (OB_FAIL(
col_stat_array_.at(i)->col_stat_->merge_column_stat(*osg_col_stat->col_stat_))) {
LOG_WARN("fail to merge col stat", KR(ret));
}
}
}
return ret;
}
int ObTableLoadSqlStatistics::get_table_stat_array(ObIArray<ObOptTableStat*> &table_stat_array) const
int ObTableLoadSqlStatistics::get_table_stat(int64_t idx, ObOptTableStat *&table_stat)
{
int ret = OB_SUCCESS;
table_stat = nullptr;
if (OB_UNLIKELY(idx >= table_stat_array_.count())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(idx), K(table_stat_array_.count()));
} else {
table_stat = table_stat_array_.at(idx);
}
return ret;
}
int ObTableLoadSqlStatistics::get_col_stat(int64_t idx, ObOptOSGColumnStat *&osg_col_stat)
{
int ret = OB_SUCCESS;
osg_col_stat = nullptr;
if (OB_UNLIKELY(idx >= col_stat_array_.count())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(idx), K(col_stat_array_.count()));
} else {
osg_col_stat = col_stat_array_.at(idx);
}
return ret;
}
int ObTableLoadSqlStatistics::get_table_stat_array(ObIArray<ObOptTableStat *> &table_stat_array) const
{
int ret = OB_SUCCESS;
for (int64_t i = 0; OB_SUCC(ret) && i < table_stat_array_.count(); ++i) {
if (OB_ISNULL(table_stat_array_.at(i))) {
ObOptTableStat *table_stat = nullptr;
if (OB_ISNULL(table_stat = table_stat_array_.at(i))) {
ret = OB_ERR_UNEXPECTED;
OB_LOG(WARN, "get unexpected null");
} else if (OB_FAIL(table_stat_array.push_back(table_stat_array_.at(i)))) {
OB_LOG(WARN, "failed to push back col stat");
LOG_WARN("unexpected table stat is null", KR(ret), K(i), K(table_stat_array_));
} else if (OB_FAIL(table_stat_array.push_back(table_stat))) {
LOG_WARN("fail to push back table stat", KR(ret));
}
}
return ret;
}
int ObTableLoadSqlStatistics::get_col_stat_array(ObIArray<ObOptColumnStat*> &col_stat_array) const
int ObTableLoadSqlStatistics::get_col_stat_array(ObIArray<ObOptColumnStat *> &col_stat_array) const
{
int ret = OB_SUCCESS;
for (int64_t i = 0; OB_SUCC(ret) && i < col_stat_array_.count(); ++i) {
if (OB_ISNULL(col_stat_array_.at(i))) {
ObOptOSGColumnStat *osg_col_stat = nullptr;
if (OB_ISNULL(osg_col_stat = col_stat_array_.at(i))) {
ret = OB_ERR_UNEXPECTED;
OB_LOG(WARN, "get unexpected null");
} else if (OB_FAIL(col_stat_array_.at(i)->set_min_max_datum_to_obj())) {
OB_LOG(WARN, "failed to persistence min max");
} else if (OB_FAIL(col_stat_array.push_back(col_stat_array_.at(i)->col_stat_))) {
OB_LOG(WARN, "failed to push back col stat");
LOG_WARN("unexpected col stat is null", KR(ret), K(i), K(col_stat_array_));
} else if (OB_FAIL(osg_col_stat->set_min_max_datum_to_obj())) {
LOG_WARN("fail to persistence min max", KR(ret));
} else if (OB_FAIL(col_stat_array.push_back(osg_col_stat->col_stat_))) {
LOG_WARN("fail to push back col stat", KR(ret));
}
}
return ret;
}
int ObTableLoadSqlStatistics::persistence_col_stats()
int ObTableLoadSqlStatistics::get_table_stats(TabStatIndMap &table_stats) const
{
int ret = OB_SUCCESS;
for (int64_t i = 0; OB_SUCC(ret) && i < col_stat_array_.count(); ++i) {
if (OB_ISNULL(col_stat_array_.at(i))) {
const uint64_t tenant_id = MTL_ID();
ObOptTableStat *table_stat = nullptr;
for (int64_t i = 0; OB_SUCC(ret) && i < table_stat_array_.count(); ++i) {
if (OB_ISNULL(table_stat = table_stat_array_.at(i))) {
ret = OB_ERR_UNEXPECTED;
OB_LOG(WARN, "get unexpected null");
} else if (OB_FAIL(col_stat_array_.at(i)->set_min_max_datum_to_obj())) {
OB_LOG(WARN, "failed to persistence min max");
LOG_WARN("unexpected table stat is null", KR(ret));
} else {
ObOptTableStat::Key key(tenant_id,
table_stat->get_table_id(),
table_stat->get_partition_id());
if (OB_FAIL(table_stats.set_refactored(key, table_stat))) {
LOG_WARN("fail to set table stat", KR(ret), K(key), KPC(table_stat));
if (OB_HASH_EXIST == ret) {
ret = OB_ENTRY_EXIST;
}
}
}
}
return ret;
}
int ObTableLoadSqlStatistics::get_col_stats(ColStatIndMap &col_stats) const
{
int ret = OB_SUCCESS;
const uint64_t tenant_id = MTL_ID();
ObOptOSGColumnStat *osg_col_stat = nullptr;
ObOptColumnStat *col_stat = nullptr;
for (int64_t i = 0; OB_SUCC(ret) && i < col_stat_array_.count(); ++i) {
if (OB_ISNULL(osg_col_stat = col_stat_array_.at(i))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected osg col stat is null", KR(ret));
} else if (OB_ISNULL(col_stat = osg_col_stat->col_stat_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected col stat is null", KR(ret));
} else {
ObOptColumnStat::Key key(tenant_id,
col_stat->get_table_id(),
col_stat->get_partition_id(),
col_stat->get_column_id());
if (OB_FAIL(col_stats.set_refactored(key, col_stat))) {
LOG_WARN("fail to set col stat", KR(ret), K(key), KPC(col_stat));
if (OB_HASH_EXIST == ret) {
ret = OB_ENTRY_EXIST;
}
}
}
}
return ret;
@ -275,5 +392,5 @@ OB_DEF_SERIALIZE_SIZE(ObTableLoadSqlStatistics)
return len;
}
} // namespace table
} // namespace oceanbase
} // namespace table
} // namespace oceanbase

25
src/share/table/ob_table_load_sql_statistics.h
View File

@ -12,9 +12,10 @@
#pragma once
#include "share/stat/ob_opt_table_stat.h"
#include "share/stat/ob_opt_column_stat.h"
#include "share/stat/ob_opt_osg_column_stat.h"
#include "share/stat/ob_opt_table_stat.h"
#include "share/stat/ob_stat_define.h"
namespace oceanbase
{
@ -33,17 +34,21 @@ public:
}
~ObTableLoadSqlStatistics() { reset(); }
void reset();
bool is_empty() const
{
return table_stat_array_.count() == 0 || col_stat_array_.count() == 0;
}
OB_INLINE int64_t get_table_stat_count() const { return table_stat_array_.count(); }
OB_INLINE int64_t get_col_stat_count() const { return col_stat_array_.count(); }
OB_INLINE bool is_empty() const { return table_stat_array_.empty() && col_stat_array_.empty(); }
int create(int64_t column_count);
int merge(const ObTableLoadSqlStatistics &other);
int get_table_stat(int64_t idx, ObOptTableStat *&table_stat);
int get_col_stat(int64_t idx, ObOptOSGColumnStat *&osg_col_stat);
int get_table_stat_array(ObIArray<ObOptTableStat *> &table_stat_array) const;
int get_col_stat_array(ObIArray<ObOptColumnStat *> &col_stat_array) const;
int get_table_stats(TabStatIndMap &table_stats) const;
int get_col_stats(ColStatIndMap &col_stats) const;
TO_STRING_KV(K_(col_stat_array), K_(table_stat_array));
private:
int allocate_table_stat(ObOptTableStat *&table_stat);
int allocate_col_stat(ObOptOSGColumnStat *&col_stat);
int add(const ObTableLoadSqlStatistics& other);
int get_table_stat_array(ObIArray<ObOptTableStat*> &table_stat_array) const;
int get_col_stat_array(ObIArray<ObOptColumnStat*> &col_stat_array) const;
int persistence_col_stats();
TO_STRING_KV(K_(col_stat_array), K_(table_stat_array));
public:
common::ObArray<ObOptTableStat *> table_stat_array_;
common::ObArray<ObOptOSGColumnStat *> col_stat_array_;

1
src/share/throttle/ob_share_throttle_define.cpp
View File

@ -88,7 +88,6 @@ void FakeAllocatorForTxShare::adaptive_update_limit(const int64_t tenant_id,
usable_remain_memory = std::max(usable_remain_memory, remain_memory - MAX_UNUSABLE_MEMORY);
}
is_updated = false;
if (holding_size + usable_remain_memory < config_specify_resource_limit) {
resource_limit = holding_size + usable_remain_memory;

5
src/sql/code_generator/ob_static_engine_cg.cpp
View File

@ -6378,8 +6378,11 @@ int ObStaticEngineCG::generate_spec(ObLogInsert &op,
spec.table_location_uncertain_ = op.is_table_location_uncertain(); // row-movement target table
spec.is_pdml_update_split_ = op.is_pdml_update_split();
if (GCONF._ob_enable_direct_load) {
const ObGlobalHint &global_hint = op.get_plan()->get_optimizer_context().get_global_hint();
spec.plan_->set_append_table_id(op.get_append_table_id());
spec.plan_->set_enable_append(op.get_plan()->get_optimizer_context().get_global_hint().has_append());
spec.plan_->set_enable_append(global_hint.has_direct_load());
spec.plan_->set_enable_inc_direct_load(global_hint.has_inc_direct_load());
spec.plan_->set_enable_replace(global_hint.has_replace());
}
int64_t partition_expr_idx = OB_INVALID_INDEX;
if (OB_FAIL(get_pdml_partition_id_column_idx(spec.get_child(0)->output_, partition_expr_idx))) {

97
src/sql/engine/cmd/ob_load_data_direct_impl.cpp
View File

@ -57,7 +57,6 @@ ObLoadDataDirectImpl::LoadExecuteParam::LoadExecuteParam()
: tenant_id_(OB_INVALID_ID),
database_id_(OB_INVALID_ID),
table_id_(OB_INVALID_ID),
sql_mode_(0),
parallel_(0),
thread_count_(0),
batch_row_count_(0),
@ -66,7 +65,9 @@ ObLoadDataDirectImpl::LoadExecuteParam::LoadExecuteParam()
online_opt_stat_gather_(false),
max_error_rows_(-1),
ignore_row_num_(-1),
dup_action_(ObLoadDupActionType::LOAD_INVALID_MODE)
dup_action_(ObLoadDupActionType::LOAD_INVALID_MODE),
method_(ObDirectLoadMethod::INVALID_METHOD),
insert_mode_(ObDirectLoadInsertMode::INVALID_INSERT_MODE)
{
store_column_idxs_.set_tenant_id(MTL_ID());
}
@ -77,8 +78,19 @@ bool ObLoadDataDirectImpl::LoadExecuteParam::is_valid() const
OB_INVALID_ID != table_id_ && !database_name_.empty() && !table_name_.empty() &&
!combined_name_.empty() && parallel_ > 0 && thread_count_ > 0 && batch_row_count_ > 0 &&
data_mem_usage_limit_ > 0 && max_error_rows_ >= 0 && ignore_row_num_ >= 0 &&
ObLoadDupActionType::LOAD_INVALID_MODE != dup_action_ && data_access_param_.is_valid() &&
!store_column_idxs_.empty();
ObLoadDupActionType::LOAD_INVALID_MODE != dup_action_ &&
ObDirectLoadMethod::is_type_valid(method_) &&
ObDirectLoadInsertMode::is_type_valid(insert_mode_) &&
(storage::ObDirectLoadMethod::is_full(method_)
? storage::ObDirectLoadInsertMode::is_valid_for_full_method(insert_mode_)
: true) &&
(storage::ObDirectLoadMethod::is_incremental(method_)
? storage::ObDirectLoadInsertMode::is_valid_for_incremental_method(insert_mode_)
: true) &&
(storage::ObDirectLoadInsertMode::INC_REPLACE == insert_mode_
? sql::ObLoadDupActionType::LOAD_REPLACE == dup_action_
: true) &&
data_access_param_.is_valid() && !store_column_idxs_.empty();
}
/**
@ -1831,10 +1843,10 @@ int ObLoadDataDirectImpl::execute(ObExecContext &ctx, ObLoadDataStmt &load_stmt)
}
if (OB_SUCC(ret)) {
ObTableLoadResultInfo result_info;
if (OB_FAIL(direct_loader_.commit(result_info))) {
if (OB_FAIL(direct_loader_.commit())) {
LOG_WARN("fail to commit direct loader", KR(ret));
} else {
const ObTableLoadResultInfo &result_info = direct_loader_.get_result_info();
ObPhysicalPlanCtx *phy_plan_ctx = ctx.get_physical_plan_ctx();
phy_plan_ctx->set_affected_rows(result_info.rows_affected_);
phy_plan_ctx->set_row_matched_count(total_line_count);
@ -1889,31 +1901,36 @@ int ObLoadDataDirectImpl::init_execute_param()
hint_batch_size > 0 ? hint_batch_size : DEFAULT_BUFFERRED_ROW_COUNT;
}
}
// need_sort_
// direct load hint
if (OB_SUCC(ret)) {
int64_t enable_direct = 0;
int64_t hint_need_sort = 0;
if (OB_FAIL(hint.get_value(ObLoadDataHint::ENABLE_DIRECT, enable_direct))) {
LOG_WARN("fail to get value of ENABLE_DIRECT", K(ret));
} else if (OB_FAIL(hint.get_value(ObLoadDataHint::NEED_SORT, hint_need_sort))) {
LOG_WARN("fail to get value of NEED_SORT", KR(ret), K(hint));
} else if (enable_direct != 0) {
execute_param_.need_sort_ = hint_need_sort > 0 ? true : false;
} else {
const ObDirectLoadHint &direct_load_hint = hint.get_direct_load_hint();
if (direct_load_hint.is_enable()) {
execute_param_.need_sort_ = direct_load_hint.need_sort();
execute_param_.max_error_rows_ = direct_load_hint.get_max_error_row_count();
execute_param_.method_ =
(direct_load_hint.is_inc_direct_load() ? ObDirectLoadMethod::INCREMENTAL
: ObDirectLoadMethod::FULL);
execute_param_.insert_mode_ = ObDirectLoadInsertMode::NORMAL;
if (OB_UNLIKELY(direct_load_hint.is_inc_load_method())) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("inc load method not supported", KR(ret), K(direct_load_hint));
LOG_USER_ERROR(OB_NOT_SUPPORTED, "inc load method in direct load is");
} else if (direct_load_hint.is_inc_replace_load_method()) {
if (OB_UNLIKELY(ObLoadDupActionType::LOAD_STOP_ON_DUP != load_args.dupl_action_)) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("replace or ignore for inc_replace load method not supported", KR(ret),
K(direct_load_hint), K(load_args.dupl_action_));
LOG_USER_ERROR(OB_NOT_SUPPORTED, "replace or ignore for inc_replace load method in direct load is");
} else {
execute_param_.dup_action_ = ObLoadDupActionType::LOAD_REPLACE; // rewrite dup action
execute_param_.insert_mode_ = ObDirectLoadInsertMode::INC_REPLACE;
}
}
} else { // append
execute_param_.need_sort_ = true;
}
}
// sql_mode_
if (OB_SUCC(ret)) {
ObSQLSessionInfo *session = nullptr;
uint64_t sql_mode;
if (OB_ISNULL(session = ctx_->get_my_session())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("session is null", KR(ret));
} else if (OB_FAIL(session->get_sys_variable(SYS_VAR_SQL_MODE, sql_mode))) {
LOG_WARN("fail to get sys variable", K(ret));
} else {
execute_param_.sql_mode_ = sql_mode;
execute_param_.max_error_rows_ = 0;
execute_param_.method_ = ObDirectLoadMethod::FULL;
execute_param_.insert_mode_ = ObDirectLoadInsertMode::NORMAL;
}
}
// online_opt_stat_gather_
@ -1934,23 +1951,6 @@ int ObLoadDataDirectImpl::init_execute_param()
execute_param_.online_opt_stat_gather_ = false;
}
}
// max_error_rows_
if (OB_SUCC(ret)) {
int64_t append = 0;
int64_t enable_direct = 0;
int64_t hint_error_rows = 0;
if (OB_FAIL(hint.get_value(ObLoadDataHint::APPEND, append))) {
LOG_WARN("fail to get value of APPEND", K(ret));
} else if (OB_FAIL(hint.get_value(ObLoadDataHint::ENABLE_DIRECT, enable_direct))) {
LOG_WARN("fail to get value of ENABLE_DIRECT", K(ret));
} else if (OB_FAIL(hint.get_value(ObLoadDataHint::ERROR_ROWS, hint_error_rows))) {
LOG_WARN("fail to get value of ERROR_ROWS", KR(ret), K(hint));
} else if (enable_direct != 0) {
execute_param_.max_error_rows_ = hint_error_rows;
} else {
execute_param_.max_error_rows_ = 0;
}
}
// data_access_param_
if (OB_SUCC(ret)) {
DataAccessParam &data_access_param = execute_param_.data_access_param_;
@ -2040,11 +2040,10 @@ int ObLoadDataDirectImpl::init_execute_context()
load_param.column_count_ = execute_param_.store_column_idxs_.count();
load_param.need_sort_ = execute_param_.need_sort_;
load_param.dup_action_ = execute_param_.dup_action_;
load_param.sql_mode_ = execute_param_.sql_mode_;
load_param.px_mode_ = false;
load_param.online_opt_stat_gather_ = execute_param_.online_opt_stat_gather_;
load_param.method_ = ObDirectLoadMethod::FULL;
load_param.insert_mode_ = ObDirectLoadInsertMode::NORMAL;
load_param.method_ = execute_param_.method_;
load_param.insert_mode_ = execute_param_.insert_mode_;
if (OB_FAIL(direct_loader_.init(load_param, execute_param_.store_column_idxs_,
&execute_ctx_.exec_ctx_))) {
LOG_WARN("fail to init direct loader", KR(ret));

Some files were not shown because too many files have changed in this diff Show More