hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Fix issue that px hang when init sqc failed

Browse Source
This commit is contained in:
obdev 2024-09-25 17:46:58 +00:00 committed by ob-robot
parent e69d21839f
commit eee3b2ce8e
430 changed files with 45561 additions and 5853 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
deps/oblib/src/common/ob_common_types.h vendored
View File

@ -87,6 +87,7 @@ struct ObQueryFlag
static const uint64_t OBSF_MASK_IS_MDS_QUERY = (0x1UL << OBSF_BIT_IS_MDS_QUERY) - 1;
static const uint64_t OBSF_MASK_IS_SELECT_FOLLOWER = (0x1UL << OBSF_BIT_IS_SELECT_FOLLOWER) - 1;
static const uint64_t OBSF_MASK_ENABLE_LOB_PREFETCH = (0x1UL << OBSF_BIT_ENABLE_LOB_PREFETCH) - 1;
static const uint64_t OBSF_MASK_IS_DIRECT_SCAN = (0x1UL << OBSF_BIT_IS_BARE_ROW_SCAN) - 1;
enum ScanOrder
@ -242,6 +243,7 @@ struct ObQueryFlag
inline bool is_for_foreign_key_check() const { return for_foreign_key_check_; }
inline bool is_sstable_cut() const { return is_sstable_cut_; }
inline bool is_skip_read_lob() const { return skip_read_lob_; }
inline bool is_bare_row_scan() const {return is_bare_row_scan_; }
inline bool is_mds_query() const { return is_mds_query_; }
inline void set_is_select_follower() { is_select_follower_ = true; }
inline bool is_select_follower() const { return is_select_follower_; }
@ -279,6 +281,7 @@ struct ObQueryFlag
"is_sstable_cut", is_sstable_cut_,
"skip_read_lob", skip_read_lob_,
"is_lookup_for_4377", is_lookup_for_4377_,
"is_bare_row_scan", is_bare_row_scan_,
"enable_rich_format", enable_rich_format_,
"is_for_foreign_key_check", for_foreign_key_check_,
"is_mds_query", is_mds_query_,

2
deps/oblib/src/common/object/ob_object.h vendored
View File

@ -554,7 +554,7 @@ struct ObLobDataOutRowCtx
ERASE,
EMPTY_SQL, // lob col not change in full sql update, out row ctx is empty
DIFF,
EXT_INFO_LOG,
EXT_INFO_LOG, // only used in new row value
VALID_OLD_VALUE_EXT_INFO_LOG,
VALID_OLD_VALUE,
DIFF_V2, // add lob id in ext info log, so deprecated old, use new diff type

2
deps/oblib/src/common/storage/ob_device_common.h vendored
View File

@ -180,7 +180,7 @@ struct ObStorageIdMod
static const ObStorageIdMod get_default_id_mod()
{
static const ObStorageIdMod storage_id_mod(0, ObStorageUsedMod::STORAGE_USED_OTHER);;
static const ObStorageIdMod storage_id_mod(0, ObStorageUsedMod::STORAGE_USED_OTHER);
return storage_id_mod;
}

4
deps/oblib/src/lib/geo/ob_geo_mvt.cpp vendored
View File

@ -267,7 +267,7 @@ int mvt_agg_result::transform_json_column(ObObj &json)
default:
// ignore other type, do nothing
ignore_type = true;
break;;
break;
}
if (OB_SUCC(ret) && !ignore_type) {
tile_value.value_ = value;
@ -507,4 +507,4 @@ bool mvt_agg_result::is_upper_char_exist(const ObString &str)
}
} // namespace common
} // namespace oceanbase
} // namespace oceanbase

2
deps/oblib/src/lib/json_type/ob_json_bin.cpp vendored
View File

@ -4729,7 +4729,7 @@ int ObJsonBin::get_value(int index, ObJsonBin &value) const
uint64_t value_offset = 0;
uint8_t value_type = 0;
if (OB_FAIL(get_value_entry(index, value_offset, value_type))) {
LOG_WARN("get_value_entry fail", K(ret), K(index));;
LOG_WARN("get_value_entry fail", K(ret), K(index));
} else if (OB_JSON_TYPE_IS_INLINE(value_type)) {
offset += pos_;
} else if (is_forward_v0(value_type)) {

5
deps/oblib/src/lib/lock/ob_tc_rwlock.h vendored
View File

@ -60,7 +60,7 @@ public:
};
struct RLockGuardWithTimeout
{
[[nodiscard]] explicit RLockGuardWithTimeout(TCRWLock &lock, const int64_t abs_timeout_us, int &ret): lock_(lock), need_unlock_(true)
[[nodiscard]] explicit RLockGuardWithTimeout(const TCRWLock &lock, const int64_t abs_timeout_us, int &ret): lock_(const_cast<TCRWLock&>(lock)), need_unlock_(true)
{
if (OB_FAIL(lock_.rdlock(abs_timeout_us))) {
need_unlock_ = false;
@ -78,6 +78,7 @@ public:
TCRWLock &lock_;
bool need_unlock_;
};
struct WLockGuard
{
[[nodiscard]] explicit WLockGuard(TCRWLock& lock): lock_(lock) { lock_.wrlock(); }
@ -86,7 +87,7 @@ public:
};
struct WLockGuardWithTimeout
{
[[nodiscard]] explicit WLockGuardWithTimeout(TCRWLock &lock, const int64_t abs_timeout_us, int &ret): lock_(lock), need_unlock_(true)
[[nodiscard]] explicit WLockGuardWithTimeout(const TCRWLock &lock, const int64_t abs_timeout_us, int &ret): lock_(const_cast<TCRWLock&>(lock)), need_unlock_(true)
{
if (OB_FAIL(lock_.wrlock(abs_timeout_us))) {
need_unlock_ = false;

2
deps/oblib/src/lib/mysqlclient/ob_dblink_error_trans.cpp vendored
View File

@ -142,7 +142,7 @@ int ObTenantDblinkKeeper::clean_dblink_conn(uint32_t sessid, bool force_disconne
connection = next;
}
if (OB_SUCC(ret) && OB_FAIL(dblink_conn_map_.erase_refactored(sessid))) {
LOG_WARN("failed to erase_refactored", K(tenant_id_), K(sessid), K(ret));;
LOG_WARN("failed to erase_refactored", K(tenant_id_), K(sessid), K(ret));
}
}
return ret;

2
deps/oblib/src/lib/net/ob_addr.h vendored
View File

@ -243,7 +243,7 @@ inline int ObAddr::compare_refactored(const ObAddr &rv) const
if (0 == ipcmp) {
ipcmp = port_ - rv.port_;
}
return 0 == ipcmp ? 0 : (ipcmp > 0 ? 1 : -1);;
return 0 == ipcmp ? 0 : (ipcmp > 0 ? 1 : -1);
}
// forward compatible

4
deps/oblib/src/lib/xml/ob_multi_mode_interface.h vendored
View File

@ -211,7 +211,7 @@ public:
new (dst_buf) T(*reinterpret_cast<T*>(src_buf));
}
} else {
OB_LOG(WARN, "failed to construct ObStack", K(ret), K(length));;
OB_LOG(WARN, "failed to construct ObStack", K(ret), K(length));
}
}
@ -611,4 +611,4 @@ protected:
} // namespace common
} // namespace oceanbase
#endif // OCEANBASE_SQL_OB_MULTI_MODE_INTERFACE
#endif // OCEANBASE_SQL_OB_MULTI_MODE_INTERFACE

18
deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h vendored
View File

@ -846,7 +846,7 @@ PCODE_DEF(OB_HIGH_PRIORITY_BATCH_TABLE_LOCK_TASK, 0x932)
PCODE_DEF(OB_ARB_CLUSTER_OP, 0x933)
#endif
PCODE_DEF(OB_DETECT_SESSION_ALIVE, 0x934)
// PCODE_DEF(OB_BATCH_REPLACE_TABLE_LOCK_TASK, 0x935)
PCODE_DEF(OB_BATCH_REPLACE_TABLE_LOCK_TASK, 0x935)
#ifdef OB_BUILD_ARBITRATION
// PCODE_DEF(OB_LOG_PROBE_RS, 0x936)
#endif
@ -865,20 +865,20 @@ PCODE_DEF(OB_CHECK_MEDIUM_INFO_LIST_CNT, 0x959)
PCODE_DEF(OB_SPLIT_TABLET_DATA_FINISH_REQUEST, 0x960)
PCODE_DEF(OB_MVIEW_COMPLETE_REFRESH, 0x961)
PCODE_DEF(OB_CLEAR_TABLET_AUTOINC_SEQ_CACHE, 0x962)
//PCODE_DEF(OB_FREEZE_SPLIT_SRC_TABLET, 0x963)
PCODE_DEF(OB_FREEZE_SPLIT_SRC_TABLET, 0x963)
PCODE_DEF(OB_CHECK_AND_CANCEL_DDL_COMPLEMENT_DAG, 0x964)
//PCODE_DEF(OB_CLEAN_SPLITTED_TABLET, 0x965)
PCODE_DEF(OB_CLEAN_SPLITTED_TABLET, 0x965)
PCODE_DEF(OB_REMOTE_WRITE_DDL_FINISH_LOG, 0x966)
PCODE_DEF(OB_REMOTE_WRITE_DDL_INC_COMMIT_LOG, 0x967)
PCODE_DEF(OB_REMOTE_WRITE_DDL_INC_REDO_LOG, 0x968)
//PCODE_DEF(OB_AUTO_SPLIT_TABLET_TASK_REQUEST, 0x969)
//PCODE_DEF(OB_FETCH_SPLIT_TABLET_INFO, 0x96A)
//PCODE_DEF(OB_SPLIT_GLOBAL_INDEX_TABLET, 0x96B)
//PCODE_DEF(OB_PREPARE_TABLET_SPLIT_TASK_RANGES, 0x96C)
PCODE_DEF(OB_AUTO_SPLIT_TABLET_TASK_REQUEST, 0x969)
PCODE_DEF(OB_FETCH_SPLIT_TABLET_INFO, 0x96A)
PCODE_DEF(OB_SPLIT_GLOBAL_INDEX_TABLET, 0x96B)
PCODE_DEF(OB_PREPARE_TABLET_SPLIT_TASK_RANGES, 0x96C)
PCODE_DEF(OB_BATCH_GET_TABLET_BINDING, 0x96D)
//PCODE_DEF(OB_BATCH_UPGRADE_TABLE_SCHEMA, 0x96E)
//PCODE_DEF(OB_SPLIT_TABLET_DATA_START_REQUEST, 0x96F)
//PCODE_DEF(OB_BATCH_GET_TABLET_SPLIT, 0x970)
PCODE_DEF(OB_SPLIT_TABLET_DATA_START_REQUEST, 0x96F)
PCODE_DEF(OB_BATCH_GET_TABLET_SPLIT, 0x970)
// Depedency Detector
PCODE_DEF(OB_DETECTOR_LCL_MESSAGE, 0x9F0)

2
deps/oblib/src/rpc/obrpc/ob_rpc_proxy.h vendored
View File

@ -163,7 +163,7 @@ public:
|| pcode == OB_OUT_TRANS_LOCK_TABLE || pcode == OB_OUT_TRANS_UNLOCK_TABLE
|| pcode == OB_TABLE_LOCK_TASK
|| pcode == OB_HIGH_PRIORITY_TABLE_LOCK_TASK || pcode == OB_BATCH_TABLE_LOCK_TASK
|| pcode == OB_HIGH_PRIORITY_BATCH_TABLE_LOCK_TASK
|| pcode == OB_HIGH_PRIORITY_BATCH_TABLE_LOCK_TASK || pcode == OB_BATCH_REPLACE_TABLE_LOCK_TASK
|| pcode == OB_REGISTER_TX_DATA
|| pcode == OB_REFRESH_SYNC_VALUE || pcode == OB_CLEAR_AUTOINC_CACHE
|| pcode == OB_CLEAN_SEQUENCE_CACHE || pcode == OB_FETCH_TABLET_AUTOINC_SEQ_CACHE

4
mittest/mtlenv/storage/CMakeLists.txt
View File

@ -34,6 +34,10 @@ storage_dml_unittest(test_tablet_aggregated_info test_tablet_aggregated_info.cpp
# storage_unittest(test_speed_limit test_speed_limit.cpp)
storage_dml_unittest(test_tablet_block_id_list test_tablet_block_id_list.cpp)
storage_dml_unittest(test_ls_tablet_info_writer_and_reader test_ls_tablet_info_writer_and_reader.cpp)
storage_dml_unittest(test_auto_partition_split_range test_auto_partition_split_range.cpp)
#storage_dml_unittest(test_auto_partition_estimate)
#storage_dml_unittest(test_auto_partition_get_read_tables)
#storage_dml_unittest(test_auto_partition_scan_table)
storage_dml_unittest(test_transfer_barrier test_transfer_barrier.cpp)
add_subdirectory(checkpoint)

2
mittest/mtlenv/storage/blocksstable/CMakeLists.txt
View File

@ -4,7 +4,7 @@ storage_dml_unittest(test_clustered_index_writer)
storage_dml_unittest(test_index_block_row_struct)
storage_dml_unittest(test_index_block_tree_cursor)
storage_dml_unittest(test_index_block_row_scanner)
storage_unittest(test_index_tree)
storage_dml_unittest(test_index_tree)
storage_dml_unittest(test_index_dumper)
storage_dml_unittest(test_sstable_row_getter)
storage_dml_unittest(test_sstable_row_multi_getter)

2
mittest/mtlenv/storage/blocksstable/ob_index_block_data_prepare.h
View File

@ -256,6 +256,7 @@ int TestIndexBlockDataPrepare::gen_create_tablet_arg(const int64_t tenant_id,
obrpc::ObCreateTabletInfo tablet_info;
ObArray<common::ObTabletID> index_tablet_ids;
ObArray<int64_t> index_tablet_schema_idxs;
ObArray<int64_t> create_commit_versions;
arg.reset();
for(int64_t i = 0; OB_SUCC(ret) && i < count; i++) {
@ -273,6 +274,7 @@ int TestIndexBlockDataPrepare::gen_create_tablet_arg(const int64_t tenant_id,
index_tablet_schema_idxs,
lib::Worker::CompatMode::MYSQL,
false,
create_commit_versions,
false /*has_cs_replica*/))) {
STORAGE_LOG(WARN, "failed to init tablet info", KR(ret), K(index_tablet_ids),
K(tablet_id), K(index_tablet_schema_idxs));

1
mittest/mtlenv/storage/blocksstable/test_sstable_sec_meta_iterator.cpp
View File

@ -16,6 +16,7 @@
#include "storage/ob_partition_range_spliter.h"
#include "storage/blocksstable/index_block/ob_sstable_sec_meta_iterator.h"
#include "storage/blocksstable/index_block/ob_index_block_macro_iterator.h"
#include "storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.h"
#include "ob_index_block_data_prepare.h"

57
mittest/mtlenv/storage/checkpoint/test_lock_memtable_checkpoint.cpp
View File

@ -105,24 +105,15 @@ TEST_F(TestLockMemtableCheckpoint, replay_disorder)
commit_version.set_base();
commit_scn.set_base();
// 1. get ls checkpoint
// 1.recover unlock op and lock op
LOG_INFO("TestLockMemtableCheckpoint::replay_disorder 1");
ObCheckpointExecutor *checkpoint_executor = ls_->get_checkpoint_executor();
ObCommonCheckpoint *checkpoint =
dynamic_cast<ObLSTxService *>(
checkpoint_executor
->handlers_[logservice::TRANS_SERVICE_LOG_BASE_TYPE])
->common_checkpoints_[ObCommonCheckpointType::LOCK_MEMTABLE_TYPE];
// 2.recover unlock op and lock op
LOG_INFO("TestLockMemtableCheckpoint::replay_disorder 2");
ret = memtable_->recover_obj_lock(DEFAULT_OUT_TRANS_UNLOCK_OP);
ASSERT_EQ(OB_SUCCESS, ret);
ret = memtable_->recover_obj_lock(DEFAULT_OUT_TRANS_LOCK_OP);
ASSERT_EQ(OB_SUCCESS, ret);
// 3. update lock status disorder
LOG_INFO("TestLockMemtableCheckpoint::replay_disorder 3");
// 2. update lock status disorder
LOG_INFO("TestLockMemtableCheckpoint::replay_disorder 2");
commit_version.val_ = 3;
commit_scn.val_ = 3;
ret = memtable_->update_lock_status(DEFAULT_OUT_TRANS_UNLOCK_OP,
@ -139,19 +130,17 @@ TEST_F(TestLockMemtableCheckpoint, replay_disorder)
COMMIT_LOCK_OP_STATUS);
ASSERT_EQ(OB_SUCCESS, ret);
// 4. check checkpoint
// 3. check checkpoint
// The rec_scn should be equal with the smaller commit_scn
LOG_INFO("TestLockMemtableCheckpoint::replay_disorder 4");
LOG_INFO("TestLockMemtableCheckpoint::replay_disorder 3");
ASSERT_EQ(commit_scn.val_, memtable_->get_rec_scn().val_);
share::SCN rec_scn = checkpoint->get_rec_scn();
ASSERT_EQ(commit_scn.val_, rec_scn.val_);
// 5. flush and get a previous commit log
// 4. flush and get a previous commit log
// You will find the log about disordered replay in the log file.
LOG_INFO("TestLockMemtableCheckpoint::replay_disorder 5");
LOG_INFO("TestLockMemtableCheckpoint::replay_disorder 4");
ret = memtable_->recover_obj_lock(DEFAULT_OUT_TRANS_UNLOCK_OP);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, checkpoint->flush(share::SCN::max_scn(), 0));
ASSERT_EQ(OB_SUCCESS, memtable_->flush(share::SCN::max_scn(), 0));
commit_version.val_ = 1;
commit_scn.val_ = 1;
ret = memtable_->update_lock_status(DEFAULT_OUT_TRANS_UNLOCK_OP,
@ -160,17 +149,15 @@ TEST_F(TestLockMemtableCheckpoint, replay_disorder)
COMMIT_LOCK_OP_STATUS);
ASSERT_EQ(OB_SUCCESS, ret);
// 6. check checkpoint
// 5. check checkpoint
// The rec_scn should be equal with the smaller commit_scn
// during flushing (i.e. it's get from pre_rec_scn)
LOG_INFO("TestLockMemtableCheckpoint::replay_disorder 6");
LOG_INFO("TestLockMemtableCheckpoint::replay_disorder 5");
ASSERT_EQ(commit_scn.val_, memtable_->get_rec_scn().val_);
rec_scn = checkpoint->get_rec_scn();
ASSERT_EQ(commit_scn.val_, rec_scn.val_);
// 7. get a commit log with a commit_scn which
// 6. get a commit log with a commit_scn which
// is larger than freeze_scn during flushing
LOG_INFO("TestLockMemtableCheckpoint::replay_disorder 7");
LOG_INFO("TestLockMemtableCheckpoint::replay_disorder 6");
ret = memtable_->recover_obj_lock(DEFAULT_OUT_TRANS_LOCK_OP);
ASSERT_EQ(OB_SUCCESS, ret);
commit_version.val_ = 4;
@ -180,29 +167,25 @@ TEST_F(TestLockMemtableCheckpoint, replay_disorder)
commit_scn,
COMMIT_LOCK_OP_STATUS);
// 8. check checkpoint
// 7. check checkpoint
// The rec_scn should still be equal with the smaller
// commit_scn during flushing (i.e. it's get from pre_rec_scn)
LOG_INFO("TestLockMemtableCheckpoint::replay_disorder 8");
LOG_INFO("TestLockMemtableCheckpoint::replay_disorder 7");
ASSERT_EQ(1, memtable_->get_rec_scn().val_);
rec_scn = checkpoint->get_rec_scn();
ASSERT_EQ(1, rec_scn.val_);
// 9. flush finish
LOG_INFO("TestLockMemtableCheckpoint::replay_disorder 9");
// 8. flush finish
LOG_INFO("TestLockMemtableCheckpoint::replay_disorder 8");
ret = memtable_->on_memtable_flushed();
ASSERT_EQ(OB_SUCCESS, ret);
// 10. check checkpoint
// 9. check checkpoint
// The rec_scn should be equal with the latest commit_scn
// which got during previous flushing (i.e. it's get from rec_scn)
LOG_INFO("TestLockMemtableCheckpoint::replay_disorder 10");
LOG_INFO("TestLockMemtableCheckpoint::replay_disorder 9");
ASSERT_EQ(commit_scn.val_, memtable_->get_rec_scn().val_);
rec_scn = checkpoint->get_rec_scn();
ASSERT_EQ(commit_scn.val_, rec_scn.val_);
// 11. clean up
LOG_INFO("TestLockMemtableCheckpoint::replay_disorder 11");
// 10. clean up
LOG_INFO("TestLockMemtableCheckpoint::replay_disorder 10");
table_handle_.reset();
ls_handle_.reset();
ASSERT_EQ(OB_SUCCESS, MTL(ObLSService*)->remove_ls(ls_id_));

566
mittest/mtlenv/storage/test_auto_partition_estimate.cpp Normal file
View File

@ -0,0 +1,566 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#define private public
#define protected public
#include "share/schema/ob_table_param.h"
#include "share/schema/ob_table_dml_param.h"
#include "mtlenv/storage/blocksstable/ob_index_block_data_prepare.h"
namespace oceanbase
{
using namespace common;
using namespace storage;
using namespace share;
namespace unittest
{
typedef ObSEArray<share::schema::ObColDesc, 8> ColDescArray;
static int64_t TEST_COLUMN_CNT=5;
static int64_t TEST_ROWKEY_COLUMN_CNT=1;
class FakeObTabletEstimate : public ::testing::Test
{
public:
FakeObTabletEstimate() :
tenant_id_(OB_SYS_TENANT_ID),
ls_id_(TestDmlCommon::TEST_LS_ID),
tablet_id_(TestDmlCommon::TEST_DATA_TABLE_ID),
access_service_(nullptr),
ls_service_(nullptr),
tx_desc_(nullptr),
table_param_(allocator_)
{}
~FakeObTabletEstimate(){}
virtual void SetUp();
virtual void TearDown();
static void SetUpTestCase();
static void TearDownTestCase();
int prepare_scan_range();
int reset_scan_range();
int prepare_scan_param();
int insert_data();
int set_tablet_split_info(ObTabletID &src_tablet_id, ObTabletID &origin_tablet_id, ObLSID &ls_id, ObTabletSplitType split_type, const int64_t split_cnt);
int estimate_row_count(int64_t &logical_row_count, int64_t &physical_row_count);
int estimate_block_count_and_row_count(ObTabletID &tablet_id, int64_t &macro_block_count, int64_t &micro_block_count, int64_t &sstable_row_count, int64_t &memtable_row_count);
ObLSTabletService *get_ls_service() { return ls_service_; }
int gen_datum_rowkey(const int64_t key_val, const int64_t key_cnt, ObDatumRowkey &rowkey);
ObDatumRowkey &get_start_key() { return start_key_; }
ObDatumRowkey &get_end_key() { return end_key_; }
private:
uint64_t tenant_id_;
share::ObLSID ls_id_;
common::ObTabletID tablet_id_;
MockObAccessService *access_service_;
ObLSTabletService *ls_service_;
transaction::ObTxDesc *tx_desc_;
ColDescArray col_descs_;
ObTableScanRange scan_range_;
ObTableScanParam scan_param_;
ObTableParam table_param_;
ObTableSchema table_schema_;
ObTabletHandle tablet_handle_;
ObArenaAllocator allocator_;
ObDatumRowkey start_key_;
ObDatumRowkey end_key_;
};
void FakeObTabletEstimate::SetUpTestCase()
{
ASSERT_EQ(OB_SUCCESS, MockTenantModuleEnv::get_instance().init());
MTL(transaction::ObTransService*)->tx_desc_mgr_.tx_id_allocator_ =
[](transaction::ObTransID &tx_id) { tx_id = transaction::ObTransID(1001); return OB_SUCCESS; };
ObServerCheckpointSlogHandler::get_instance().is_started_ = true;
}
void FakeObTabletEstimate::TearDownTestCase()
{
MockTenantModuleEnv::get_instance().destroy();
}
void FakeObTabletEstimate::SetUp()
{
int ret = OB_SUCCESS;
ObLSHandle ls_handle;
ret = TestDmlCommon::create_ls(tenant_id_, ls_id_, ls_handle);
ASSERT_EQ(OB_SUCCESS, ret);
// mock ls tablet service and access service
ret = TestDmlCommon::mock_ls_tablet_service(ls_id_, ls_service_);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_NE(nullptr, ls_service_);
ret = TestDmlCommon::mock_access_service(ls_service_, access_service_);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_NE(nullptr, access_service_);
ObTableSchema table_schema;
TestDmlCommon::build_data_table_schema(tenant_id_, table_schema);
ASSERT_EQ(OB_SUCCESS, TestTabletHelper::create_tablet(ls_handle, tablet_id_, table_schema, allocator_));
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id_, tablet_handle_));
ASSERT_EQ(OB_SUCCESS, prepare_scan_param()); // prepare scan range (defaut whole range)
ret = col_descs_.assign(tablet_handle_.get_obj()->get_rowkey_read_info().get_columns_desc());
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, insert_data());
}
void FakeObTabletEstimate::TearDown()
{
access_service_ = nullptr;
tx_desc_ = nullptr;
tablet_handle_.reset();
col_descs_.reset();
scan_range_.reset();
table_param_.reset();
start_key_.reset();
end_key_.reset();
ASSERT_EQ(OB_SUCCESS, MTL(ObLSService*)->remove_ls(ls_id_, false));
TestDmlCommon::delete_mocked_access_service(access_service_);
TestDmlCommon::delete_mocked_ls_tablet_service(ls_service_);
}
int FakeObTabletEstimate::set_tablet_split_info(
ObTabletID &src_tablet_id,
ObTabletID &origin_tablet_id,
ObLSID &ls_id,
ObTabletSplitType split_type,
const int64_t split_count)
{
int ret = OB_SUCCESS;
ObTabletHandle tablet_handle;
ObTabletSplitTscInfo split_info;
ObLSHandle ls_handle;
ObLSID test_ls_id(ls_id_);
if (OB_FAIL(MTL(ObLSService *)->get_ls(test_ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) {
STORAGE_LOG(WARN, "fail to get log stream", K(ret), K(ls_handle));
} else if (OB_UNLIKELY(nullptr == ls_handle.get_ls())) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "ls is null", K(ret), K(ls_handle));
} else if (OB_FAIL(ls_handle.get_ls()->get_tablet(src_tablet_id, tablet_handle))) {
STORAGE_LOG(WARN, "fail to get tablet", K(ret), K(src_tablet_id));
} else if (OB_ISNULL(tablet_handle.get_obj())) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "tablet handle obj is null", K(ret), K(tablet_handle));
} else {
split_info.split_cnt_= split_count;
split_info.split_type_ = split_type;
split_info.start_key_ = start_key_; // not set is invalid
split_info.end_key_ = end_key_; // not set is invalid
split_info.src_tablet_handle_ = tablet_handle;
tablet_handle.get_obj()->set_split_info(split_info);
}
return ret;
}
int FakeObTabletEstimate::prepare_scan_range()
{
int ret = OB_SUCCESS;
if (OB_FAIL(scan_range_.init(scan_param_))) {
STORAGE_LOG(WARN, "fail to init scan range.", K(ret), K(scan_range_));
}
return ret;
}
int FakeObTabletEstimate::reset_scan_range()
{
scan_range_.reset();
return OB_SUCCESS;
}
int FakeObTabletEstimate::estimate_row_count(int64_t &logical_row_count, int64_t &physical_row_count)
{
int ret = OB_SUCCESS;
logical_row_count = 0;
physical_row_count = 0;
common::ObSEArray<common::ObEstRowCountRecord, 2, common::ModulePageAllocator, true> est_records;
ls_service_->enable_to_read();
if (OB_ISNULL(ls_service_)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "error unexpected.", K(ret), K(ls_service_));
} else if (OB_FAIL(ls_service_->estimate_row_count(
scan_param_, scan_range_, est_records, logical_row_count, physical_row_count))) {
STORAGE_LOG(WARN, "fail to do estimate row count", K(ret), K(scan_param_), K(scan_range_));
}
return ret;
}
int FakeObTabletEstimate::insert_data()
{
int ret = OB_SUCCESS;
if (nullptr == access_service_ || OB_ISNULL(tablet_handle_.get_obj()) || OB_ISNULL(tx_desc_)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "null pointer, unexpected.", K(ret), K(access_service_), K(tablet_handle_), K(tx_desc_));
} else {
// insert rows
ObMockNewRowIterator mock_iter;
ObSEArray<uint64_t, 512> column_ids;
column_ids.push_back(OB_APP_MIN_COLUMN_ID + 0); // pk
column_ids.push_back(OB_APP_MIN_COLUMN_ID + 1); // c1
column_ids.push_back(OB_APP_MIN_COLUMN_ID + 2); // c2
column_ids.push_back(OB_APP_MIN_COLUMN_ID + 3); // c3
column_ids.push_back(OB_APP_MIN_COLUMN_ID + 4); // c4
if (OB_FAIL(mock_iter.from(TestDmlCommon::data_row_str))) {
STORAGE_LOG(WARN, "mock iter from fail.", K(ret), K(mock_iter));
} else {
ObTxParam tx_param;
int64_t savepoint = 0;
transaction::ObTransService *tx_service = MTL(transaction::ObTransService*);
if (FALSE_IT(TestDmlCommon::build_tx_param(tx_param))) {
} else if (OB_FAIL(tx_service->create_implicit_savepoint(*tx_desc_, tx_param, savepoint, true))) {
STORAGE_LOG(WARN, "fail to create implicit savepoint.", K(ret), K(tx_desc_), K(tx_param));
} else {
ObTxReadSnapshot read_snapshot;
ObTxIsolationLevel isolation = ObTxIsolationLevel::RC;
int64_t expire_ts = ObTimeUtility::current_time() + TestDmlCommon::TX_EXPIRE_TIME_US;
if (OB_FAIL(tx_service->get_read_snapshot(*tx_desc_, isolation, expire_ts, read_snapshot))) {
STORAGE_LOG(WARN, "fail to get read snapshot.", K(ret), K(expire_ts));
} else {
ObDMLBaseParam dml_param;
dml_param.timeout_ = ObTimeUtility::current_time() + TestDmlCommon::TX_EXPIRE_TIME_US;
dml_param.is_total_quantity_log_ = false;
dml_param.tz_info_ = NULL;
dml_param.sql_mode_ = SMO_DEFAULT;
dml_param.schema_version_ = share::OB_CORE_SCHEMA_VERSION + 1;
dml_param.tenant_schema_version_ = share::OB_CORE_SCHEMA_VERSION + 1;
dml_param.encrypt_meta_ = &dml_param.encrypt_meta_legacy_;
dml_param.snapshot_ = read_snapshot;
share::schema::ObTableDMLParam table_dml_param(allocator_);
if (OB_FAIL(table_dml_param.convert(&table_schema_, 1, column_ids))) {
STORAGE_LOG(WARN, "fail to covert dml param.", K(ret), K(table_schema_));
} else {
dml_param.table_param_ = &table_dml_param;
int64_t affected_rows = 0;
if (OB_FAIL(access_service_->insert_rows(
ls_id_, tablet_id_, *tx_desc_, dml_param, column_ids, &mock_iter, affected_rows))) {
STORAGE_LOG(WARN, "fail to insert rows.", K(ret));
} else if (affected_rows != 12) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "affected rows not equal to insert rows(12).", K(affected_rows));
} else {
expire_ts = ObTimeUtility::current_time() + TestDmlCommon::TX_EXPIRE_TIME_US;
if (OB_FAIL(tx_service->commit_tx(*tx_desc_, expire_ts))) {
STORAGE_LOG(WARN, "fail to commit tx.", K(ret), K(expire_ts));
} else {
tx_service->release_tx(*tx_desc_);
}
}
}
}
}
}
}
return ret;
}
int FakeObTabletEstimate::estimate_block_count_and_row_count(
ObTabletID &tablet_id,
int64_t &macro_block_count,
int64_t &micro_block_count,
int64_t &sstable_row_count,
int64_t &memtable_row_count)
{
int ret = OB_SUCCESS;
macro_block_count = 0;
micro_block_count = 0;
if (!tablet_id.is_valid()) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid tablet id.", K(ret), K(tablet_id));
} else if (OB_ISNULL(ls_service_)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "error unexpected.", K(ret), K(ls_service_));
} else if (OB_FAIL(ls_service_->estimate_block_count_and_row_count(tablet_id, macro_block_count, micro_block_count, sstable_row_count, memtable_row_count))) {
STORAGE_LOG(WARN, "fail to do estimate block count.", K(ret), K(tablet_id));
}
return ret;
}
int FakeObTabletEstimate::prepare_scan_param()
{
int ret = OB_SUCCESS;
// prepare table schema
if (OB_FALSE_IT(TestDmlCommon::build_data_table_schema(tenant_id_, table_schema_))) {
} else if (OB_FAIL(TestDmlCommon::build_tx_desc(tenant_id_, tx_desc_))) { // 1. get tx desc
STORAGE_LOG(WARN, "fail to build tx desc.", K(ret), K(tenant_id_));
} else {
ObTxIsolationLevel isolation = ObTxIsolationLevel::RC;
int64_t expire_ts = ObTimeUtility::current_time() + TestDmlCommon::TX_EXPIRE_TIME_US;
ObTxReadSnapshot read_snapshot;
transaction::ObTransService *tx_service = MTL(transaction::ObTransService*);
if (OB_FAIL(tx_service->get_read_snapshot(*tx_desc_, isolation, expire_ts, read_snapshot))) { // 2. get read snapshot
STORAGE_LOG(WARN, "fail to get_read_snapshot.", K(ret));
} else {
ObSArray<uint64_t> colunm_ids;
colunm_ids.push_back(OB_APP_MIN_COLUMN_ID + 0);
colunm_ids.push_back(OB_APP_MIN_COLUMN_ID + 1);
colunm_ids.push_back(OB_APP_MIN_COLUMN_ID + 2);
colunm_ids.push_back(OB_APP_MIN_COLUMN_ID + 3);
colunm_ids.push_back(OB_APP_MIN_COLUMN_ID + 4);
if (OB_FAIL(TestDmlCommon::build_table_param(table_schema_, colunm_ids, table_param_))) { // build table param
STORAGE_LOG(WARN, "fail to build table param.", K(ret), K(colunm_ids), K(table_schema_));
} else if (OB_FAIL(TestDmlCommon::build_table_scan_param(tenant_id_, read_snapshot, table_param_, scan_param_))) { // 4. build scan param
STORAGE_LOG(WARN, "fail to build table scan param.", K(ret), K(tenant_id_), K(read_snapshot));
}
}
}
return ret;
}
int FakeObTabletEstimate::gen_datum_rowkey(const int64_t key_val, const int64_t key_cnt, ObDatumRowkey &datum_rowkey)
{
int ret = OB_SUCCESS;
ObObj *key_val_obj = NULL;
ObRowkey rowkey;
if (NULL == (key_val_obj = static_cast<ObObj*>(allocator_.alloc(sizeof(ObObj) * key_cnt)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "out of memory", K(ret));
} else {
for (int64_t i = 0; i < key_cnt; ++i) {
key_val_obj[i].set_int(key_val);
// key_val_obj[i].set_min();
}
rowkey.assign(key_val_obj, key_cnt);
if (OB_FAIL(datum_rowkey.from_rowkey(rowkey, allocator_))) {
STORAGE_LOG(WARN, "fail to from rowkey", K(ret));
}
}
return ret;
}
TEST_F(FakeObTabletEstimate, test_split_tablet_estimate)
{
// =================================== not set split info ===============================================
int ret = OB_SUCCESS;
int64_t macro_block_cnt = 0;
int64_t micro_block_cnt = 0;
int64_t sstable_row_cnt = 0;
int64_t memtable_row_cnt = 0;
int64_t logical_row_count = 0;
int64_t physical_row_count = 0;
int64_t split_cnt = 1;
// 1. do not set split info
ObTabletID src_tablet_id(TestDmlCommon::TEST_DATA_TABLE_ID);
ObTabletID ori_tablet_id(TestDmlCommon::TEST_DATA_TABLE_ID);
ObLSID ls_id(TestDmlCommon::TEST_LS_ID);
ASSERT_EQ(OB_SUCCESS, prepare_scan_range());
ret = estimate_row_count(logical_row_count, physical_row_count);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(12, logical_row_count); // TestIndexBlockDataPrepare::12
ret = estimate_block_count_and_row_count(src_tablet_id, macro_block_cnt, micro_block_cnt, sstable_row_cnt, memtable_row_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, reset_scan_range());
// TODO: local index table
// ============================test_estimate_row_set_wrong_split_info===================================
// 2. wrong ls_id;
ObLSID wrong_ls_id(111);
/* main table */
ret = set_tablet_split_info(src_tablet_id, ori_tablet_id, wrong_ls_id, ObTabletSplitType::RANGE, split_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, prepare_scan_range());
ret = estimate_row_count(logical_row_count, physical_row_count);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(12, logical_row_count);
ret = estimate_block_count_and_row_count(src_tablet_id, macro_block_cnt, micro_block_cnt, sstable_row_cnt, memtable_row_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, reset_scan_range());
/* local index table */
ret = set_tablet_split_info(src_tablet_id, ori_tablet_id, wrong_ls_id, ObTabletSplitType::NONE_RANGE, split_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, prepare_scan_range());
ret = estimate_row_count(logical_row_count, physical_row_count);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(12, logical_row_count);
ret = estimate_block_count_and_row_count(src_tablet_id, macro_block_cnt, micro_block_cnt, sstable_row_cnt, memtable_row_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, reset_scan_range());
// 3. wrong ori tablet_id;
ObTabletID wrong_ori_tablet_id(222);
/* main table */
ret = set_tablet_split_info(src_tablet_id, wrong_ori_tablet_id, ls_id, ObTabletSplitType::RANGE, split_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, prepare_scan_range());
ret = estimate_row_count(logical_row_count, physical_row_count);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(12, logical_row_count);
ret = estimate_block_count_and_row_count(src_tablet_id, macro_block_cnt, micro_block_cnt, sstable_row_cnt, memtable_row_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, reset_scan_range());
/* local index table */
ret = set_tablet_split_info(src_tablet_id, wrong_ori_tablet_id, ls_id, ObTabletSplitType::NONE_RANGE, split_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, prepare_scan_range());
ret = estimate_row_count(logical_row_count, physical_row_count);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(12, logical_row_count);
ret = estimate_block_count_and_row_count(src_tablet_id, macro_block_cnt, micro_block_cnt, sstable_row_cnt, memtable_row_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, reset_scan_range());
// 4. wrong split_cnt
split_cnt = 0;
/* main table */
ret = set_tablet_split_info(src_tablet_id, ori_tablet_id, ls_id, ObTabletSplitType::RANGE, split_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, prepare_scan_range());
ret = estimate_row_count(logical_row_count, physical_row_count);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(12, logical_row_count);
ret = estimate_block_count_and_row_count(src_tablet_id, macro_block_cnt, micro_block_cnt, sstable_row_cnt, memtable_row_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, reset_scan_range());
/* local index table */
ret = set_tablet_split_info(src_tablet_id, ori_tablet_id, ls_id, ObTabletSplitType::NONE_RANGE, split_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, prepare_scan_range());
ret = estimate_row_count(logical_row_count, physical_row_count);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(12, logical_row_count);
ret = estimate_block_count_and_row_count(src_tablet_id, macro_block_cnt, micro_block_cnt, sstable_row_cnt, memtable_row_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, reset_scan_range());
// // ==================================== test_estimate_row_right_set_split_info ======================================
int64_t start_val = 0;
int64_t end_val = 0;
// 1. do not split range
split_cnt = 1.0;
/* main table */
ret = set_tablet_split_info(src_tablet_id, ori_tablet_id, ls_id, ObTabletSplitType::RANGE, split_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, prepare_scan_range());
ret = estimate_row_count(logical_row_count, physical_row_count);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(12, logical_row_count); // not set split key, invalid spilt info.
ret = estimate_block_count_and_row_count(src_tablet_id, macro_block_cnt, micro_block_cnt, sstable_row_cnt, memtable_row_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, reset_scan_range());
/* local index table */
ret = set_tablet_split_info(src_tablet_id, ori_tablet_id, ls_id, ObTabletSplitType::NONE_RANGE, split_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, prepare_scan_range());
ret = estimate_row_count(logical_row_count, physical_row_count);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(12, logical_row_count);
ret = estimate_block_count_and_row_count(src_tablet_id, macro_block_cnt, micro_block_cnt, sstable_row_cnt, memtable_row_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, reset_scan_range());
// 2. do split range
split_cnt = 1.0;
start_val = 1; // include eage
end_val = 6; // not include eage
int64_t key_cnt = 1;
ret = gen_datum_rowkey(start_val, key_cnt, get_start_key());
ASSERT_EQ(OB_SUCCESS, ret);
ret = gen_datum_rowkey(end_val, key_cnt, get_end_key());
ASSERT_EQ(OB_SUCCESS, ret);
/* main table */
ret = set_tablet_split_info(src_tablet_id, ori_tablet_id, ls_id, ObTabletSplitType::RANGE, split_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, prepare_scan_range()); // cut scan range using split info
ret = estimate_row_count(logical_row_count, physical_row_count);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(10, logical_row_count); // two tablets
ret = estimate_block_count_and_row_count(src_tablet_id, macro_block_cnt, micro_block_cnt, sstable_row_cnt, memtable_row_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, reset_scan_range());
/* local index table */
ret = set_tablet_split_info(src_tablet_id, ori_tablet_id, ls_id, ObTabletSplitType::NONE_RANGE, split_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, prepare_scan_range()); // cut scan range using split info
ret = estimate_row_count(logical_row_count, physical_row_count);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(24, logical_row_count); // two tablets total data
ret = estimate_block_count_and_row_count(src_tablet_id, macro_block_cnt, micro_block_cnt, sstable_row_cnt, memtable_row_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, reset_scan_range());
// ### because estimate block is calculate block metas count, has no relation with range
// 3. set split ratio 0.5;
split_cnt = 2;
start_val = 1; // include edge
end_val = 6; // not include edge
int64_t key_cnt = 1;
ret = gen_datum_rowkey(start_val, key_cnt, get_start_key());
ASSERT_EQ(OB_SUCCESS, ret);
ret = gen_datum_rowkey(end_val, key_cnt, get_end_key());
ASSERT_EQ(OB_SUCCESS, ret);
/* main table */
ret = set_tablet_split_info(src_tablet_id, ori_tablet_id, ls_id, ObTabletSplitType::RANGE, split_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, prepare_scan_range()); // cut scan range using split info
ret = estimate_row_count(logical_row_count, physical_row_count);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(10, logical_row_count); // counts of two tablets in split range
ret = estimate_block_count_and_row_count(src_tablet_id, macro_block_cnt, micro_block_cnt, sstable_row_cnt, memtable_row_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, reset_scan_range());
/* local index table */
ret = set_tablet_split_info(src_tablet_id, ori_tablet_id, ls_id, ObTabletSplitType::NONE_RANGE, split_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, prepare_scan_range()); // cut scan range using split info
ret = estimate_row_count(logical_row_count, physical_row_count);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(24, logical_row_count);
ret = estimate_block_count_and_row_count(src_tablet_id, macro_block_cnt, micro_block_cnt, sstable_row_cnt, memtable_row_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, reset_scan_range());
// ### because estimate block is calculate block metas count, has no relation with range
}
} // namespace unittest
} // namespace oceanbase
int main(int argc, char **argv)
{
system("rm -rf test_auto_partition_estimate.log");
OB_LOGGER.set_file_name("test_auto_partition_estimate.log", true);
OB_LOGGER.set_log_level("WARN");
CLOG_LOG(INFO, "begin unittest: test_auto_partition_estimate");
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

222
mittest/mtlenv/storage/test_auto_partition_get_read_tables.cpp Normal file
View File

@ -0,0 +1,222 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#define private public
#define protected public
#include "src/storage/ob_storage_struct.h"
#include "src/share/ob_rpc_struct.h"
#include "src/storage/meta_mem/ob_tablet_handle.h"
#include "src/storage/tx_storage/ob_ls_handle.h"
#include "src/storage/tx_storage/ob_ls_service.h"
#include "src/storage/tablet/ob_tablet.h"
#include "storage/tablet/ob_tablet_create_delete_helper.h"
#include "storage/test_tablet_helper.h"
#include "storage/init_basic_struct.h"
#include "mtlenv/mock_tenant_module_env.h"
#include "storage/test_dml_common.h"
#include "mtlenv/mock_tenant_module_env.h"
#include "observer/ob_safe_destroy_thread.h"
namespace oceanbase
{
using namespace common;
using namespace storage;
using namespace share;
namespace unittest
{
static uint64_t TEST_TENANT_ID = 1001;
static int64_t TEST_LS_ID = 101;
static int64_t TEST_SRC_TABLET_ID = 10001;
static int64_t TEST_ORI_TABLET_ID = 10002;
class FakeObGetReadTables : public ::testing::Test
{
public:
static void SetUpTestCase();
static void TearDownTestCase();
public:
FakeObGetReadTables() :
snapshot_version_(0),
allow_not_ready_(false),
split_type_(ObTabletSplitType::MAX_TYPE)
{}
~FakeObGetReadTables()
{}
int set_tablet_split_info(ObTabletID &src_tablet_id, ObTabletID &origin_tablet_id, ObLSID &ls_id);
int gen_datum_rowkey(const int64_t key_val, const int64_t key_cnt, ObDatumRowkey &datum_rowkey);
int64_t get_read_tables_count(ObTabletID &src_tablet_id);
void set_snapshot_version(int64_t snapshot_version) { snapshot_version_ = snapshot_version; }
void set_split_type(ObTabletSplitType split_type) { split_type_ = split_type; }
void set_allow_not_ready(bool need) { allow_not_ready_ = need; }
private:
ObTabletTableIterator iterator_;
static ObArenaAllocator allocator_;
int64_t snapshot_version_;
bool allow_not_ready_;
ObTabletSplitType split_type_;
};
ObArenaAllocator FakeObGetReadTables::allocator_;
void FakeObGetReadTables::SetUpTestCase()
{
int ret = OB_SUCCESS;
ret = MockTenantModuleEnv::get_instance().init();
ASSERT_EQ(OB_SUCCESS, ret);
ObServerCheckpointSlogHandler::get_instance().is_started_ = true;
// create ls
const ObLSID ls_id(TEST_LS_ID);
ObLSHandle ls_handle;
ret = TestDmlCommon::create_ls(TEST_TENANT_ID, ls_id, ls_handle);
ASSERT_EQ(OB_SUCCESS, ret);
const int64_t table_id = 12345;
share::schema::ObTableSchema table_schema;
ASSERT_EQ(OB_SUCCESS, build_test_schema(table_schema, table_id));
// create tablet_1
ObTabletID src_tablet_id(TEST_SRC_TABLET_ID);
ASSERT_EQ(OB_SUCCESS, TestTabletHelper::create_tablet(ls_handle, src_tablet_id, table_schema, allocator_));
// create tablet_2
ObTabletID ori_tablet_id(TEST_ORI_TABLET_ID);
ASSERT_EQ(OB_SUCCESS, TestTabletHelper::create_tablet(ls_handle, ori_tablet_id, table_schema, allocator_));
}
void FakeObGetReadTables::TearDownTestCase()
{
int ret = OB_SUCCESS;
ret = MTL(ObLSService*)->remove_ls(ObLSID(TEST_LS_ID), false);
ASSERT_EQ(OB_SUCCESS, ret);
MockTenantModuleEnv::get_instance().destroy();
}
// return tables count
int64_t FakeObGetReadTables::get_read_tables_count(ObTabletID &src_tablet_id)
{
int ret = OB_SUCCESS;
ObLSHandle ls_handle;
int64_t tables_count = 0;
ObLSID test_ls_id(TEST_LS_ID);
if (OB_FAIL(MTL(ObLSService *)->get_ls(test_ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) {
STORAGE_LOG(WARN, "fail to get log stream", K(ret), K(ls_handle));
} else if (OB_UNLIKELY(nullptr == ls_handle.get_ls())) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "ls is null", K(ret), K(ls_handle));
} else if (OB_FAIL(ls_handle.get_ls()->get_tablet_svr()->get_read_tables(
src_tablet_id,
snapshot_version_,
iterator_,
allow_not_ready_,
true /* get split src table if need */))) {
STORAGE_LOG(WARN, "fail to get tablet tables", K(ret), K(src_tablet_id));
} else {
if (!iterator_.is_valid()) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "invalid iterator.", K(ret), K(src_tablet_id));
} else {
tables_count = iterator_.table_iter()->count();
}
}
return tables_count;
}
int FakeObGetReadTables::gen_datum_rowkey(const int64_t key_val, const int64_t key_cnt, ObDatumRowkey &datum_rowkey)
{
int ret = OB_SUCCESS;
ObObj *key_val_obj = NULL;
ObRowkey rowkey;
if (NULL == (key_val_obj = static_cast<ObObj*>(allocator_.alloc(sizeof(ObObj) * key_cnt)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "out of memory", K(ret));
} else {
for (int64_t i = 0; i < key_cnt; ++i) {
key_val_obj[i].set_int(key_val);
// key_val_obj[i].set_min();
}
rowkey.assign(key_val_obj, key_cnt);
if (OB_FAIL(datum_rowkey.from_rowkey(rowkey, allocator_))) {
STORAGE_LOG(WARN, "fail to from rowkey", K(ret));
}
}
return ret;
}
int FakeObGetReadTables::set_tablet_split_info(ObTabletID &src_tablet_id)
{
int ret = OB_SUCCESS;
ObTabletHandle tablet_handle;
ObTabletSplitTscInfo split_info;
ObLSHandle ls_handle;
ObLSID test_ls_id(TEST_LS_ID);
if (OB_FAIL(MTL(ObLSService *)->get_ls(test_ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) {
STORAGE_LOG(WARN, "fail to get log stream", K(ret), K(ls_handle));
} else if (OB_UNLIKELY(nullptr == ls_handle.get_ls())) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "ls is null", K(ret), K(ls_handle));
} else if (OB_FAIL(ls_handle.get_ls()->get_tablet(src_tablet_id, tablet_handle))) {
STORAGE_LOG(WARN, "fail to get tablet", K(ret), K(src_tablet_id));
} else if (OB_ISNULL(tablet_handle.get_obj())) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "tablet handle obj is null", K(ret), K(tablet_handle));
} else if (OB_FAIL(gen_datum_rowkey(1, 1, split_info.start_key_))) {
STORAGE_LOG(WARN, "gen start key fail.", K(ret));
} else if (OB_FAIL(gen_datum_rowkey(2, 1, split_info.end_key_))) {
STORAGE_LOG(WARN, "gen end key fail.", K(ret));
} else {
split_info.split_cnt_= 1;
split_info.src_tablet_handle_ = tablet_handle;
split_info.split_type_ = ObTabletSplitType::RANGE;
}
return ret;
}
TEST_F(FakeObGetReadTables, test_right_set_split_info)
{
int ret = OB_SUCCESS;
set_snapshot_version(INT64_MAX);
set_allow_not_ready(false);
ObTabletID src_tablet_id(TEST_SRC_TABLET_ID);
ret = set_tablet_split_info(src_tablet_id);
ASSERT_EQ(ret, OB_SUCCESS);
ret = get_read_tables_count(src_tablet_id);
ASSERT_EQ(ret, 2); // both src tablet and origin tables, major;
}
} // namespace unittest
} // namespace oceanbase
int main(int argc, char **argv)
{
system("rm -rf test_auto_partition_get_read_tables.log");
OB_LOGGER.set_file_name("test_auto_partition_get_read_tables.log", true);
OB_LOGGER.set_log_level("WARN");
CLOG_LOG(INFO, "begin unittest: test_auto_partition_get_read_tables");
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

382
mittest/mtlenv/storage/test_auto_partition_scan_table.cpp Normal file
View File

@ -0,0 +1,382 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#define private public
#define protected public
#include "share/schema/ob_table_param.h"
#include "share/schema/ob_table_dml_param.h"
#include "mtlenv/storage/blocksstable/ob_index_block_data_prepare.h"
/*
0. 用主表模拟局部索引表扫描，两者差别在于主表会进行范围切割，局部索引表会进行 行过滤
1. 创建分区
2. 设置分区为split状态，把自己设置为局部索引，并且把origin tablet id设置为自己
3. 启动table_scan
4. 预期逻辑
1）同时获取origin table的tables ；
2）不会cut range；
3）根据当前tablet id 过滤数据，当然当前代码里分裂后的schema没有变更，计算分区方式也没有改变，因此数据全部符合
*/
namespace oceanbase
{
using namespace common;
using namespace storage;
using namespace share;
namespace unittest
{
class FakeObScanTable : public ::testing::Test
{
public:
FakeObScanTable() :
tenant_id_(OB_SYS_TENANT_ID),
ls_id_(TestDmlCommon::TEST_LS_ID),
tablet_id_(TestDmlCommon::TEST_DATA_TABLE_ID),
orig_tablet_id_(51)
{}
~FakeObScanTable(){}
static void SetUpTestCase();
static void TearDownTestCase();
void table_scan(ObAccessService *access_service, ObNewRowIterator *&result);
void insert_data_to_tablet(MockObAccessService *access_service, ObTabletID &tablet_id, const char *data);
int set_tablet_split_info(ObTabletID &src_tablet_id, ObTabletSplitType split_type, const int64_t split_cnt);
int gen_datum_rowkey(const int64_t key_val, const int64_t key_cnt, ObDatumRowkey &datum_rowkey);
ObDatumRowkey &get_start_key() { return start_key_; }
ObDatumRowkey &get_end_key() { return end_key_; }
protected:
uint64_t tenant_id_;
share::ObLSID ls_id_;
common::ObTabletID tablet_id_;
common::ObTabletID orig_tablet_id_;
private:
ObArenaAllocator allocator_;
ObRowkey start_key_;
ObRowkey end_key_;
public:
static constexpr const char *data_row_str_01 =
"bigint bigint bigint var var dml \n"
"1 62 20 Houston Rockets T_DML_INSERT \n"
"2 65 17 SanAntonio Spurs T_DML_INSERT \n"
"3 58 24 Dallas Mavericks T_DML_INSERT \n"
"4 51 31 LosAngeles Lakers T_DML_INSERT \n"
"5 57 25 Phoenix Suns T_DML_INSERT \n"
"6 32 50 NewJersey Nets T_DML_INSERT \n"
"7 44 38 Miami Heats T_DML_INSERT \n"
"8 21 61 Chicago Bulls T_DML_INSERT \n"
"9 47 35 Cleveland Cavaliers T_DML_INSERT \n"
"10 59 23 Detroit Pistons T_DML_INSERT \n"
"11 40 42 Utah Jazz T_DML_INSERT \n"
"12 50 32 Boston Celtics T_DML_INSERT \n";
static constexpr const char *data_row_str_02 =
"bigint bigint bigint var var dml \n"
"13 62 20 Houston Rockets T_DML_INSERT \n"
"14 65 17 SanAntonio Spurs T_DML_INSERT \n"
"15 58 24 Dallas Mavericks T_DML_INSERT \n"
"16 51 31 LosAngeles Lakers T_DML_INSERT \n"
"17 57 25 Phoenix Suns T_DML_INSERT \n"
"18 32 50 NewJersey Nets T_DML_INSERT \n"
"19 44 38 Miami Heats T_DML_INSERT \n"
"20 21 61 Chicago Bulls T_DML_INSERT \n"
"21 47 35 Cleveland Cavaliers T_DML_INSERT \n"
"22 59 23 Detroit Pistons T_DML_INSERT \n"
"23 40 42 Utah Jazz T_DML_INSERT \n"
"24 50 32 Boston Celtics T_DML_INSERT \n";
};
void FakeObScanTable::SetUpTestCase()
{
ASSERT_EQ(OB_SUCCESS, MockTenantModuleEnv::get_instance().init());
MTL(transaction::ObTransService*)->tx_desc_mgr_.tx_id_allocator_ =
[](transaction::ObTransID &tx_id) { tx_id = transaction::ObTransID(1001); return OB_SUCCESS; };
ObServerCheckpointSlogHandler::get_instance().is_started_ = true;
}
void FakeObScanTable::TearDownTestCase()
{
MockTenantModuleEnv::get_instance().destroy();
}
int FakeObScanTable::set_tablet_split_info(
ObTabletID &src_tablet_id,
ObTabletSplitType split_type,
const int64_t split_cnt)
{
int ret = OB_SUCCESS;
ObTabletHandle tablet_handle;
ObTabletSplitTscInfo split_info;
ObLSHandle ls_handle;
ObLSID test_ls_id(ls_id_);
if (OB_FAIL(MTL(ObLSService *)->get_ls(test_ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) {
STORAGE_LOG(WARN, "fail to get log stream", K(ret), K(ls_handle));
} else if (OB_UNLIKELY(nullptr == ls_handle.get_ls())) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "ls is null", K(ret), K(ls_handle));
} else if (OB_FAIL(ls_handle.get_ls()->get_tablet(src_tablet_id, tablet_handle))) {
STORAGE_LOG(WARN, "fail to get tablet", K(ret), K(src_tablet_id));
} else if (OB_ISNULL(tablet_handle.get_obj())) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "tablet handle obj is null", K(ret), K(tablet_handle));
} else {
split_info.split_cnt_= split_cnt;
split_info.split_type_ = split_type;
split_info.start_key_ = start_key_; // not set is invalid
split_info.end_key_ = end_key_; // not set is invalid
split_info.src_tablet_handle_ = tablet_handle;
// tablet_handle.get_obj()->set_split_info(split_info);
}
return ret;
}
int FakeObScanTable::gen_datum_rowkey(const int64_t key_val, const int64_t key_cnt, ObDatumRowkey &datum_rowkey)
{
int ret = OB_SUCCESS;
ObObj *key_val_obj = NULL;
ObRowkey rowkey;
if (NULL == (key_val_obj = static_cast<ObObj*>(allocator_.alloc(sizeof(ObObj) * key_cnt)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "out of memory", K(ret));
} else {
for (int64_t i = 0; i < key_cnt; ++i) {
key_val_obj[i].set_int(key_val);
// key_val_obj[i].set_min();
}
rowkey.assign(key_val_obj, key_cnt);
if (OB_FAIL(datum_rowkey.from_rowkey(rowkey, allocator_))) {
STORAGE_LOG(WARN, "fail to from rowkey", K(ret));
}
}
return ret;
}
void FakeObScanTable::insert_data_to_tablet(MockObAccessService *access_service, ObTabletID &tablet_id, const char *data)
{
ASSERT_NE(nullptr, access_service);
ASSERT_NE(nullptr, data);
ObLSHandle ls_handle;
ObTabletHandle tablet_handle;
ObLSService *ls_svr = MTL(ObLSService*);
ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD));
ObLS *ls = ls_handle.get_ls();
ASSERT_NE(nullptr, ls);
ASSERT_EQ(OB_SUCCESS, ls->get_tablet(tablet_id, tablet_handle));
ObTablet *tablet = tablet_handle.get_obj();
ASSERT_NE(nullptr, tablet);
// insert rows
ObMockNewRowIterator mock_iter;
ObSEArray<uint64_t, 512> column_ids;
column_ids.push_back(OB_APP_MIN_COLUMN_ID + 0); // pk
column_ids.push_back(OB_APP_MIN_COLUMN_ID + 1); // c1
column_ids.push_back(OB_APP_MIN_COLUMN_ID + 2); // c2
column_ids.push_back(OB_APP_MIN_COLUMN_ID + 3); // c3
column_ids.push_back(OB_APP_MIN_COLUMN_ID + 4); // c4
ASSERT_EQ(OB_SUCCESS, mock_iter.from(data));
transaction::ObTransService *tx_service = MTL(transaction::ObTransService*);
// 1. get tx desc
transaction::ObTxDesc *tx_desc = nullptr;
ASSERT_EQ(OB_SUCCESS, TestDmlCommon::build_tx_desc(tenant_id_, tx_desc));
// 2. create savepoint (can be rollbacked)
ObTxParam tx_param;
TestDmlCommon::build_tx_param(tx_param);
int64_t savepoint = 0;
ASSERT_EQ(OB_SUCCESS, tx_service->create_implicit_savepoint(*tx_desc, tx_param, savepoint, true));
// 3. acquire snapshot (write also need snapshot)
ObTxIsolationLevel isolation = ObTxIsolationLevel::RC;
int64_t expire_ts = ObTimeUtility::current_time() + TestDmlCommon::TX_EXPIRE_TIME_US;
ObTxReadSnapshot read_snapshot;
ASSERT_EQ(OB_SUCCESS, tx_service->get_read_snapshot(*tx_desc, isolation, expire_ts, read_snapshot));
// 4. storage dml
ObDMLBaseParam dml_param;
dml_param.timeout_ = ObTimeUtility::current_time() + TestDmlCommon::TX_EXPIRE_TIME_US;
dml_param.is_total_quantity_log_ = false;
dml_param.tz_info_ = NULL;
dml_param.sql_mode_ = SMO_DEFAULT;
dml_param.schema_version_ = share::OB_CORE_SCHEMA_VERSION + 1;
dml_param.tenant_schema_version_ = share::OB_CORE_SCHEMA_VERSION + 1;
dml_param.encrypt_meta_ = &dml_param.encrypt_meta_legacy_;
dml_param.snapshot_ = read_snapshot;
ObArenaAllocator allocator;
share::schema::ObTableDMLParam table_dml_param(allocator);
share::schema::ObTableSchema table_schema;
TestDmlCommon::build_data_table_schema(tenant_id_, table_schema);
ASSERT_EQ(OB_SUCCESS, table_dml_param.convert(&table_schema, 1, column_ids));
dml_param.table_param_ = &table_dml_param;
int64_t affected_rows = 0;
ASSERT_EQ(OB_SUCCESS, access_service->insert_rows(ls_id_, tablet_id,
*tx_desc, dml_param, column_ids, &mock_iter, affected_rows));
ASSERT_EQ(12, affected_rows);
// 5. submit transaction, or rollback
expire_ts = ObTimeUtility::current_time() + TestDmlCommon::TX_EXPIRE_TIME_US;
ASSERT_EQ(OB_SUCCESS, tx_service->commit_tx(*tx_desc, expire_ts));
// 6. release tx desc
tx_service->release_tx(*tx_desc);
}
void FakeObScanTable::table_scan(
ObAccessService *access_service,
ObNewRowIterator *&result)
{
// prepare table schema
share::schema::ObTableSchema table_schema;
TestDmlCommon::build_data_table_schema(tenant_id_, table_schema);
// 1. get tx desc
transaction::ObTxDesc *tx_desc = nullptr;
ASSERT_EQ(OB_SUCCESS, TestDmlCommon::build_tx_desc(tenant_id_, tx_desc));
// 2. get read snapshot
ObTxIsolationLevel isolation = ObTxIsolationLevel::RC;
int64_t expire_ts = ObTimeUtility::current_time() + TestDmlCommon::TX_EXPIRE_TIME_US;
ObTxReadSnapshot read_snapshot;
transaction::ObTransService *tx_service = MTL(transaction::ObTransService*);
ASSERT_EQ(OB_SUCCESS, tx_service->get_read_snapshot(*tx_desc, isolation, expire_ts, read_snapshot));
// 3. storage dml
// build table param
ObArenaAllocator allocator;
share::schema::ObTableParam table_param(allocator);
ObSArray<uint64_t> colunm_ids;
colunm_ids.push_back(OB_APP_MIN_COLUMN_ID + 0);
colunm_ids.push_back(OB_APP_MIN_COLUMN_ID + 1);
colunm_ids.push_back(OB_APP_MIN_COLUMN_ID + 2);
colunm_ids.push_back(OB_APP_MIN_COLUMN_ID + 3);
colunm_ids.push_back(OB_APP_MIN_COLUMN_ID + 4);
ASSERT_EQ(OB_SUCCESS, TestDmlCommon::build_table_param(table_schema, colunm_ids, table_param));
ObTableScanParam scan_param;
ASSERT_EQ(OB_SUCCESS, TestDmlCommon::build_table_scan_param(tenant_id_, read_snapshot, table_param, scan_param));
const share::ObLSID ls_id(TestDmlCommon::TEST_LS_ID);
ASSERT_EQ(OB_SUCCESS, access_service->table_scan(scan_param, result));
ASSERT_TRUE(nullptr != result);
// print data
int ret = OB_SUCCESS;
int cnt = 0;
ObNewRow *row = nullptr;
while (OB_SUCC(ret)) {
ret = result->get_next_row(row);
if (OB_SUCCESS == ret) {
++cnt;
}
STORAGE_LOG(WARN, "table scan row", KPC(row));
}
ASSERT_EQ(24, cnt);
// 4. submit transaction, or rollback
expire_ts = ObTimeUtility::current_time() + TestDmlCommon::TX_EXPIRE_TIME_US;
ASSERT_EQ(OB_SUCCESS, tx_service->commit_tx(*tx_desc, expire_ts));
// 5. release tx desc
tx_service->release_tx(*tx_desc);
}
TEST_F(FakeObScanTable, table_scan_pure_data_table)
{
int ret = OB_SUCCESS;
ObLSHandle ls_handle;
ret = TestDmlCommon::create_ls(tenant_id_, ls_id_, ls_handle);
ASSERT_EQ(OB_SUCCESS, ret);
// mock ls tablet service and access service
ObLSTabletService *tablet_service = nullptr;
ret = TestDmlCommon::mock_ls_tablet_service(ls_id_, tablet_service);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_NE(nullptr, tablet_service);
MockObAccessService *access_service = nullptr;
ret = TestDmlCommon::mock_access_service(tablet_service, access_service);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_NE(nullptr, access_service);
ObTableSchema table_schema;
TestDmlCommon::build_data_table_schema(tenant_id_, table_schema);
ASSERT_EQ(OB_SUCCESS, TestTabletHelper::create_tablet(ls_handle, tablet_id_, table_schema, allocator_));
TestDmlCommon::build_data_table_schema(tenant_id_, table_schema);
ASSERT_EQ(OB_SUCCESS, TestTabletHelper::create_tablet(ls_handle, orig_tablet_id_, table_schema, allocator_));
insert_data_to_tablet(access_service, tablet_id_, data_row_str_01);
insert_data_to_tablet(access_service, orig_tablet_id_, data_row_str_02);
// set split info
ObTabletID src_tablet_id(TestDmlCommon::TEST_DATA_TABLE_ID);
ObTabletID ori_tablet_id(51);
int64_t split_cnt = 1.0;
int64_t start_val = 0; // not include eage
int64_t end_val = 26; // not include eage
int64_t key_cnt = 1;
ret = gen_datum_rowkey(start_val, key_cnt, get_start_key());
ASSERT_EQ(OB_SUCCESS, ret);
ret = gen_datum_rowkey(end_val, key_cnt, get_end_key());
ASSERT_EQ(OB_SUCCESS, ret);
ret = set_tablet_split_info(src_tablet_id, ObTabletSplitType::RANGE, split_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
// table scan
ObNewRowIterator *iter = nullptr;
table_scan(access_service, iter);
// clean env
TestDmlCommon::delete_mocked_access_service(access_service);
TestDmlCommon::delete_mocked_ls_tablet_service(tablet_service);
// for exist
// the iter has store ctx and store ctx has one ls handle.
iter->reset();
ASSERT_EQ(OB_SUCCESS, MTL(ObLSService*)->remove_ls(ls_id_, false));
}
} // namespace unittest
} // namespace oceanbase
int main(int argc, char **argv)
{
system("rm -rf test_auto_partition_scan_table.log");
OB_LOGGER.set_file_name("test_auto_partition_scan_table.log", true);
OB_LOGGER.set_log_level("WARN");
CLOG_LOG(INFO, "begin unittest: test_auto_partition_scan_table");
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

1059
mittest/mtlenv/storage/test_auto_partition_split_range.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

3
mittest/mtlenv/storage/test_ddl_create_tablet.cpp
View File

@ -298,13 +298,14 @@ int TestDDLCreateTablet::build_create_tablet_arg(
obrpc::ObCreateTabletInfo tablet_info;
ObArray<common::ObTabletID> tablet_id_array;
ObArray<int64_t> tablet_schema_index_array;
ObArray<int64_t> create_commit_versions;
TestSchemaUtils::prepare_data_schema(data_table_schema);
if (OB_FAIL(tablet_id_array.push_back(data_tablet_id))) {
STORAGE_LOG(WARN, "failed to push tablet id into array", K(ret), K(data_tablet_id));
} else if (OB_FAIL(tablet_schema_index_array.push_back(0))) {
STORAGE_LOG(WARN, "failed to push index into array", K(ret));
} else if (OB_FAIL(tablet_info.init(tablet_id_array, data_tablet_id, tablet_schema_index_array,
lib::get_compat_mode(), false/*is_create_bind_hidden_tablets*/, false /*has_cs_replica*/))) {
lib::get_compat_mode(), false/*is_create_bind_hidden_tablets*/, create_commit_versions, false /*has_cs_replica*/))) {
STORAGE_LOG(WARN, "failed to init tablet info", K(ret), K(tablet_id_array),
K(data_tablet_id), K(tablet_schema_index_array));
} else if (OB_FAIL(arg.tablets_.push_back(tablet_info))) {

6
mittest/mtlenv/storage/test_ls_migration_param.cpp
View File

@ -253,8 +253,7 @@ TEST_F(TestLSMigrationParam, test_migrate_tablet_param)
SCN scn;
scn.convert_from_ts(ObTimeUtility::current_time());
ret = src_handle.get_obj()->init_for_first_time_creation(allocator_, src_key.ls_id_, src_key.tablet_id_, src_key.tablet_id_,
scn, 2022, create_tablet_schema, true/*need_create_empty_major_sstable*/,
true/*micro_index_clustered*/, false/*need_generate_cs_replica_cg_array*/, false/*has_cs_replica*/, ls_handle.get_ls()->get_freezer());
scn, 2022, create_tablet_schema, true/*need_create_empty_major_sstable*/, SCN::invalid_scn(), true/*micro_index_clustered*/, false/*need_generate_cs_replica_cg_array*/, false/*has_cs_replica*/, ls_handle.get_ls()->get_freezer());
ASSERT_EQ(common::OB_SUCCESS, ret);
share::SCN create_commit_scn;
@ -338,8 +337,7 @@ TEST_F(TestLSMigrationParam, test_migration_param_compat)
SCN scn;
scn.convert_from_ts(ObTimeUtility::current_time());
ret = src_handle.get_obj()->init_for_first_time_creation(allocator_, src_key.ls_id_, src_key.tablet_id_, src_key.tablet_id_,
scn, 2022, create_tablet_schema, true/*need_create_empty_major_sstable*/,
true/*micro_index_clustered*/, false/*need_generate_cs_replica_cg_array*/, false/*has_cs_replica*/, ls_handle.get_ls()->get_freezer());
scn, 2022, create_tablet_schema, true/*need_create_empty_major_sstable*/, SCN::invalid_scn(), true/*micro_index_clustered*/, false/*need_generate_cs_replica_cg_array*/, false/*has_cs_replica*/, ls_handle.get_ls()->get_freezer());
ASSERT_EQ(common::OB_SUCCESS, ret);
share::SCN create_commit_scn;

3
mittest/mtlenv/storage/test_ls_tablet_info_writer_and_reader.cpp
View File

@ -170,8 +170,7 @@ void TestLSTabletInfoWR::fill_tablet_meta()
SCN scn;
scn.convert_from_ts(ObTimeUtility::current_time());
ret = src_handle.get_obj()->init_for_first_time_creation(arena_allocator_, src_key.ls_id_, src_key.tablet_id_, src_key.tablet_id_,
scn, 2022, create_tablet_schema, true/*need_create_empty_major_sstable*/,
false/*micro_index_clustered*/, false/*need_generate_cs_replica_cg_array*/, false/*has_cs_replica*/, ls_handle.get_ls()->get_freezer());
scn, 2022, create_tablet_schema, true/*need_create_empty_major_sstable*/, share::SCN::invalid_scn(), false/*micro_index_clustered*/, false/*need_generate_cs_replica_cg_array*/, false/*has_cs_replica*/, ls_handle.get_ls()->get_freezer());
ASSERT_EQ(common::OB_SUCCESS, ret);
share::SCN create_commit_scn;

15
mittest/mtlenv/storage/test_tenant_meta_mem_mgr.cpp
View File

@ -728,8 +728,7 @@ TEST_F(TestTenantMetaMemMgr, test_wash_tablet)
ObTabletID empty_tablet_id;
ret = tablet->init_for_first_time_creation(allocator_, ls_id_, tablet_id, tablet_id,
create_scn, create_scn.get_val_for_tx(), create_tablet_schema, true/*need_create_empty_major_sstable*/,
false/*micro_index_clustered*/, false/*need_generate_cs_replica_cg_array*/, false/*has_cs_replica*/, &freezer);
create_scn, create_scn.get_val_for_tx(), create_tablet_schema, true/*need_create_empty_major_sstable*/, SCN::invalid_scn(), false/*micro_index_clustered*/, false/*need_generate_cs_replica_cg_array*/, false/*has_cs_replica*/, &freezer);
ASSERT_EQ(common::OB_SUCCESS, ret);
ASSERT_EQ(1, tablet->get_ref());
const ObTabletPersisterParam persist_param(ls_id_, ls_handle.get_ls()->get_ls_epoch(), tablet_id, tablet->get_transfer_seq());
@ -830,8 +829,7 @@ TEST_F(TestTenantMetaMemMgr, test_wash_inner_tablet)
bool make_empty_co_sstable = true;
ret = tablet->init_for_first_time_creation(allocator_, ls_id_, tablet_id, tablet_id,
create_scn, create_scn.get_val_for_tx(), create_tablet_schema,
make_empty_co_sstable/*need_create_empty_major_sstable*/,
false/*micro_index_clustered*/, false/*need_generate_cs_replica_cg_array*/, false/*has_cs_replica*/, &freezer);
make_empty_co_sstable/*need_create_empty_major_sstable*/, share::SCN::invalid_scn(), false/*micro_index_clustered*/, false/*need_generate_cs_replica_cg_array*/, false/*has_cs_replica*/, &freezer);
ASSERT_EQ(common::OB_SUCCESS, ret);
ASSERT_EQ(1, tablet->get_ref());
@ -944,8 +942,7 @@ TEST_F(TestTenantMetaMemMgr, test_wash_no_sstable_tablet)
bool make_empty_co_sstable = false;
ret = tablet->init_for_first_time_creation(allocator_, ls_id_, tablet_id, tablet_id,
create_scn, create_scn.get_val_for_tx(), create_tablet_schema,
make_empty_co_sstable/*need_create_empty_major_sstable*/,
false/*micro_index_clustered*/, false/*need_generate_cs_replica_cg_array*/, false/*has_cs_replica*/, &freezer);
make_empty_co_sstable/*need_create_empty_major_sstable*/, share::SCN::invalid_scn(), false/*micro_index_clustered*/, false/*need_generate_cs_replica_cg_array*/, false/*has_cs_replica*/, &freezer);
ASSERT_EQ(common::OB_SUCCESS, ret);
ASSERT_EQ(1, tablet->get_ref());
@ -1044,8 +1041,7 @@ TEST_F(TestTenantMetaMemMgr, test_get_tablet_with_allocator)
bool make_empty_co_sstable = true;
ret = tablet->init_for_first_time_creation(allocator_, ls_id_, tablet_id, tablet_id,
create_scn, create_scn.get_val_for_tx(), create_tablet_schema,
make_empty_co_sstable/*need_create_empty_major_sstable*/,
false/*micro_index_clustered*/, false/*need_generate_cs_replica_cg_array*/, false/*has_cs_replica*/, &freezer);
make_empty_co_sstable/*need_create_empty_major_sstable*/, share::SCN::invalid_scn(), false/*micro_index_clustered*/, false/*need_generate_cs_replica_cg_array*/, false/*has_cs_replica*/, &freezer);
ASSERT_EQ(common::OB_SUCCESS, ret);
ASSERT_EQ(1, tablet->get_ref());
@ -1177,8 +1173,7 @@ TEST_F(TestTenantMetaMemMgr, test_wash_mem_tablet)
bool make_empty_co_sstable = false;
ret = tablet->init_for_first_time_creation(allocator_, ls_id_, tablet_id, tablet_id,
create_scn, create_scn.get_val_for_tx(), create_tablet_schema,
make_empty_co_sstable/*need_create_empty_major_sstable*/,
false/*micro_index_clustered*/, false/*need_generate_cs_replica_cg_array*/, false/*has_cs_replica*/, &freezer);
make_empty_co_sstable/*need_create_empty_major_sstable*/, share::SCN::invalid_scn(), false/*micro_index_clustered*/, false/*need_generate_cs_replica_cg_array*/, false/*has_cs_replica*/, &freezer);
ASSERT_EQ(common::OB_SUCCESS, ret);
ASSERT_EQ(1, tablet->get_ref());

5
mittest/mtlenv/tablelock/table_lock_common_env.h
View File

@ -35,7 +35,7 @@ ObLockID DEFAULT_TABLE_LOCK_ID;
ObLockID TABLE_LOCK_ID2;
ObLockID TABLE_LOCK_ID3;
ObTableLockMode DEFAULT_LOCK_MODE = ROW_EXCLUSIVE;
ObTableLockMode DEFAULT_COFLICT_LOCK_MODE = EXCLUSIVE;
ObTableLockMode DEFAULT_CONFLICT_LOCK_MODE = EXCLUSIVE;
ObTableLockOwnerID DEFAULT_IN_TRANS_OWNER_ID(ObTableLockOwnerID::default_owner());
ObTableLockOwnerID DEFAULT_OUT_TRANS_OWNER_ID(ObTableLockOwnerID::get_owner_by_value(100));
ObTableLockOwnerID CONFLICT_OWNER_ID(ObTableLockOwnerID::get_owner_by_value(1));
@ -55,6 +55,7 @@ ObTableLockOp DEFAULT_OUT_TRANS_LOCK_OP;
ObTableLockOp DEFAULT_OUT_TRANS_UNLOCK_OP;
ObTableLockOp DEFAULT_CONFLICT_OUT_TRANS_LOCK_OP;
ObTableLockOp DEFAULT_OUT_TRANS_REPLACE_LOCK_OP;
void init_default_lock_test_value()
{
@ -108,7 +109,7 @@ void init_default_lock_test_value()
create_schema_version);
DEFAULT_CONFLICT_OUT_TRANS_LOCK_OP.set(DEFAULT_TABLET_LOCK_ID,
DEFAULT_COFLICT_LOCK_MODE,
DEFAULT_CONFLICT_LOCK_MODE,
CONFLICT_OWNER_ID,
TRANS_ID2,
OUT_TRANS_LOCK_OP_TYPE,

162
mittest/mtlenv/tablelock/test_lock_memtable.cpp
View File

@ -561,7 +561,8 @@ TEST_F(TestLockMemtable, pre_check_lock)
LOG_INFO("TestLockMemtable::pre_check_lock 1.2");
ret = memtable_.check_lock_conflict(mem_ctx,
DEFAULT_IN_TRANS_LOCK_OP,
conflict_tx_set);
conflict_tx_set,
expired_time);
ASSERT_EQ(OB_SUCCESS, ret);
// 1.3 lock
LOG_INFO("TestLockMemtable::pre_check_lock 1.3");
@ -575,7 +576,8 @@ TEST_F(TestLockMemtable, pre_check_lock)
LOG_INFO("TestLockMemtable::pre_check_lock 1.4");
ret = memtable_.check_lock_conflict(mem_ctx,
DEFAULT_IN_TRANS_LOCK_OP,
conflict_tx_set);
conflict_tx_set,
expired_time);
ASSERT_EQ(OB_SUCCESS, ret);
// 1.5 check allow lock
LOG_INFO("TestLockMemtable::pre_check_lock 1.5");
@ -583,7 +585,8 @@ TEST_F(TestLockMemtable, pre_check_lock)
lock_op.lock_mode_ = ROW_SHARE;
ret = memtable_.check_lock_conflict(mem_ctx,
lock_op,
conflict_tx_set);
conflict_tx_set,
expired_time);
ASSERT_EQ(OB_SUCCESS, ret);
// 1.6 remove lock op at memtable.
LOG_INFO("TestLockMemtable::pre_check_lock 1.6");
@ -1161,7 +1164,7 @@ TEST_F(TestLockMemtable, test_lock_retry_lock_conflict)
ObTableLockOp lock_first = DEFAULT_OUT_TRANS_LOCK_OP; // RX, owner 0
ObTableLockOp lock_second = DEFAULT_OUT_TRANS_LOCK_OP;
lock_second.lock_mode_ = DEFAULT_COFLICT_LOCK_MODE; // X
lock_second.lock_mode_ = DEFAULT_CONFLICT_LOCK_MODE; // X
lock_second.owner_id_ = CONFLICT_OWNER_ID; // owner 1
MyTxCtx default_ctx;
@ -1298,6 +1301,157 @@ TEST_F(TestLockMemtable, test_lock_retry_lock_conflict)
ASSERT_EQ(min_commited_scn, share::SCN::max_scn());
}
TEST_F(TestLockMemtable, test_replace)
{
LOG_INFO("TestLockMemtable::test_replace");
int ret = OB_SUCCESS;
bool is_try_lock = false;
int64_t expired_time = ObClockGenerator::getClock() + 10 * 1000 * 1000;
ObReplaceLockParam param;
ObMemtableCtx *mem_ctx = nullptr;
bool lock_exist = false;
share::SCN min_commited_scn;
share::SCN flushed_scn;
uint64_t lock_mode_cnt_in_same_trans[TABLE_LOCK_MODE_COUNT] = {0, 0, 0, 0, 0};
ObTableLockOp lock_first = DEFAULT_OUT_TRANS_LOCK_OP; // RX, owner 0
ObTableLockOp lock_second = DEFAULT_OUT_TRANS_LOCK_OP;
lock_second.lock_mode_ = DEFAULT_CONFLICT_LOCK_MODE; // X
lock_second.owner_id_ = CONFLICT_OWNER_ID; // owner 1
ObTableLockOp unlock_first = DEFAULT_OUT_TRANS_UNLOCK_OP;
ObTableLockOp unlock_second = DEFAULT_OUT_TRANS_UNLOCK_OP;
unlock_second.lock_mode_ = DEFAULT_CONFLICT_LOCK_MODE;
unlock_second.owner_id_ = CONFLICT_OWNER_ID;
MyTxCtx ctx1;
ObStoreCtx store_ctx1;
MyTxCtx ctx2;
ObStoreCtx store_ctx2;
start_tx(DEFAULT_TRANS_ID, ctx1);
get_store_ctx(ctx1, store_ctx1);
ctx1.tx_ctx_.change_to_leader();
start_tx(TRANS_ID2, ctx2);
get_store_ctx(ctx2, store_ctx2);
ctx2.tx_ctx_.change_to_leader();
LOG_INFO("TestLockMemtable::test_replace 1");
// 1.1 lock first
LOG_INFO("TestLockMemtable::test_replace 1.1");
param.is_try_lock_ = is_try_lock;
param.expired_time_ = expired_time;
ret = memtable_.lock(param,
store_ctx1,
lock_first);
ASSERT_EQ(OB_SUCCESS, ret);
// 1.2 check lock exist at memctx.
LOG_INFO("TestLockMemtable::test_replace 1.2");
mem_ctx = store_ctx1.mvcc_acc_ctx_.mem_ctx_;
ret = mem_ctx->check_lock_exist(lock_first.lock_id_,
lock_first.owner_id_,
lock_first.lock_mode_,
lock_first.op_type_,
lock_exist,
lock_mode_cnt_in_same_trans);
ASSERT_EQ(lock_exist, true);
// 1.3 commit out trans lock
LOG_INFO("TestLockMemtable::test_replace 1.3");
share::SCN commit_version;
share::SCN commit_scn;
commit_version.set_base();
commit_scn.set_base();
ret = memtable_.update_lock_status(lock_first,
commit_version,
commit_scn,
COMMIT_LOCK_OP_STATUS);
ASSERT_EQ(OB_SUCCESS, ret);
min_commited_scn = memtable_.obj_lock_map_.get_min_ddl_committed_scn(
flushed_scn);
ASSERT_EQ(min_commited_scn, commit_scn);
memtable_.obj_lock_map_.print();
// 2. replace lock_first with lock_second
LOG_INFO("TestLockMemtable::test_replace 2");
// 2.1 replace to owner 1
LOG_INFO("TestLockMemtable::test_replace 2.1");
param.is_for_replace_ = true;
ret = memtable_.replace(store_ctx2, param, unlock_first, lock_second);
ASSERT_EQ(OB_SUCCESS, ret);
memtable_.obj_lock_map_.print();
// 2.2 check lock exist at memctx
LOG_INFO("TestLockMemtable::test_replace 2.2");
mem_ctx = store_ctx2.mvcc_acc_ctx_.mem_ctx_;
ret = mem_ctx->check_lock_exist(lock_second.lock_id_,
lock_second.owner_id_,
lock_second.lock_mode_,
lock_second.op_type_,
lock_exist,
lock_mode_cnt_in_same_trans);
ASSERT_EQ(lock_exist, true);
// 2.3 commit replace lock
// We should commit all lock_ops in the same tx,
// so we commit unlock_op and lock_op together here
LOG_INFO("TestLockMemtable::test_replace 2.3");
commit_version.set_base();
commit_scn.set_base();
ret = memtable_.update_lock_status(unlock_first,
commit_version,
commit_scn,
COMMIT_LOCK_OP_STATUS);
ASSERT_EQ(OB_SUCCESS, ret);
ret = memtable_.update_lock_status(lock_second,
commit_version,
commit_scn,
COMMIT_LOCK_OP_STATUS);
ASSERT_EQ(OB_SUCCESS, ret);
min_commited_scn = memtable_.obj_lock_map_.get_min_ddl_committed_scn(
flushed_scn);
ASSERT_EQ(min_commited_scn, commit_scn);
memtable_.obj_lock_map_.print();
// 3. unlock
LOG_INFO("TestLockMemtable::test_replace 3");
MyTxCtx ctx3;
ObStoreCtx unlock_store_ctx3;
start_tx(TRANS_ID3, ctx3);
get_store_ctx(ctx3, unlock_store_ctx3);
ctx3.tx_ctx_.change_to_leader();
// 3.1 unlock with owner 0: should be OB_OBJ_LOCK_NOT_EXIST.
LOG_INFO("TestLockMemtable::test_replace 3.1");
ret = memtable_.unlock(unlock_store_ctx3,
unlock_first,
is_try_lock,
expired_time);
ASSERT_EQ(OB_OBJ_LOCK_NOT_EXIST, ret);
// 3.2 unlock with owner 1: should be success
LOG_INFO("TestLockMemtable::test_replace 3.2");
ret = memtable_.unlock(unlock_store_ctx3,
unlock_second,
is_try_lock,
expired_time);
ASSERT_EQ(OB_SUCCESS, ret);
// 3.3 commit unlock
LOG_INFO("TestLockMemtable::test_replace 3.3");
commit_version.set_base();
commit_scn.set_base();
ret = memtable_.update_lock_status(unlock_second,
commit_version,
commit_scn,
COMMIT_LOCK_OP_STATUS);
ASSERT_EQ(OB_SUCCESS, ret);
min_commited_scn = memtable_.obj_lock_map_.get_min_ddl_committed_scn(
flushed_scn);
ASSERT_EQ(min_commited_scn, share::SCN::max_scn());
memtable_.obj_lock_map_.print();
}
} // tablelock
} // transaction
} // oceanbase

2
mittest/mtlenv/tablelock/test_lock_table_callback.cpp
View File

@ -140,7 +140,7 @@ TEST_F(TestLockTableCallback, callback)
bool lock_exist = false;
uint64_t lock_mode_cnt_in_same_trans[TABLE_LOCK_MODE_COUNT] = {0, 0, 0, 0, 0};
const bool for_replay = false;
ObIMemtable *memtable = nullptr;
storage::ObIMemtable *memtable = nullptr;
ObOBJLockCallback *cb = nullptr;
// 1. UNNSED CALLBACK TYPE
LOG_INFO("TestLockTableCallback::callback 1.");

2
mittest/simple_server/CMakeLists.txt
View File

@ -77,6 +77,7 @@ ob_unittest_observer(test_ob_obj_lock_garbage_collector test_ob_obj_lock_garbage
ob_unittest_observer(test_observer_expand_shrink test_observer_expand_shrink.cpp)
ob_unittest_observer(test_replay_from_middle test_replay_from_middle.cpp)
ob_unittest_observer(test_special_tablet_flush test_special_tablet_flush.cpp)
ob_unittest_observer(test_table_lock_split test_table_lock_split.cpp)
ob_unittest_observer(test_tx_data_table_mit test_tx_data_table_mit.cpp)
ob_unittest_observer(test_tx_ctx_table_mit test_tx_ctx_table_mit.cpp)
ob_unittest_observer(test_lock_table_persistence test_lock_table_persistence.cpp)
@ -106,6 +107,7 @@ ob_unittest_observer(test_get_stopped_zone_list test_get_stopped_zone_list.cpp)
ob_unittest_observer(test_lock_table_with_tx test_lock_table_with_tx.cpp)
ob_unittest_observer(test_ob_detect_manager_in_simple_server test_ob_detect_manager_in_simple_server.cpp)
ob_unittest_observer(test_transfer_lock_info_operator storage_ha/test_transfer_lock_info_operator.cpp)
ob_unittest_observer(test_tablet_reorganize_history_table_operator storage_ha/test_backup_tablet_reorganize_helper.cpp)
ob_unittest_observer(test_mds_recover test_mds_recover.cpp)
ob_unittest_observer(test_keep_alive_min_start_scn test_keep_alive_min_start_scn.cpp)
ob_unittest_observer(test_ls_replica test_ls_replica.cpp)

4
mittest/simple_server/test_callbacks_with_reverse_order.cpp
View File

@ -135,7 +135,9 @@ int ObStorageTableGuard::refresh_and_protect_memtable_for_write(ObRelativeTable
store_ctx_.mvcc_acc_ctx_.get_snapshot_version().get_val_for_tx(),
store_ctx_.mvcc_acc_ctx_.get_snapshot_version().get_val_for_tx(),
iter,
relative_table.allow_not_ready()))) {
relative_table.allow_not_ready(),
true/*need_split_src_table*/,
false/*need_split_dst_table*/))) {
TRANS_LOG(WARN, "fail to get", K(store_ctx_.mvcc_acc_ctx_.tx_id_), K(ret));
} else {
// no worry. iter will hold tablet reference and its life cycle is longer than guard

22
mittest/simple_server/test_lock_table_persistence.cpp
View File

@ -140,8 +140,10 @@ TEST_F(ObLockTableBeforeRestartTest, test_lock_table_flush)
share::ObLSID ls_id = share::LOCK_SERVICE_LS;
ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, handle, ObLSGetMod::STORAGE_MOD));
ASSERT_NE(nullptr, ls = handle.get_ls());
ObCheckpointExecutor *checkpoint_executor = ls->get_checkpoint_executor();
ASSERT_NE(nullptr, checkpoint_executor);
ObTableHandleV2 table_handle;
ObLockMemtable *lock_memtable = nullptr;
ASSERT_EQ(OB_SUCCESS, ls->lock_table_.get_lock_memtable(table_handle));
ASSERT_EQ(OB_SUCCESS, table_handle.get_lock_memtable(lock_memtable));
ObSchemaGetterGuard guard;
ASSERT_EQ(OB_SUCCESS, GCTX.schema_service_->get_tenant_schema_guard(RunCtx.tenant_id_, guard));
@ -155,11 +157,6 @@ TEST_F(ObLockTableBeforeRestartTest, test_lock_table_flush)
ASSERT_EQ(OB_SUCCESS, table_lock_ser->lock_table(table_id, EXCLUSIVE, owner_id, 0));
usleep(1000 * 1000);
ObLockMemtable *lock_memtable
= dynamic_cast<ObLockMemtable *>(dynamic_cast<ObLSTxService *>(checkpoint_executor
->handlers_[logservice::TRANS_SERVICE_LOG_BASE_TYPE])
->common_checkpoints_[ObCommonCheckpointType::LOCK_MEMTABLE_TYPE]);
SCN rec_scn = lock_memtable->get_rec_scn();
ASSERT_NE(rec_scn, SCN::max_scn());
lock_scn = rec_scn;
@ -251,17 +248,14 @@ TEST_F(ObLockTableAfterRestartTest, test_recover_lock_table)
share::ObLSID ls_id = share::LOCK_SERVICE_LS;
ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, handle, ObLSGetMod::STORAGE_MOD));
ASSERT_NE(nullptr, ls = handle.get_ls());
ObCheckpointExecutor *checkpoint_executor = ls->get_checkpoint_executor();
ASSERT_NE(nullptr, checkpoint_executor);
// get lock_scn from table
select_existed_data(lock_scn, unlock_scn);
// check lock_memtable scn
ObLockMemtable *lock_memtable
= dynamic_cast<ObLockMemtable *>(dynamic_cast<ObLSTxService *>(checkpoint_executor
->handlers_[logservice::TRANS_SERVICE_LOG_BASE_TYPE])
->common_checkpoints_[ObCommonCheckpointType::LOCK_MEMTABLE_TYPE]);
ObTableHandleV2 table_handle;
ObLockMemtable *lock_memtable = nullptr;
ASSERT_EQ(OB_SUCCESS, ls->lock_table_.get_lock_memtable(table_handle));
ASSERT_EQ(OB_SUCCESS, table_handle.get_lock_memtable(lock_memtable));
ASSERT_EQ(lock_memtable->get_rec_scn(), unlock_scn);
ASSERT_EQ(lock_memtable->flushed_scn_, lock_scn);
ASSERT_EQ(lock_memtable->max_committed_scn_, unlock_scn);

22
mittest/simple_server/test_lock_table_with_tx.cpp
View File

@ -184,9 +184,12 @@ TEST_F(ObLockTableBeforeRestartTest, test_commit_log)
checkpoint_executor = ls->get_checkpoint_executor();
ASSERT_NE(nullptr, checkpoint_executor);
lock_memtable = dynamic_cast<ObLockMemtable *>(dynamic_cast<ObLSTxService *>(checkpoint_executor
->handlers_[logservice::TRANS_SERVICE_LOG_BASE_TYPE])
->common_checkpoints_[ObCommonCheckpointType::LOCK_MEMTABLE_TYPE]);
// lock_memtable = dynamic_cast<ObLockMemtable *>(dynamic_cast<ObLSTxService *>(checkpoint_executor
// ->handlers_[logservice::TRANS_SERVICE_LOG_BASE_TYPE])
// ->common_checkpoints_[ObCommonCheckpointType::LOCK_MEMTABLE_TYPE]);
ObTableHandleV2 table_handle;
ASSERT_EQ(OB_SUCCESS, ls->lock_table_.get_lock_memtable(table_handle));
ASSERT_EQ(OB_SUCCESS, table_handle.get_lock_memtable(lock_memtable));
lock_memtable->obj_lock_map_.print();
LOG_INFO("ObLockTableBeforeRestartTest::test_commit_log 1.2 wait tablelock committed");
@ -310,10 +313,15 @@ TEST_F(ObLockTableAfterRestartTest, test_recover_lock_table)
select_existed_data(lock_scn, ls_checkpoint_scn);
// check lock_memtable scn
ObLockMemtable *lock_memtable
= dynamic_cast<ObLockMemtable *>(dynamic_cast<ObLSTxService *>(checkpoint_executor
->handlers_[logservice::TRANS_SERVICE_LOG_BASE_TYPE])
->common_checkpoints_[ObCommonCheckpointType::LOCK_MEMTABLE_TYPE]);
// ObLockMemtable *lock_memtable
// = dynamic_cast<ObLockMemtable *>(dynamic_cast<ObLSTxService *>(checkpoint_executor
// ->handlers_[logservice::TRANS_SERVICE_LOG_BASE_TYPE])
// ->common_checkpoints_[ObCommonCheckpointType::LOCK_MEMTABLE_TYPE]);
ObTableHandleV2 table_handle;
ObLockMemtable *lock_memtable;
ASSERT_EQ(OB_SUCCESS, ls->lock_table_.get_lock_memtable(table_handle));
ASSERT_EQ(OB_SUCCESS, table_handle.get_lock_memtable(lock_memtable));
lock_memtable->obj_lock_map_.print();
ASSERT_EQ(lock_memtable->get_rec_scn(), lock_scn);
ASSERT_EQ(lock_memtable->max_committed_scn_, lock_scn);

4
mittest/simple_server/test_memtable_new_safe_to_destroy.cpp
View File

@ -222,13 +222,13 @@ TEST_F(ObTestMemtableNewSafeToDestroy, test_safe_to_destroy)
storage::ObTabletMemtableMgr *memtable_mgr = memtable->get_memtable_mgr();
EXPECT_EQ(OB_SUCCESS, memtable_mgr->release_memtables());
TRANS_LOG(INFO, "qcc print2", KPC(memtable));;
TRANS_LOG(INFO, "qcc print2", KPC(memtable));
ObTabletHandle tablet_handle;
get_tablet(tenant_id, share::ObLSID(1001), tablet_id, tablet_handle);
tablet_handle.get_obj()->reset_memtable();
TRANS_LOG(INFO, "qcc print3", KPC(memtable));;
TRANS_LOG(INFO, "qcc print3", KPC(memtable));
usleep(5 * 1000 * 1000);

1062
mittest/simple_server/test_ob_table_lock_service.cpp
View File

File diff suppressed because it is too large Load Diff

8
mittest/simple_server/test_special_tablet_flush.cpp
View File

@ -228,10 +228,10 @@ TEST_F(ObTabletFlushTest, test_special_tablet_flush)
->handlers_[logservice::TRANS_SERVICE_LOG_BASE_TYPE])
->common_checkpoints_[ObCommonCheckpointType::TX_CTX_MEMTABLE_TYPE]);
ObLockMemtable *lock_memtable
= dynamic_cast<ObLockMemtable *>(dynamic_cast<ObLSTxService *>(checkpoint_executor
->handlers_[logservice::TRANS_SERVICE_LOG_BASE_TYPE])
->common_checkpoints_[ObCommonCheckpointType::LOCK_MEMTABLE_TYPE]);
ObTableHandleV2 table_handle;
ObLockMemtable *lock_memtable = nullptr;
ASSERT_EQ(OB_SUCCESS, ls->lock_table_.get_lock_memtable(table_handle));
ASSERT_EQ(OB_SUCCESS, table_handle.get_lock_memtable(lock_memtable));
ObTxDataMemtableMgr *tx_data_mgr
= dynamic_cast<ObTxDataMemtableMgr *>(dynamic_cast<ObLSTxService *>(checkpoint_executor

284
mittest/simple_server/test_table_lock_split.cpp Normal file
View File

@ -0,0 +1,284 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#define USING_LOG_PREFIX TABLELOCK
#define protected public
#define private public
#include "env/ob_simple_cluster_test_base.h"
#include "env/ob_simple_server_restart_helper.h"
#include "storage/tx_storage/ob_ls_service.h"
#include "storage/tablelock/ob_lock_memtable.h"
#include "logservice/ob_log_base_type.h"
#include "mtlenv/tablelock/table_lock_tx_common_env.h"
#include "storage/tx_storage/ob_ls_handle.h" //ObLSHandle
static const char *TEST_FILE_NAME = "test_table_lock_split";
static const char *BORN_CASE_NAME = "ObLockTableSplitBeforeRestartTest";
static const char *RESTART_CASE_NAME = "ObLockTableSplitAfterRestartTest";
namespace oceanbase
{
namespace unittest
{
using namespace oceanbase::transaction;
using namespace oceanbase::storage;
using namespace oceanbase::storage::checkpoint;
class TestRunCtx
{
public:
uint64_t tenant_id_ = 0;
int time_sec_ = 0;
};
TestRunCtx RunCtx;
class ObLockTableSplitBeforeRestartTest : public ObSimpleClusterTestBase
{
public:
ObLockTableSplitBeforeRestartTest() : ObSimpleClusterTestBase(TEST_FILE_NAME) {}
void add_rx_in_trans_lock(sqlclient::ObISQLConnection *&connection);
void commit_tx(sqlclient::ObISQLConnection *&connection);
};
#define EXE_SQL(sql_str) \
ASSERT_EQ(OB_SUCCESS, sql.assign(sql_str)); \
ASSERT_EQ(OB_SUCCESS, sql_proxy.write(sql.ptr(), affected_rows));
#define EXE_SQL_FMT(...) \
ASSERT_EQ(OB_SUCCESS, sql.assign_fmt(__VA_ARGS__)); \
ASSERT_EQ(OB_SUCCESS, sql_proxy.write(sql.ptr(), affected_rows));
#define WRITE_SQL_BY_CONN(conn, sql_str) \
ASSERT_EQ(OB_SUCCESS, sql.assign(sql_str)); \
ASSERT_EQ(OB_SUCCESS, conn->execute_write(OB_SYS_TENANT_ID, sql.ptr(), affected_rows));
#define WRITE_SQL_FMT_BY_CONN(conn, ...) \
ASSERT_EQ(OB_SUCCESS, sql.assign_fmt(__VA_ARGS__)); \
ASSERT_EQ(OB_SUCCESS, conn->execute_write(OB_SYS_TENANT_ID, sql.ptr(), affected_rows));
void ObLockTableSplitBeforeRestartTest::add_rx_in_trans_lock(sqlclient::ObISQLConnection *&connection)
{
LOG_INFO("insert data start");
// common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2();
int64_t affected_rows = 0;
ObSqlString sql;
// sqlclient::ObISQLConnection *connection = nullptr;
// ASSERT_EQ(OB_SUCCESS, sql_proxy.acquire(connection));
ASSERT_NE(nullptr, connection);
WRITE_SQL_BY_CONN(connection, "set ob_trx_timeout = 3000000000");
WRITE_SQL_BY_CONN(connection, "set ob_trx_idle_timeout = 3000000000");
WRITE_SQL_BY_CONN(connection, "set ob_query_timeout = 3000000000");
WRITE_SQL_BY_CONN(connection, "set autocommit=0");
// start and commit transaction
// create data in tx data table.
WRITE_SQL_BY_CONN(connection, "begin;");
WRITE_SQL_FMT_BY_CONN(connection, "insert into test_table_lock_split_t values(1, 1);");
}
void ObLockTableSplitBeforeRestartTest::commit_tx(sqlclient::ObISQLConnection *&connection)
{
int64_t affected_rows = 0;
ObSqlString sql;
ASSERT_NE(nullptr, connection);
WRITE_SQL_BY_CONN(connection, "commit;");
ObTransService *txs = MTL(ObTransService*);
share::ObLSID ls_id(1001);
ASSERT_EQ(OB_SUCCESS, txs->get_tx_ctx_mgr().clear_all_tx(ls_id));
}
TEST_F(ObLockTableSplitBeforeRestartTest, add_tenant)
{
// create tenant
LOG_INFO("step 1: 创建普通租户tt1");
ASSERT_EQ(OB_SUCCESS, create_tenant());
LOG_INFO("step 2: 获取租户tt1的tenant_id");
ASSERT_EQ(OB_SUCCESS, get_tenant_id(RunCtx.tenant_id_));
LOG_INFO("step 3: 初始化普通租户tt1的sql proxy");
ASSERT_NE(0, RunCtx.tenant_id_);
ASSERT_EQ(OB_SUCCESS, get_curr_simple_server().init_sql_proxy2());
}
TEST_F(ObLockTableSplitBeforeRestartTest, create_table)
{
common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2();
{
OB_LOG(INFO, "create_table start");
ObSqlString sql;
int64_t affected_rows = 0;
EXE_SQL("create table test_table_lock_split_t (c1 int, c2 int, primary key(c1))");
OB_LOG(INFO, "create_table succ");
}
}
TEST_F(ObLockTableSplitBeforeRestartTest, test_table_lock_split)
{
// switch tenant
share::ObTenantSwitchGuard tenant_guard;
ASSERT_EQ(OB_SUCCESS, tenant_guard.switch_to(RunCtx.tenant_id_));
common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2();
sqlclient::ObISQLConnection *connection = nullptr;
ASSERT_EQ(OB_SUCCESS, sql_proxy.acquire(connection));
add_rx_in_trans_lock(connection);
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
ObLSService *ls_svr = MTL(ObLSService*);
ASSERT_NE(nullptr, ls_svr);
ObLS *ls = nullptr;
ObLSHandle handle;
share::ObLSID ls_id(1001);
ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, handle, ObLSGetMod::STORAGE_MOD));
ASSERT_NE(nullptr, ls = handle.get_ls());
ObTableHandleV2 table_handle;
ObLockMemtable *lock_memtable = nullptr;
ASSERT_EQ(OB_SUCCESS, ls->lock_table_.get_lock_memtable(table_handle));
ASSERT_EQ(OB_SUCCESS, table_handle.get_lock_memtable(lock_memtable));
// table_lock split
ObTabletID src_tablet_id(200001);
ObSArray<common::ObTabletID> dst_tablet_ids;
ObTabletID dst_tablet_id1(300000);
dst_tablet_ids.push_back(dst_tablet_id1);
ObTabletID dst_tablet_id2(400000);
dst_tablet_ids.push_back(dst_tablet_id2);
ObTransID trans_id(1000);
ObOBJLock *obj_lock = NULL;
ObLockID lock_id;
ASSERT_EQ(OB_SUCCESS, get_lock_id(dst_tablet_id1, lock_id));
ASSERT_EQ(OB_SUCCESS, lock_memtable->table_lock_split(src_tablet_id, dst_tablet_ids, trans_id));
int retry_time = 0;
while (OB_FAIL(lock_memtable->obj_lock_map_.get_obj_lock_with_ref_(lock_id, obj_lock))) {
usleep(1000 * 1000);
retry_time++;
if (retry_time % 5 == 0) {
OB_LOG(WARN, "wait log callback use too much time", K(retry_time));
}
}
ASSERT_NE(share::SCN::min_scn(), obj_lock->max_split_epoch_);
// share::SCN split_scn = obj_lock->max_split_epoch_;
// ASSERT_EQ(lock_memtable->get_rec_scn(), split_scn);
// ASSERT_EQ(OB_SUCCESS, lock_memtable->flush(share::SCN::max_scn(), true));
// ASSERT_EQ(lock_memtable->freeze_scn_, split_scn);
// int retry_time = 0;
// while (lock_memtable->is_frozen_memtable()) {
// usleep(1000 * 1000);
// retry_time++;
// if (retry_time % 5 == 0) {
// OB_LOG(WARN, "wait lock memtable flush finish use too much time",
// K(retry_time), KPC(lock_memtable));
// }
// }
// ASSERT_EQ(lock_memtable->get_rec_scn(), share::SCN::max_scn());
// ASSERT_EQ(lock_memtable->flushed_scn_, lock_memtable->freeze_scn_);
// // conflict with split lock
// ObStoreCtx store_ctx;
// store_ctx.ls_id_ = ls_id;
// transaction::tablelock::ObTableLockOp out_trans_lock_op;
// out_trans_lock_op.lock_id_ = lock_id;
// out_trans_lock_op.lock_mode_ = SHARE;
// out_trans_lock_op.op_type_ = OUT_TRANS_LOCK;
// out_trans_lock_op.lock_op_status_ = LOCK_OP_DOING;
// ASSERT_EQ(OB_TRY_LOCK_ROW_CONFLICT, obj_lock->eliminate_conflict_caused_by_split_if_need_(out_trans_lock_op, store_ctx));
// commit_tx(connection);
// ASSERT_EQ(OB_SUCCESS, obj_lock->eliminate_conflict_caused_by_split_if_need_(out_trans_lock_op, store_ctx));
// ASSERT_EQ(share::SCN::min_scn(), obj_lock->max_split_epoch_);
// ASSERT_EQ(OB_SUCCESS, ls->lock_table_.offline());
// ASSERT_EQ(OB_SUCCESS, ls->lock_table_.load_lock());
// ASSERT_EQ(OB_SUCCESS, get_lock_id(dst_tablet_id1, lock_id));
// ASSERT_EQ(OB_SUCCESS, ls->lock_table_.get_lock_memtable(table_handle));
// ASSERT_EQ(OB_SUCCESS, table_handle.get_lock_memtable(lock_memtable));
// ASSERT_EQ(OB_SUCCESS, lock_memtable->obj_lock_map_.get_obj_lock_with_ref_(lock_id, obj_lock));
// ASSERT_EQ(split_scn, obj_lock->max_split_epoch_);
}
class ObLockTableSplitAfterRestartTest : public ObSimpleClusterTestBase
{
public:
ObLockTableSplitAfterRestartTest() : ObSimpleClusterTestBase(TEST_FILE_NAME) {}
};
TEST_F(ObLockTableSplitAfterRestartTest, test_recover)
{
// ============================== restart successfully ==============================
//switch tenant
uint64_t tenant_id = 0;
ASSERT_EQ(OB_SUCCESS, get_tenant_id(tenant_id));
share::ObTenantSwitchGuard tenant_guard;
ASSERT_EQ(OB_SUCCESS, tenant_guard.switch_to(tenant_id));
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
ObLSService *ls_svr = MTL(ObLSService*);
ASSERT_NE(nullptr, ls_svr);
ObLS *ls = nullptr;
ObLSHandle handle;
share::ObLSID ls_id(1001);
ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, handle, ObLSGetMod::STORAGE_MOD));
ASSERT_NE(nullptr, ls = handle.get_ls());
// check lock_memtable scn
ObTableHandleV2 table_handle;
ObLockMemtable *lock_memtable = nullptr;
ASSERT_EQ(OB_SUCCESS, ls->lock_table_.get_lock_memtable(table_handle));
ASSERT_EQ(OB_SUCCESS, table_handle.get_lock_memtable(lock_memtable));
ObOBJLock *obj_lock = NULL;
ObLockID lock_id;
ObTabletID dst_tablet_id1(300000);
ASSERT_EQ(OB_SUCCESS, get_lock_id(dst_tablet_id1, lock_id));
ASSERT_EQ(OB_SUCCESS, lock_memtable->obj_lock_map_.get_obj_lock_with_ref_(lock_id, obj_lock));
ASSERT_NE(share::SCN::min_scn(), obj_lock->max_split_epoch_);
}
} //unitest
} //oceanbase
int main(int argc, char **argv)
{
int c = 0;
int time_sec = 0;
char *log_level = (char *)"INFO";
while (EOF != (c = getopt(argc, argv, "t:l:"))) {
switch (c) {
case 't':
time_sec = atoi(optarg);
break;
case 'l':
log_level = optarg;
oceanbase::unittest::ObSimpleClusterTestBase::enable_env_warn_log_ = false;
break;
default:
break;
}
}
std::string gtest_file_name = std::string(TEST_FILE_NAME) + "_gtest.log";
oceanbase::unittest::init_gtest_output(gtest_file_name);
int ret = 0;
ObSimpleServerRestartHelper restart_helper(argc, argv, TEST_FILE_NAME, BORN_CASE_NAME,
RESTART_CASE_NAME);
restart_helper.set_sleep_sec(time_sec + 10); // sleep 10s for schema restore
restart_helper.run();
return ret;
}

3
src/logservice/libobcdc/src/ob_cdc_lob_ctx.cpp
View File

@ -104,6 +104,9 @@ void ObLobDataGetCtx::reset(
} else if (lob_data_out_row_ctx->is_diff()) {
type_ = ObLobDataGetTaskType::EXT_INFO_LOG;
LOG_DEBUG("lob_data_out_row_ctx is diff", K(column_id), KPC(new_lob_data_), KPC(lob_data_out_row_ctx));
} else if (ObLobDataOutRowCtx::OpType::EXT_INFO_LOG == lob_data_out_row_ctx->op_) {
type_ = ObLobDataGetTaskType::EXT_INFO_LOG;
LOG_DEBUG("lob_data_out_row_ctx is outrow ext", K(column_id), KPC(new_lob_data_), KPC(lob_data_out_row_ctx));
}
}

62
src/logservice/libobcdc/src/ob_cdc_lob_data_merger.cpp
View File

@ -480,14 +480,21 @@ int ObCDCLobDataMerger::handle_when_outrow_log_fragment_progress_done_(
if (OB_FAIL(merge_fragments_(task, lob_data_get_ctx, lob_data_out_row_ctx_list, src_data))) {
LOG_ERROR("merge_fragments_ fail", KR(ret), K(task), K(lob_data_get_ctx), K(lob_data_out_row_ctx_list));
} else {
// if is_progress_done is false, it means that after_fragment_progress_done_ cannot be called to push task info formatter
// if task_type is FULL_LOB, current src_data is the final data, so is_progress_done always is true
// if task_type is EXT_INFO_LOG, there is further judgment based on the ext info log type
// for OB_JSON_DIFF_EXT_INFO_LOG, is_progress_done is true after handling
// for OB_OUTROW_DISK_LOB_LOCATOR_EXT_INFO_LOG, it is not final data, and the merge task will be generated, so is_progress_done is false
bool is_progress_done = false;
ObLobDataGetTaskType task_type = lob_data_get_ctx.get_type();
LOG_DEBUG("handle", K(task_type), K(task), K(lob_data_get_ctx), K(lob_data_out_row_ctx_list), K(src_data));
switch (task_type) {
case ObLobDataGetTaskType::FULL_LOB:
output_data = src_data;
is_progress_done = true;
break;
case ObLobDataGetTaskType::EXT_INFO_LOG:
if (OB_FAIL(handle_ext_info_log_(lob_data_get_ctx, lob_data_out_row_ctx_list, src_data, output_data))) {
if (OB_FAIL(handle_ext_info_log_(lob_data_get_ctx, lob_data_out_row_ctx_list, src_data, output_data, is_progress_done, stop_flag))) {
LOG_ERROR("handle ext info log fail", KR(ret), K(task_type), K(task), K(lob_data_get_ctx), K(lob_data_out_row_ctx_list));
}
break;
@ -497,6 +504,8 @@ int ObCDCLobDataMerger::handle_when_outrow_log_fragment_progress_done_(
break;
}
if (OB_FAIL(ret)) {
} else if (! is_progress_done) {
LOG_DEBUG("progress not done", K(task_type), "src_data_length", src_data.length());
} else if (OB_FAIL(lob_data_get_ctx.set_col_value(is_new_col, output_data.ptr(), output_data.length()))) {
LOG_ERROR("lob_data_get_ctx set_col_value failed", KR(ret), K(src_data), K(output_data));
} else if (OB_FAIL(after_fragment_progress_done_(lob_data_get_ctx, lob_data_out_row_ctx_list, stop_flag))) {
@ -555,7 +564,7 @@ int ObCDCLobDataMerger::merge_fragments_(
if (pos + len > data_len) {
ret = OB_BUF_NOT_ENOUGH;
LOG_ERROR("buf not enough, not expected", KR(ret), K(pos), K(len), K(data_len));
LOG_ERROR("buf not enough, not expected", KR(ret), K(pos), K(len), K(data_len), K(seq_no_cnt), K(idx));
} else {
MEMCPY(buf + pos, ptr, len);
pos += len;
@ -644,7 +653,9 @@ int ObCDCLobDataMerger::handle_ext_info_log_(
ObLobDataGetCtx &lob_data_get_ctx,
ObLobDataOutRowCtxList &lob_data_out_row_ctx_list,
const ObString &src_data,
ObString &output_data)
ObString &output_data,
bool &is_progress_done,
volatile bool &stop_flag)
{
int ret = OB_SUCCESS;
storage::ObExtInfoLogHeader ext_info_log_header;
@ -663,6 +674,16 @@ int ObCDCLobDataMerger::handle_ext_info_log_(
case storage::OB_JSON_DIFF_EXT_INFO_LOG:
if (OB_FAIL(handle_json_diff_ext_info_log_(allocator, src_data.ptr(), src_data.length(), pos, output_data))) {
LOG_ERROR("handle_json_diff_ext_info_log_ fail", KR(ret), K(src_data), K(ext_info_log_header), K(lob_data_get_ctx), K(lob_data_out_row_ctx_list));
} else {
is_progress_done = true;
}
break;
case storage::OB_OUTROW_DISK_LOB_LOCATOR_EXT_INFO_LOG:
if (OB_FAIL(handle_outrow_lob_locator_ext_info_log_(allocator, src_data.ptr(), src_data.length(), pos, lob_data_get_ctx, lob_data_out_row_ctx_list, stop_flag))) {
LOG_ERROR("handle_outrow_lob_locator_ext_info_log_ fail", KR(ret), K(src_data), K(ext_info_log_header), K(lob_data_get_ctx), K(lob_data_out_row_ctx_list));
} else {
// it is not final data, and the new merge task is generated, so is_progress_done is false
is_progress_done = false;
}
break;
default:
@ -674,6 +695,41 @@ int ObCDCLobDataMerger::handle_ext_info_log_(
return ret;
}
int ObCDCLobDataMerger::handle_outrow_lob_locator_ext_info_log_(
ObIAllocator &allocator,
const char *buf, uint64_t len, int64_t pos,
ObLobDataGetCtx &lob_data_get_ctx,
ObLobDataOutRowCtxList &lob_data_out_row_ctx_list,
volatile bool &stop_flag)
{
int ret = OB_SUCCESS;
if (len - pos != ObLobLocatorV2::DISK_LOB_OUTROW_FULL_SIZE) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("size is incorrect", KR(ret), K(len), K(pos));
} else {
const ObLobCommon *lob_common = reinterpret_cast<const ObLobCommon*>(buf + pos);
const ObLobData *lob_data = reinterpret_cast<const ObLobData *>(lob_common->buffer_);
const ObLobDataOutRowCtx *lob_data_out_row_ctx = reinterpret_cast<const ObLobDataOutRowCtx *>(lob_data->buffer_);
const ObLobData *pre_lob_data = lob_data_get_ctx.new_lob_data_;
LOG_DEBUG("push lob merge task suscces", KPC(lob_common), KPC(lob_data), KPC(lob_data_out_row_ctx), KPC(pre_lob_data), K(lob_data_get_ctx), K(lob_data_out_row_ctx_list));
// check and set lob_data in ext info log
if (OB_ISNULL(pre_lob_data)) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("pre lob data for ext info log is null", K(ret), K(lob_data_get_ctx), KPC(lob_data), KPC(lob_data_out_row_ctx));
} else if (lob_data_out_row_ctx->op_ != ObLobDataOutRowCtx::OpType::SQL || pre_lob_data->id_ != lob_data->id_ || pre_lob_data->byte_size_ != lob_data->byte_size_) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("lob data is not match", K(ret), KPC(lob_data), KPC(pre_lob_data), KPC(lob_data_out_row_ctx), K(lob_data_get_ctx));
} else if (OB_FALSE_IT(lob_data_get_ctx.new_lob_data_ = lob_data)) {
} else if (OB_FALSE_IT(lob_data_get_ctx.set_type(ObLobDataGetTaskType::FULL_LOB))) {
} else if (OB_FAIL(push_lob_column_(allocator, lob_data_out_row_ctx_list, lob_data_get_ctx, stop_flag))) {
LOG_ERROR("push_lob_column_ fail", KR(ret), KPC(lob_data), KPC(pre_lob_data), KPC(lob_data_out_row_ctx), K(lob_data_get_ctx));
}
}
return ret;
}
int ObCDCLobDataMerger::handle_json_diff_ext_info_log_(
ObIAllocator &allocator,
const char *buf, uint64_t len, int64_t pos,

10
src/logservice/libobcdc/src/ob_cdc_lob_data_merger.h
View File

@ -122,11 +122,19 @@ private:
ObLobDataGetCtx &lob_data_get_ctx,
ObLobDataOutRowCtxList &lob_data_out_row_ctx_list,
const ObString &src_data,
ObString &format_data);
ObString &format_data,
bool &is_progress_done,
volatile bool &stop_flag);
int handle_json_diff_ext_info_log_(
ObIAllocator &allocator,
const char *buf, uint64_t len, int64_t pos,
ObString &format_data);
int handle_outrow_lob_locator_ext_info_log_(
ObIAllocator &allocator,
const char *buf, uint64_t len, int64_t pos,
ObLobDataGetCtx &lob_data_get_ctx,
ObLobDataOutRowCtxList &lob_data_out_row_ctx_list,
volatile bool &stop_flag);
bool is_in_stop_status(volatile bool stop_flag) const { return stop_flag || LobDataMergerThread::is_stoped(); }
// TODO

3
src/logservice/ob_log_base_type.h
View File

@ -124,6 +124,7 @@ enum ObLogBaseType
// for workload repository service
WORKLOAD_REPOSITORY_SERVICE_LOG_BASE_TYPE = 38,
TTL_LOG_BASE_TYPE = 39,
// for table load resource manager
@ -266,6 +267,8 @@ int log_base_type_to_string(const ObLogBaseType log_type,
strncpy(str, "MVIEW_MAINTENANCE_SERVICE_LOG_BASE_TYPE", str_len);
} else if (log_type == SHARE_STORAGE_NRT_THROT_LOG_BASE_TYPE) {
strncpy(str ,"SHARE_STORAGE_NRT_THROT_SERVICE", str_len);
} else if (log_type == TABLE_LOCK_LOG_BASE_TYPE) {
strncpy(str ,"TABLE_LOCK", str_len);
} else if (log_type == SHARED_STORAGE_PRE_WARM_LOG_BASE_TYPE) {
strncpy(str ,"SHARED_STORAGE_PRE_WARM_LOG_BASE_TYPE", str_len);
#ifdef OB_BUILD_SHARED_STORAGE

4
src/observer/ob_inner_sql_rpc_processor.cpp
View File

@ -419,8 +419,8 @@ int ObInnerSqlRpcP::process()
case ObInnerSQLTransmitArg::OPERATION_TYPE_LOCK_ALONE_TABLET:
case ObInnerSQLTransmitArg::OPERATION_TYPE_UNLOCK_ALONE_TABLET:
case ObInnerSQLTransmitArg::OPERATION_TYPE_LOCK_OBJS:
case ObInnerSQLTransmitArg::OPERATION_TYPE_UNLOCK_OBJS: {
case ObInnerSQLTransmitArg::OPERATION_TYPE_UNLOCK_OBJS:
case ObInnerSQLTransmitArg::OPERATION_TYPE_REPLACE_LOCK: {
if (OB_FAIL(ObInnerConnectionLockUtil::process_lock_rpc(transmit_arg, conn))) {
LOG_WARN("process lock rpc failed", K(ret), K(transmit_arg.get_operation_type()));
}

101
src/observer/ob_rpc_processor_simple.cpp
View File

@ -556,7 +556,7 @@ int ObRpcCheckandCancelDDLComplementDagP::process()
int ret = OB_SUCCESS;
if (OB_ISNULL(gctx_.ob_service_)) {
ret = OB_INVALID_ARGUMENT;
LOG_ERROR("invalid argument", K(ret), K(gctx_.ob_service_));
LOG_ERROR("invalid arguments", K(ret), KP(gctx_.ob_service_));
} else {
bool is_dag_exist = true;
ret = gctx_.ob_service_->check_and_cancel_ddl_complement_data_dag(arg_, is_dag_exist);
@ -579,6 +579,56 @@ int ObRpcCheckandCancelDeleteLobMetaRowDagP::process()
return ret;
}
int ObRpcBuildSplitTabletDataStartRequestP::process()
{
int ret = OB_SUCCESS;
if (OB_ISNULL(gctx_.ob_service_)) {
ret = OB_INVALID_ARGUMENT;
LOG_ERROR("invalid arguments", K(ret), KP(gctx_.ob_service_));
} else {
ret = gctx_.ob_service_->build_split_tablet_data_start_request(arg_, result_);
}
return ret;
}
int ObRpcBuildSplitTabletDataFinishRequestP::process()
{
int ret = OB_SUCCESS;
if (OB_ISNULL(gctx_.ob_service_)) {
ret = OB_INVALID_ARGUMENT;
LOG_ERROR("invalid arguments", K(ret), KP(gctx_.ob_service_));
} else {
ret = gctx_.ob_service_->build_split_tablet_data_finish_request(arg_, result_);
}
return ret;
}
int ObRpcFreezeSplitSrcTabletP::process()
{
int ret = OB_SUCCESS;
if (OB_ISNULL(gctx_.ob_service_)) {
ret = OB_INVALID_ARGUMENT;
LOG_ERROR("invalid arguments", K(ret), KP(gctx_.ob_service_));
} else {
const int64_t abs_timeout_us = nullptr == rpc_pkt_ ? 0 : get_receive_timestamp() + rpc_pkt_->get_timeout();
ret = gctx_.ob_service_->freeze_split_src_tablet(arg_, result_, abs_timeout_us);
}
return ret;
}
int ObRpcFetchSplitTabletInfoP::process()
{
int ret = OB_SUCCESS;
if (OB_ISNULL(gctx_.ob_service_)) {
ret = OB_INVALID_ARGUMENT;
LOG_ERROR("invalid arguments", K(ret), KP(gctx_.ob_service_));
} else {
const int64_t abs_timeout_us = nullptr == rpc_pkt_ ? 0 : get_receive_timestamp() + rpc_pkt_->get_timeout();
ret = gctx_.ob_service_->fetch_split_tablet_info(arg_, result_, abs_timeout_us);
}
return ret;
}
int ObRpcFetchSysLSP::process()
{
int ret = OB_SUCCESS;
@ -749,6 +799,42 @@ int ObRpcCheckSchemaVersionElapsedP::process()
return ret;
}
int ObRpcCheckMemtableCntP::process()
{
int ret = OB_SUCCESS;
if (OB_ISNULL(gctx_.ob_service_)) {
ret = OB_INVALID_ARGUMENT;
LOG_ERROR("invalid argument", K(ret), K(gctx_.ob_service_));
} else {
ret = gctx_.ob_service_->check_memtable_cnt(arg_, result_);
}
return ret;
}
int ObRpcCheckMediumCompactionInfoListP::process()
{
int ret = OB_SUCCESS;
if (OB_ISNULL(gctx_.ob_service_)) {
ret = OB_INVALID_ARGUMENT;
LOG_ERROR("invalid argument", K(ret), K(gctx_.ob_service_));
} else {
ret = gctx_.ob_service_->check_medium_compaction_info_list_cnt(arg_, result_);
}
return ret;
}
int ObRpcPrepareTabletSplitTaskRangesP::process()
{
int ret = OB_SUCCESS;
if (OB_ISNULL(gctx_.ob_service_)) {
ret = OB_INVALID_ARGUMENT;
LOG_ERROR("invalid argument", K(ret), K(gctx_.ob_service_));
} else {
ret = gctx_.ob_service_->prepare_tablet_split_task_ranges(arg_, result_);
}
return ret;
}
int ObRpcCheckCtxCreateTimestampElapsedP::process()
{
int ret = OB_SUCCESS;
@ -2316,6 +2402,19 @@ int ObRpcBatchGetTabletBindingP::process()
return ret;
}
int ObRpcBatchGetTabletSplitP::process()
{
int ret = OB_SUCCESS;
if (OB_ISNULL(rpc_pkt_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid rpc pkt", K(ret));
} else {
const int64_t abs_timeout_us = get_send_timestamp() + rpc_pkt_->get_timeout();
ret = ObTabletSplitMdsHelper::batch_get_tablet_split(abs_timeout_us, arg_, result_);
}
return ret;
}
#ifdef OB_BUILD_TDE_SECURITY
int ObDumpTenantCacheMasterKeyP::process()
{

8
src/observer/ob_rpc_processor_simple.h
View File

@ -116,13 +116,21 @@ OB_DEFINE_PROCESSOR_S(Srv, OB_FETCH_SYS_LS, ObRpcFetchSysLSP);
OB_DEFINE_PROCESSOR_S(Srv, OB_BROADCAST_RS_LIST, ObRpcBroadcastRsListP);
OB_DEFINE_PROCESSOR_S(Srv, OB_MINOR_FREEZE, ObRpcMinorFreezeP);
OB_DEFINE_PROCESSOR_S(Srv, OB_CHECK_SCHEMA_VERSION_ELAPSED, ObRpcCheckSchemaVersionElapsedP);
OB_DEFINE_PROCESSOR_S(Srv, OB_CHECK_MEMTABLE_CNT, ObRpcCheckMemtableCntP);
OB_DEFINE_PROCESSOR_S(Srv, OB_CHECK_MEDIUM_INFO_LIST_CNT, ObRpcCheckMediumCompactionInfoListP);
OB_DEFINE_PROCESSOR_S(Srv, OB_DDL_BUILD_SINGLE_REPLICA_REQUEST, ObRpcBuildDDLSingleReplicaRequestP);
OB_DEFINE_PROCESSOR_S(Srv, OB_SPLIT_TABLET_DATA_START_REQUEST, ObRpcBuildSplitTabletDataStartRequestP);
OB_DEFINE_PROCESSOR_S(Srv, OB_SPLIT_TABLET_DATA_FINISH_REQUEST, ObRpcBuildSplitTabletDataFinishRequestP);
OB_DEFINE_PROCESSOR_S(Srv, OB_PREPARE_TABLET_SPLIT_TASK_RANGES, ObRpcPrepareTabletSplitTaskRangesP);
OB_DEFINE_PROCESSOR_S(Srv, OB_FREEZE_SPLIT_SRC_TABLET, ObRpcFreezeSplitSrcTabletP);
OB_DEFINE_PROCESSOR_S(Srv, OB_FETCH_SPLIT_TABLET_INFO, ObRpcFetchSplitTabletInfoP);
OB_DEFINE_PROCESSOR_S(Srv, OB_FETCH_TABLET_AUTOINC_SEQ_CACHE, ObRpcFetchTabletAutoincSeqCacheP);
OB_DEFINE_PROCESSOR_S(Srv, OB_BATCH_GET_TABLET_AUTOINC_SEQ, ObRpcBatchGetTabletAutoincSeqP);
OB_DEFINE_PROCESSOR_S(Srv, OB_BATCH_SET_TABLET_AUTOINC_SEQ, ObRpcBatchSetTabletAutoincSeqP);
OB_DEFINE_PROCESSOR_S(Srv, OB_SET_TABLET_AUTOINC_SEQ, ObRpcSetTabletAutoincSeqP);
OB_DEFINE_PROCESSOR_S(Srv, OB_CLEAR_TABLET_AUTOINC_SEQ_CACHE, ObRpcClearTabletAutoincSeqCacheP);
OB_DEFINE_PROCESSOR_S(Srv, OB_BATCH_GET_TABLET_BINDING, ObRpcBatchGetTabletBindingP);
OB_DEFINE_PROCESSOR_S(Srv, OB_BATCH_GET_TABLET_SPLIT, ObRpcBatchGetTabletSplitP);
OB_DEFINE_PROCESSOR_S(Srv, OB_CHECK_MODIFY_TIME_ELAPSED, ObRpcCheckCtxCreateTimestampElapsedP);
OB_DEFINE_PROCESSOR_S(Srv, OB_UPDATE_BASELINE_SCHEMA_VERSION, ObRpcUpdateBaselineSchemaVersionP);
OB_DEFINE_PROCESSOR_S(Srv, OB_SWITCH_LEADER, ObRpcSwitchLeaderP);

3
src/observer/ob_server.cpp
View File

@ -110,6 +110,7 @@
#include "share/ob_server_blacklist.h"
#include "rootserver/standby/ob_standby_service.h" // ObStandbyService
#include "share/scheduler/ob_dag_warning_history_mgr.h"
#include "share/scheduler/ob_partition_auto_split_helper.h"
#include "share/longops_mgr/ob_longops_mgr.h"
#include "logservice/palf/election/interface/election.h"
#include "share/ob_ddl_sim_point.h"
@ -518,6 +519,8 @@ int ObServer::init(const ObServerOptions &opts, const ObPLogWriterCfg &log_cfg)
LOG_ERROR("set sys task status self addr failed", KR(ret));
} else if (OB_FAIL(ObTableStoreStatMgr::get_instance().init())) {
LOG_ERROR("init table store stat mgr failed", KR(ret));
} else if (OB_FAIL(ObServerAutoSplitScheduler::get_instance().init())) {
LOG_ERROR("init auto split scheduler failed", KR(ret));
} else if (OB_FAIL(ObCompatModeGetter::instance().init(&sql_proxy_))) {
LOG_ERROR("init get compat mode server failed",KR(ret));
} else if (OB_FAIL(table_service_.init())) {

2
src/observer/ob_server_schema_updater.cpp
View File

@ -436,6 +436,7 @@ int ObServerSchemaUpdater::try_reload_schema(
LOG_WARN("fail to set tenant received broadcast version", K(tmp_ret), K(schema_info));
}
DEBUG_SYNC(BEFORE_ADD_REFRESH_SCHEMA_TASK);
const bool did_retry = true;
ObServerSchemaTask refresh_task(ObServerSchemaTask::REFRESH, did_retry, schema_info);
if (OB_FAIL(task_queue_.add(refresh_task))) {
@ -471,6 +472,7 @@ int ObServerSchemaUpdater::async_refresh_schema(
const int64_t schema_version)
{
int ret = OB_SUCCESS;
DEBUG_SYNC(BEFORE_ADD_ASYNC_REFRESH_SCHEMA_TASK);
ObServerSchemaTask refresh_task(ObServerSchemaTask::ASYNC_REFRESH,
tenant_id, schema_version);
if (!inited_) {

306
src/observer/ob_service.cpp
View File

@ -32,6 +32,7 @@
#include "common/ob_tenant_data_version_mgr.h"
#include "share/ob_version.h"
#include "share/ob_ddl_common.h"
#include "share/ob_version.h"
#include "share/inner_table/ob_inner_table_schema.h"
#include "share/deadlock/ob_deadlock_inner_table_service.h"
@ -54,10 +55,14 @@
#include "observer/ob_server_schema_updater.h"
#include "ob_server_event_history_table_operator.h"
#include "share/ob_alive_server_tracer.h"
#include "storage/ddl/ob_tablet_split_task.h"
#include "storage/ddl/ob_tablet_lob_split_task.h"
#include "storage/ddl/ob_complement_data_task.h" // complement data for drop column
#include "storage/ddl/ob_ddl_clog.h"
#include "storage/ddl/ob_delete_lob_meta_row_task.h" // delete lob meta row for drop vec index
#include "storage/ddl/ob_ddl_merge_task.h"
#include "storage/ddl/ob_build_index_task.h"
#include "storage/ddl/ob_ddl_redo_log_writer.h"
#include "storage/tablet/ob_tablet_multi_source_data.h"
#include "storage/tx_storage/ob_tenant_freezer.h"
#include "storage/tx_storage/ob_ls_map.h"
@ -84,6 +89,7 @@
#ifdef OB_BUILD_TDE_SECURITY
#include "share/ob_master_key_getter.h"
#endif
#include "storage/compaction/ob_schedule_dag_func.h"
#include "storage/compaction/ob_tenant_tablet_scheduler.h"
#include "share/ob_cluster_event_history_table_operator.h"//CLUSTER_EVENT_INSTANCE
#include "storage/ddl/ob_tablet_ddl_kv_mgr.h"
@ -1266,7 +1272,10 @@ int ObService::check_schema_version_elapsed(
LOG_WARN("ddl sim failure: check schema version elapsed slow", K(tmp_ret), K(arg));
} else if (OB_TMP_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::OBSERVER_MOD))) {
LOG_WARN("get ls failed", K(tmp_ret), K(i), K(ls_id));
} else if (OB_TMP_FAIL(ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle))) {
} else if (OB_TMP_FAIL(ls_handle.get_ls()->get_tablet(tablet_id,
tablet_handle,
ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US,
ObMDSGetTabletMode::READ_ALL_COMMITED))) {
LOG_WARN("fail to get tablet", K(tmp_ret), K(i), K(ls_id), K(tablet_id));
} else if (OB_TMP_FAIL(tablet_handle.get_obj()->check_schema_version_elapsed(arg.schema_version_,
arg.need_wait_trans_end_,
@ -1286,6 +1295,108 @@ int ObService::check_schema_version_elapsed(
return ret;
}
// 1. minor freeze
// 2. get memtable cnt
int ObService::check_memtable_cnt(
const obrpc::ObCheckMemtableCntArg &arg,
obrpc::ObCheckMemtableCntResult &result)
{
int ret = OB_SUCCESS;
LOG_INFO("receive check memtable cnt request", K(arg));
if (OB_UNLIKELY(!inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_UNLIKELY(!arg.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(arg));
} else {
ObMinorFreezeArg minor_freeze_arg;
minor_freeze_arg.tablet_id_ = arg.tablet_id_;
if (OB_FAIL(minor_freeze_arg.tenant_ids_.push_back(arg.tenant_id_))) {
LOG_WARN("failed to push back tenant id", K(ret));
} else if (OB_FAIL(handle_ls_freeze_req_(minor_freeze_arg))) {
LOG_WARN("failed to handle tablet freeze", K(ret));
} else {
MTL_SWITCH(arg.tenant_id_) {
bool freeze_finished = false;
ObTabletID tablet_id = arg.tablet_id_;
const int64_t expire_renew_time = INT64_MAX;
bool is_cache_hit = false;
ObLSID ls_id = arg.ls_id_;
ObLSService *ls_srv = MTL(ObLSService *);
ObLSHandle ls_handle;
ObLS *ls = nullptr;
ObLSTabletService *ls_tablet_service = nullptr;
ObTabletHandle tablet_handle;
ObTablet *tablet = nullptr;
ObArray<ObTableHandleV2> memtable_handles;
if (OB_FAIL(ls_srv->get_ls(ls_id, ls_handle, ObLSGetMod::OBSERVER_MOD))) {
LOG_WARN("fail to get ls", K(ret), K(ls_id));
} else if (OB_ISNULL(ls = ls_handle.get_ls())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls is null", K(ret), K(ls_id));
} else if (OB_ISNULL(ls_tablet_service = ls->get_tablet_svr())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tablet service should not be null", K(ret), K(ls_id));
} else if (OB_FAIL(ls_tablet_service->get_tablet(tablet_id,
tablet_handle, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_10_S, ObMDSGetTabletMode::READ_ALL_COMMITED))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get tablet handle failed", K(ret), K(tablet_id));
} else if (FALSE_IT(tablet = tablet_handle.get_obj())) {
} else if (OB_FAIL(tablet->get_all_memtables(memtable_handles))) {
LOG_WARN("failed to get_memtable_mgr for get all memtable", K(ret), KPC(tablet));
} else {
result.memtable_cnt_ = memtable_handles.count();
freeze_finished = result.memtable_cnt_ == 0 ? true : false;
if (freeze_finished) {
ObTabletFreezeLog freeze_log;
freeze_log.tablet_id_ = tablet_id;
if (OB_FAIL(storage::ObDDLRedoLogWriter::
write_auto_split_log(ls_id,
ObDDLClogType::DDL_TABLET_FREEZE_LOG,
logservice::ObReplayBarrierType::STRICT_BARRIER,
freeze_log))) {
LOG_WARN("write tablet freeze log failed", K(ret), K(freeze_log));
}
}
}
} // MTL_SWITCH
}
}
LOG_INFO("finish check memtable cnt request", K(ret), K(arg));
return ret;
}
// possible results:
// 1. ret != OB_SUCCESS
// 2. ret == OB_SUCCESS && info_list_cnt_ > 0 && invalid compaction_scn
// 3. ret == OB_SUCCESS && info_list_cnt_ == 0 && valid primary_compaction_scn_
int ObService::check_medium_compaction_info_list_cnt(
const obrpc::ObCheckMediumCompactionInfoListArg &arg,
obrpc::ObCheckMediumCompactionInfoListResult &result)
{
return ObTabletSplitUtil::check_medium_compaction_info_list_cnt(arg, result);
}
int ObService::prepare_tablet_split_task_ranges(
const obrpc::ObPrepareSplitRangesArg &arg,
obrpc::ObPrepareSplitRangesRes &result)
{
int ret = OB_SUCCESS;
result.parallel_datum_rowkey_list_.reset();
if (OB_UNLIKELY(!inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_UNLIKELY(!arg.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(arg));
} else if (OB_FAIL(ObTabletSplitUtil::split_task_ranges(result.rowkey_allocator_, arg.ls_id_,
arg.tablet_id_, arg.user_parallelism_, arg.schema_tablet_size_, result.parallel_datum_rowkey_list_))) {
LOG_WARN("split task ranges failed", K(ret));
}
return ret;
}
int ObService::check_ddl_tablet_merge_status(
const obrpc::ObDDLCheckTabletMergeStatusArg &arg,
obrpc::ObDDLCheckTabletMergeStatusResult &result)
@ -2721,23 +2832,181 @@ int ObService::broadcast_rs_list(const ObRsListArg &arg)
return ret;
}
int ObService::build_split_tablet_data_start_request(const obrpc::ObTabletSplitStartArg &arg, obrpc::ObTabletSplitStartResult &res)
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
if (OB_UNLIKELY(!arg.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arg", K(ret), K(arg));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < arg.split_info_array_.count(); i++) {
const ObTabletSplitArg &each_arg = arg.split_info_array_.at(i);
if (OB_FAIL(ObTabletLobSplitUtil::process_write_split_start_log_request(each_arg))) {
LOG_WARN("process write split start log failed", K(ret), K(tmp_ret), K(arg));
}
if (OB_TMP_FAIL(res.ret_codes_.push_back(ret))) {
LOG_WARN("push back result failed", K(ret), K(tmp_ret));
}
ret = OB_SUCC(ret) ? tmp_ret : ret;
}
}
LOG_INFO("process write split start log finished", K(ret), K(arg));
return ret;
}
int ObService::build_split_tablet_data_finish_request(const obrpc::ObTabletSplitFinishArg &arg, obrpc::ObTabletSplitFinishResult &res)
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
if (OB_UNLIKELY(!arg.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arg", K(ret), K(arg));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < arg.split_info_array_.count(); i++) {
ObTabletSplitFinishResult unused_res;
const ObTabletSplitArg &each_arg = arg.split_info_array_.at(i);
if (OB_FAIL(ObTabletLobSplitUtil::process_tablet_split_request(
each_arg.lob_col_idxs_.count() > 0/*is_lob_tablet*/,
false/*is_start_request*/,
static_cast<const void *>(&each_arg),
static_cast<void *>(&unused_res)))) {
LOG_WARN("process split finish request failed", K(ret), K(arg));
}
if (OB_TMP_FAIL(res.ret_codes_.push_back(ret))) {
LOG_WARN("push back failed", K(ret), K(tmp_ret));
}
ret = OB_SUCC(ret) ? tmp_ret : ret;
}
}
LOG_INFO("process split finish request succ", K(ret), K(arg));
return ret;
}
int ObService::freeze_split_src_tablet(const ObFreezeSplitSrcTabletArg &arg,
ObFreezeSplitSrcTabletRes &res,
const int64_t abs_timeout_us)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("service not inited", K(ret));
} else if (OB_UNLIKELY(!arg.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arg", K(ret), K(arg));
} else {
MTL_SWITCH(arg.tenant_id_) {
ObLSService *ls_service = MTL(ObLSService *);
logservice::ObLogService *log_service = MTL(logservice::ObLogService*);
ObLSHandle ls_handle;
ObLS *ls = nullptr;
ObRole role = INVALID_ROLE;
int64_t proposal_id = -1;
bool has_active_memtable = false;
if (OB_ISNULL(ls_service) || OB_ISNULL(log_service)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected ls_service or log_service", K(ret));
} else if (OB_FAIL(ls_service->get_ls(arg.ls_id_, ls_handle, ObLSGetMod::OBSERVER_MOD))) {
LOG_WARN("get ls failed", K(ret), K(arg));
} else if (OB_ISNULL(ls = ls_handle.get_ls())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid ls", K(ret), K(arg.ls_id_));
} else if (OB_FAIL(ls->get_ls_role(role))) {
LOG_WARN("get role failed", K(ret), K(MTL_ID()), K(arg.ls_id_));
} else if (OB_UNLIKELY(ObRole::LEADER != role)) {
ret = OB_NOT_MASTER;
LOG_WARN("ls not leader", K(ret), K(MTL_ID()), K(arg.ls_id_));
} else if (OB_FAIL(ls->tablet_freeze(checkpoint::INVALID_TRACE_ID, arg.tablet_ids_, true/*is_sync*/, abs_timeout_us,
false/*need_rewrite_meta*/, ObFreezeSourceFlag::TABLET_SPLIT))) {
LOG_WARN("batch tablet freeze failed", K(ret), K(arg));
} else if (OB_FAIL(ls->check_tablet_no_active_memtable(arg.tablet_ids_, has_active_memtable))) {
// safer with this check, non-mandatory
LOG_WARN("check tablet has active memtable failed", K(ret), K(arg));
} else if (has_active_memtable) {
ret = OB_EAGAIN;
LOG_WARN("tablet has active memtable need retry", K(ret), K(arg));
} else if (OB_FAIL(ls->get_log_handler()->get_max_scn(res.data_end_scn_))) {
LOG_WARN("log_handler get_max_scn failed", K(ret), K(arg));
}
}
}
return ret;
}
int ObService::fetch_split_tablet_info(const ObFetchSplitTabletInfoArg &arg,
ObFetchSplitTabletInfoRes &res,
const int64_t abs_timeout_us)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("service not inited", K(ret));
} else if (OB_UNLIKELY(!arg.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arg", K(ret), K(arg));
} else {
MTL_SWITCH(arg.tenant_id_) {
ObLSService *ls_service = MTL(ObLSService *);
ObLSHandle ls_handle;
ObLS *ls = nullptr;
ObRole role = INVALID_ROLE;
if (OB_ISNULL(ls_service)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected ls_service or log_service", K(ret));
} else if (OB_FAIL(ls_service->get_ls(arg.ls_id_, ls_handle, ObLSGetMod::OBSERVER_MOD))) {
LOG_WARN("get ls failed", K(ret), K(arg));
} else if (OB_ISNULL(ls = ls_handle.get_ls())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid ls", K(ret), K(arg.ls_id_));
} else if (OB_FAIL(ls->get_ls_role(role))) {
LOG_WARN("get role failed", K(ret), K(MTL_ID()), K(arg.ls_id_));
} else if (OB_UNLIKELY(ObRole::LEADER != role)) {
ret = OB_NOT_MASTER;
LOG_WARN("ls not leader", K(ret), K(MTL_ID()), K(arg.ls_id_));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < arg.tablet_ids_.count(); i++) {
const ObTabletID &tablet_id = arg.tablet_ids_.at(i);
ObTabletHandle tablet_handle;
ObTabletCreateDeleteMdsUserData user_data;
if (OB_FAIL(ls->get_tablet(tablet_id, tablet_handle))) {
LOG_WARN("failed to get tablet", K(ret), K(tablet_id));
} else if (OB_FAIL(tablet_handle.get_obj()->ObITabletMdsInterface::get_tablet_status(
share::SCN::max_scn(), user_data, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US))) {
LOG_WARN("failed to get tablet status", K(ret), K(arg.ls_id_), K(tablet_id));
} else if (OB_FAIL(res.create_commit_versions_.push_back(user_data.create_commit_version_))) {
LOG_WARN("failed to push back", K(ret));
}
}
}
}
}
return ret;
}
int ObService::build_ddl_single_replica_request(const ObDDLBuildSingleReplicaRequestArg &arg,
ObDDLBuildSingleReplicaRequestResult &res)
{
int ret = OB_SUCCESS;
LOG_INFO("receive build single replica request", K(arg));
ObTenantDagScheduler *dag_scheduler = nullptr;
if (OB_UNLIKELY(!arg.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(ret), K(arg));
} else if (OB_ISNULL(dag_scheduler = MTL(ObTenantDagScheduler *))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("dag scheduler is null", K(ret));
} else {
if (is_complement_data_relying_on_dag(ObDDLType(arg.ddl_type_))) {
if (share::is_tablet_split(ObDDLType(arg.ddl_type_))) {
if (OB_FAIL(ObTabletLobSplitUtil::process_tablet_split_request(
arg.lob_col_idxs_.count() > 0/*is_lob_tablet*/,
true/*is_start_request*/,
static_cast<const void *>(&arg),
static_cast<void *>(&res)))) {
LOG_WARN("process split start request failed", K(ret), K(arg));
}
} else if (is_complement_data_relying_on_dag(ObDDLType(arg.ddl_type_))) {
int saved_ret = OB_SUCCESS;
ObTenantDagScheduler *dag_scheduler = nullptr;
ObComplementDataDag *dag = nullptr;
if (OB_ISNULL(dag_scheduler = MTL(ObTenantDagScheduler *))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("dag scheduler is null", K(ret));
} else if (OB_FAIL(dag_scheduler->alloc_dag(dag))) {
if (OB_FAIL(dag_scheduler->alloc_dag(dag))) {
LOG_WARN("fail to alloc dag", K(ret));
} else if (OB_ISNULL(dag)) {
ret = OB_ERR_UNEXPECTED;
@ -2746,17 +3015,21 @@ int ObService::build_ddl_single_replica_request(const ObDDLBuildSingleReplicaReq
LOG_WARN("fail to init complement data dag", K(ret), K(arg));
} else if (OB_FAIL(dag->create_first_task())) {
LOG_WARN("create first task failed", K(ret));
} else if (OB_FAIL(dag_scheduler->add_dag(dag))) {
} else if (OB_FAIL(add_dag_and_get_progress<ObComplementDataDag>(dag, res.row_inserted_, res.physical_row_count_))) {
saved_ret = ret;
LOG_WARN("add dag failed", K(ret), K(arg));
if (OB_EAGAIN == saved_ret) {
dag_scheduler->get_complement_data_dag_progress(dag, res.row_scanned_, res.row_inserted_);
if (OB_EAGAIN == ret) {
ret = OB_SUCCESS;
} else if (OB_SIZE_OVERFLOW == ret) {
ret = OB_EAGAIN;
} else {
LOG_WARN("add dag and get progress failed", K(ret));
}
} else {
dag = nullptr;
}
if (OB_NOT_NULL(dag)) {
(void) dag->handle_init_failed_ret_code(ret);
// to free dag.
dag_scheduler->free_dag(*dag);
dag = nullptr;
}
@ -2800,11 +3073,11 @@ int ObService::build_ddl_single_replica_request(const ObDDLBuildSingleReplicaReq
dag = nullptr;
}
} else {
ret = OB_NOT_SUPPORTED;
LOG_WARN("not supported ddl type", K(ret), K(arg));
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid ddl type request", K(ret), K(arg));
}
}
LOG_INFO("receive build single replica request", K(ret), K(arg));
return ret;
}
@ -2879,7 +3152,6 @@ int ObService::check_and_cancel_delete_lob_meta_row_dag(const obrpc::ObDDLBuildS
LOG_WARN("cancel dag failed", K(ret));
}
if (OB_NOT_NULL(dag)) {
(void) dag->handle_init_failed_ret_code(ret);
dag_scheduler->free_dag(*dag);
dag = nullptr;
}

21
src/observer/ob_service.h
View File

@ -32,6 +32,8 @@ namespace share
{
class ObIAliveServerTracer;
struct ObTabletReplicaChecksumItem;
class ObTenantDagScheduler;
class ObIDag;
}
namespace storage
{
@ -124,6 +126,11 @@ public:
// ObRpcSwitchSchemaP @RS DDL
int switch_schema(const obrpc::ObSwitchSchemaArg &arg, obrpc::ObSwitchSchemaResult &result);
int calc_column_checksum_request(const obrpc::ObCalcColumnChecksumRequestArg &arg, obrpc::ObCalcColumnChecksumRequestRes &res);
int build_split_tablet_data_start_request(const obrpc::ObTabletSplitStartArg &arg, obrpc::ObTabletSplitStartResult &res);
int build_split_tablet_data_finish_request(const obrpc::ObTabletSplitFinishArg &arg, obrpc::ObTabletSplitFinishResult &res);
int freeze_split_src_tablet(const obrpc::ObFreezeSplitSrcTabletArg &arg, obrpc::ObFreezeSplitSrcTabletRes &res, const int64_t abs_timeout_us);
int fetch_split_tablet_info(const obrpc::ObFetchSplitTabletInfoArg &arg, obrpc::ObFetchSplitTabletInfoRes &res, const int64_t abs_timeout_us);
int build_ddl_single_replica_request(const obrpc::ObDDLBuildSingleReplicaRequestArg &arg);
int build_ddl_single_replica_request(const obrpc::ObDDLBuildSingleReplicaRequestArg &arg, obrpc::ObDDLBuildSingleReplicaRequestResult &res);
int check_and_cancel_ddl_complement_data_dag(const obrpc::ObDDLBuildSingleReplicaRequestArg &arg, bool &is_dag_exist);
int check_and_cancel_delete_lob_meta_row_dag(const obrpc::ObDDLBuildSingleReplicaRequestArg &arg, bool &is_dag_exist);
@ -177,6 +184,19 @@ public:
const obrpc::ObCheckSchemaVersionElapsedArg &arg,
obrpc::ObCheckSchemaVersionElapsedResult &result);
// ObRpcGetChecksumCalSnapshotP
// ObRpcCheckMemtableCntP
int check_memtable_cnt(
const obrpc::ObCheckMemtableCntArg &arg,
obrpc::ObCheckMemtableCntResult &result);
// ObRpcCheckMediumCompactionInfoListP
int check_medium_compaction_info_list_cnt(
const obrpc::ObCheckMediumCompactionInfoListArg &arg,
obrpc::ObCheckMediumCompactionInfoListResult &result);
int prepare_tablet_split_task_ranges(
const obrpc::ObPrepareSplitRangesArg &arg,
obrpc::ObPrepareSplitRangesRes &result);
int check_modify_time_elapsed(
const obrpc::ObCheckModifyTimeElapsedArg &arg,
obrpc::ObCheckModifyTimeElapsedResult &result);
@ -184,7 +204,6 @@ public:
int check_ddl_tablet_merge_status(
const obrpc::ObDDLCheckTabletMergeStatusArg &arg,
obrpc::ObDDLCheckTabletMergeStatusResult &result);
////////////////////////////////////////////////////////////////
// ObRpcBatchSwitchRsLeaderP @RS leader coordinator & admin
int batch_switch_rs_leader(const ObAddr &arg);

1
src/observer/ob_srv_xlator_partition.cpp
View File

@ -255,6 +255,7 @@ void oceanbase::observer::init_srv_xlator_for_others(ObSrvRpcXlator *xlator) {
RPC_PROCESSOR(ObOutTransLockTableP, gctx_);
RPC_PROCESSOR(ObOutTransUnlockTableP, gctx_);
RPC_PROCESSOR(ObBatchLockTaskP, gctx_);
RPC_PROCESSOR(ObBatchReplaceLockTaskP, gctx_);
RPC_PROCESSOR(ObHighPriorityBatchLockTaskP, gctx_);
RPC_PROCESSOR(ObAdminRemoveLockP);
RPC_PROCESSOR(ObAdminUpdateLockP);

4
src/observer/ob_srv_xlator_rootserver.cpp
View File

@ -275,7 +275,9 @@ void oceanbase::observer::init_srv_xlator_for_rootserver(ObSrvRpcXlator *xlator)
RPC_PROCESSOR(rootserver::ObRpcCloneTenantP, *gctx_.root_service_);
// auto part ddl
RPC_PROCESSOR(rootserver::ObRpcCleanSplittedTabletP, *gctx_.root_service_);
RPC_PROCESSOR(rootserver::ObAutoSplitTabletTaskP, *gctx_.root_service_);
RPC_PROCESSOR(rootserver::ObSplitGlobalIndexTabletTaskP, *gctx_.root_service_);
RPC_PROCESSOR(rootserver::ObRpcFlushOptStatMonitoringInfoP, *gctx_.root_service_);
RPC_PROCESSOR(rootserver::ObRpcCreateDirectoryP, *gctx_.root_service_);

8
src/observer/ob_srv_xlator_storage.cpp
View File

@ -58,6 +58,8 @@ void oceanbase::observer::init_srv_xlator_for_storage(ObSrvRpcXlator *xlator) {
//RPC_PROCESSOR(ObRpcRemoveReplicaP, gctx_);
RPC_PROCESSOR(ObRpcMinorFreezeP, gctx_);
RPC_PROCESSOR(ObRpcCheckSchemaVersionElapsedP, gctx_);
RPC_PROCESSOR(ObRpcCheckMemtableCntP, gctx_);
RPC_PROCESSOR(ObRpcCheckMediumCompactionInfoListP, gctx_);
RPC_PROCESSOR(ObRpcCheckCtxCreateTimestampElapsedP, gctx_);
RPC_PROCESSOR(ObRpcUpdateBaselineSchemaVersionP, gctx_);
RPC_PROCESSOR(ObRpcSwitchLeaderP, gctx_);
@ -101,11 +103,16 @@ void oceanbase::observer::init_srv_xlator_for_storage(ObSrvRpcXlator *xlator) {
RPC_PROCESSOR(ObPreProcessServerP, gctx_);
RPC_PROCESSOR(ObRpcBroadcastRsListP, gctx_);
RPC_PROCESSOR(ObRpcBuildDDLSingleReplicaRequestP, gctx_);
RPC_PROCESSOR(ObRpcBuildSplitTabletDataStartRequestP, gctx_);
RPC_PROCESSOR(ObRpcBuildSplitTabletDataFinishRequestP, gctx_);
RPC_PROCESSOR(ObRpcFreezeSplitSrcTabletP, gctx_);
RPC_PROCESSOR(ObRpcFetchSplitTabletInfoP, gctx_);
RPC_PROCESSOR(ObRpcFetchTabletAutoincSeqCacheP, gctx_);
RPC_PROCESSOR(ObRpcBatchGetTabletAutoincSeqP, gctx_);
RPC_PROCESSOR(ObRpcBatchSetTabletAutoincSeqP, gctx_);
RPC_PROCESSOR(ObRpcClearTabletAutoincSeqCacheP, gctx_);
RPC_PROCESSOR(ObRpcBatchGetTabletBindingP, gctx_);
RPC_PROCESSOR(ObRpcBatchGetTabletSplitP, gctx_);
RPC_PROCESSOR(ObRpcRemoteWriteDDLRedoLogP, gctx_);
RPC_PROCESSOR(ObRpcRemoteWriteDDLCommitLogP, gctx_);
RPC_PROCESSOR(ObRpcSetTabletAutoincSeqP, gctx_);
@ -138,6 +145,7 @@ void oceanbase::observer::init_srv_xlator_for_storage(ObSrvRpcXlator *xlator) {
RPC_PROCESSOR(ObRpcTabletMajorFreezeP, gctx_);
RPC_PROCESSOR(ObRpcDetectSessionAliveP, gctx_);
RPC_PROCESSOR(ObCancelGatherStatsP, gctx_);
RPC_PROCESSOR(ObRpcPrepareTabletSplitTaskRangesP, gctx_);
RPC_PROCESSOR(ObRpcCheckStorageOperationStatusP, gctx_);
#ifdef OB_BUILD_SHARED_STORAGE
RPC_PROCESSOR(ObRpcRemoteWriteDDLFinishLogP, gctx_);

4
src/observer/omt/ob_multi_tenant.cpp
View File

@ -172,6 +172,7 @@
#include "storage/tenant_snapshot/ob_tenant_snapshot_service.h"
#include "share/index_usage/ob_index_usage_info_mgr.h"
#include "rootserver/mview/ob_mview_maintenance_service.h"
#include "storage/restore/ob_tenant_restore_info_mgr.h"
#include "share/io/ob_storage_io_usage_reporter.h"
#include "share/resource_limit_calculator/ob_resource_limit_calculator.h"
#include "storage/checkpoint/ob_checkpoint_diagnose.h"
@ -179,6 +180,7 @@
#include "lib/roaringbitmap/ob_rb_memory_mgr.h"
#include "storage/tmp_file/ob_tmp_file_manager.h" // ObTenantTmpFileManager
#include "storage/restore/ob_tenant_restore_info_mgr.h"
#include "share/scheduler/ob_partition_auto_split_helper.h"
#ifdef OB_BUILD_AUDIT_SECURITY
#include "sql/audit/ob_audit_logger.h"
#include "sql/audit/ob_audit_log_mgr.h"
@ -618,6 +620,7 @@ int ObMultiTenant::init(ObAddr myaddr,
MTL_BIND2(mtl_new_default, ObIndexUsageInfoMgr::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, storage::ObTabletMemtableMgrPool::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
MTL_BIND2(mtl_new_default, rootserver::ObMViewMaintenanceService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, storage::ObTenantRestoreInfoMgr::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObStorageIOUsageRepoter::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
#ifdef OB_BUILD_SHARED_STORAGE
MTL_BIND2(mtl_new_default, ObPublicBlockGCService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
@ -638,6 +641,7 @@ int ObMultiTenant::init(ObAddr myaddr,
MTL_BIND2(mtl_new_default, table::ObTableClientInfoMgr::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, observer::ObTableQueryASyncMgr::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObPluginVectorIndexService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObAutoSplitTaskCache::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
}
if (OB_SUCC(ret)) {

2
src/observer/table_load/ob_table_load_instance.cpp
View File

@ -374,7 +374,7 @@ int ObTableLoadInstance::lock_table_in_tx()
while (OB_SUCC(ret) && !lock_succeed) {
if (OB_FAIL(execute_ctx_->check_status())) {
LOG_WARN("failed to check status", KR(ret));
} else if (OB_FAIL(table_lock_service->lock_table(*tx_desc, stmt_ctx_.tx_param_, lock_table_arg))) {
} else if (OB_FAIL(table_lock_service->lock(*tx_desc, stmt_ctx_.tx_param_, lock_table_arg))) {
if (OB_EAGAIN == ret) {
ob_usleep(sleep_time);
ret = OB_SUCCESS;

2
src/observer/table_load/ob_table_load_redef_table.cpp
View File

@ -118,6 +118,7 @@ int ObTableLoadRedefTable::finish(const ObTableLoadRedefTableFinishArg &arg,
finish_redef_table_arg.task_id_ = arg.task_id_;
finish_redef_table_arg.tenant_id_ = arg.tenant_id_;
ObAddr rs_addr;
ObDDLBuildSingleReplicaResponseArg build_single_replica_response_arg;
build_single_replica_response_arg.task_id_ = arg.task_id_;
build_single_replica_response_arg.tenant_id_ = arg.tenant_id_;
@ -132,6 +133,7 @@ int ObTableLoadRedefTable::finish(const ObTableLoadRedefTableFinishArg &arg,
build_single_replica_response_arg.snapshot_version_ = 1;
build_single_replica_response_arg.execution_id_ = 1;
build_single_replica_response_arg.ret_code_ = ret;
build_single_replica_response_arg.server_addr_ = GCTX.self_addr();
if (OB_FAIL(ObDDLServerClient::finish_redef_table(
finish_redef_table_arg, build_single_replica_response_arg, session_info))) {
LOG_WARN("failed to finish redef table", KR(ret), K(finish_redef_table_arg));

1
src/observer/virtual_table/ob_all_virtual_tablet_encrypt_info.h
View File

@ -20,6 +20,7 @@
#include "share/ob_scanner.h"
#include "share/ob_virtual_table_scanner_iterator.h"
#include "share/rc/ob_tenant_base.h"
#include "storage/blocksstable/index_block/ob_index_block_macro_iterator.h"
#include "storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.h"
#include "storage/blocksstable/ob_sstable_meta.h"
#include "storage/meta_mem/ob_tablet_handle.h"

1
src/observer/virtual_table/ob_all_virtual_tablet_sstable_macro_info.h
View File

@ -18,6 +18,7 @@
#include "share/ob_scanner.h"
#include "share/ob_virtual_table_scanner_iterator.h"
#include "share/rc/ob_tenant_base.h"
#include "storage/blocksstable/index_block/ob_index_block_macro_iterator.h"
#include "storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.h"
#include "storage/blocksstable/ob_sstable_meta.h"
#include "storage/ob_i_table.h"

1
src/pl/CMakeLists.txt
View File

@ -84,6 +84,7 @@ ob_set_subtarget(ob_pl sys_package
sys_package/ob_dbms_sql.cpp
sys_package/ob_dbms_xplan.cpp
sys_package/ob_dbms_user_define_rule.cpp
sys_package/ob_dbms_space.cpp
sys_package/ob_dbms_mview_mysql.cpp
sys_package/ob_dbms_mview_stats_mysql.cpp
sys_package/ob_dbms_workload_repository.cpp

5
src/pl/ob_pl_interface_pragma.h
View File

@ -69,6 +69,7 @@
#include "pl/sys_package/ob_dbms_xplan.h"
#include "pl/sys_package/ob_pl_dbms_resource_manager.h"
#include "pl/sys_package/ob_dbms_session.h"
#include "pl/sys_package/ob_dbms_space.h"
#include "pl/sys_package/ob_dbms_workload_repository.h"
#include "pl/sys_package/ob_dbms_mview_mysql.h"
#include "pl/sys_package/ob_dbms_mview_stats_mysql.h"
@ -620,6 +621,10 @@
INTERFACE_DEF(INTERFACE_DBMS_SESSION_RESET_PACKAGE, "RESET_PACKAGE", (ObDBMSSession::reset_package))
// end of dbms_session
// start of dbms_space
INTERFACE_DEF(INTERFACE_DBMS_SPACE_CREATE_INDEX_COST, "CREATE_INDEX_COST", (ObDbmsSpace::create_index_cost))
// end of dbms_space
#ifdef OB_BUILD_ORACLE_PL
// start of dbms_rls
INTERFACE_DEF(INTERFACE_RLS_ADD_GROUPED_POLICY, "RLS_ADD_GROUPED_POLICY", (ObDBMSRls::add_grouped_policy))

4
src/pl/ob_pl_package_manager.cpp
View File

@ -475,6 +475,7 @@ static const ObSysPackageFile oracle_syspack_file_list[] = {
{"json_object_t", "json_object_type.sql", "json_object_type_body.sql"},
{"dbms_mview", "dbms_mview.sql", "dbms_mview_body.sql"},
{"dbms_mview_stats", "dbms_mview_stats.sql", "dbms_mview_stats_body.sql"},
{"dbms_space", "dbms_space.sql", "dbms_space_body.sql"},
{"json_array_t", "json_array_type.sql", "json_array_type_body.sql"},
{"xmlsequence", "xml_sequence_type.sql", nullptr},
{"utl_recomp", "utl_recomp.sql", "utl_recomp_body.sql"},
@ -504,7 +505,8 @@ static const ObSysPackageFile mysql_syspack_file_list[] = {
{"dbms_ob_limit_calculator", "dbms_ob_limit_calculator_mysql.sql", "dbms_ob_limit_calculator_body_mysql.sql"},
{"dbms_external_table", "dbms_external_table_mysql.sql", "dbms_external_table_body_mysql.sql"},
{"external_table_alert_log", "external_table_alert_log.sql", nullptr},
{"dbms_vector", "dbms_vector_mysql.sql", "dbms_vector_body_mysql.sql"}
{"dbms_vector", "dbms_vector_mysql.sql", "dbms_vector_body_mysql.sql"},
{"dbms_space", "dbms_space_mysql.sql", "dbms_space_body_mysql.sql"}
};
// for now! we only have one special system package "__DBMS_UPGRADE"

2
src/pl/ob_pl_resolver.cpp
View File

@ -14189,7 +14189,7 @@ int ObPLResolver::resolve_routine(ObObjAccessIdent &access_ident,
ObObjAccessIdx access_idx;
if (expr_params.count() != 2 && expr_params.count() != 3) {
ret = OB_ERR_WRONG_TYPE_FOR_VAR;
LOG_WARN("PLS-00306: wrong number or types of arguments in call to 'RAISE_APPLICATION_ERROR'", K(ret));;
LOG_WARN("PLS-00306: wrong number or types of arguments in call to 'RAISE_APPLICATION_ERROR'", K(ret));
LOG_USER_ERROR(OB_ERR_WRONG_TYPE_FOR_VAR, routine_name.length(), routine_name.ptr());
} else {
ObPLDataType invalid_pl_data_type;

1052
src/pl/sys_package/ob_dbms_space.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

220
src/pl/sys_package/ob_dbms_space.h Normal file
View File

@ -0,0 +1,220 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_SRC_PL_SYS_PACKAGE_OB_DBMS_SPACE_H_
#define OCEANBASE_SRC_PL_SYS_PACKAGE_OB_DBMS_SPACE_H_
#include "sql/engine/ob_exec_context.h"
#include "pl/ob_pl_type.h"
#include "lib/ob_define.h"
#include "share/stat/ob_opt_table_stat.h"
#include "share/stat/ob_opt_column_stat.h"
#include "share/stat/ob_opt_column_stat_cache.h"
#include "sql/resolver/ddl/ob_create_index_stmt.h"
namespace oceanbase
{
using namespace sql;
namespace pl
{
class ObDbmsSpace
{
public:
struct IndexCostInfo final {
public:
IndexCostInfo(): tenant_id_(OB_SYS_TENANT_ID),
table_id_(common::OB_INVALID_ID),
part_ids_(),
column_ids_(),
compression_ratio_(1.0),
default_index_len_(0),
svr_addr_(),
tablet_ids_(),
table_tenant_id_(OB_INVALID_TENANT_ID) {}
~IndexCostInfo() = default;
TO_STRING_KV(K_(tenant_id), K_(table_id), K_(part_ids), K_(column_ids),
K_(compression_ratio), K_(default_index_len), K_(svr_addr),
K_(tablet_ids), K_(table_tenant_id));
public:
uint64_t tenant_id_;
uint64_t table_id_;
ObSEArray<int64_t, 1> part_ids_;
ObSEArray<uint64_t, 4> column_ids_;
double compression_ratio_;
uint64_t default_index_len_;
ObSEArray<ObAddr, 1> svr_addr_;
ObSEArray<ObTabletID, 4> tablet_ids_;
uint64_t table_tenant_id_;
};
struct OptStats final {
public:
OptStats(): table_stats_(), column_stats_() {};
~OptStats() = default;
TO_STRING_KV(K_(table_stats), K_(column_stats));
public:
ObSEArray<ObOptTableStat, 1> table_stats_;
ObSEArray<ObOptColumnStatHandle, 4> column_stats_;
};
struct TabletInfo final {
public:
TabletInfo(): tablet_id_(OB_INVALID_ID), partition_id_(OB_INVALID_PARTITION_ID), row_len_(0), row_count_(0), compression_ratio_(0) {}
TabletInfo(ObTabletID tablet_id, ObObjectID partition_id): tablet_id_(tablet_id), partition_id_(partition_id), row_len_(0), row_count_(0), compression_ratio_(0) {}
~TabletInfo() = default;
int assign(const TabletInfo &other) {
int ret = OB_SUCCESS;
tablet_id_ = other.tablet_id_;
partition_id_ = other.partition_id_;
row_len_ = other.row_len_;
row_count_ = other.row_count_;
compression_ratio_ = other.compression_ratio_;
return ret;
}
TO_STRING_KV(K_(tablet_id), K_(partition_id), K_(row_len), K_(row_count), K_(compression_ratio));
public:
ObTabletID tablet_id_;
ObObjectID partition_id_;
double row_len_;
uint64_t row_count_;
double compression_ratio_;
};
typedef ObSEArray<TabletInfo, 1> TabletInfoList;
public:
static int create_index_cost(sql::ObExecContext &ctx,
sql::ParamStore &params,
common::ObObj &result);
// interface for auto_split
/* in param:
* info tenant_id_; // default set to OB_SYS_TENANT_ID
* info table_id_; // data table's table id;
* info.part_ids_; // the part id(data table) we need to calc.
* info.column_ids_; // index columns' column id (in data table schema).
* out param:
* the table_size in info.part_ids_'s order
*/
static int estimate_index_table_size(ObMySQLProxy *sql_proxy,
const ObTableSchema *table_schema,
IndexCostInfo &info,
ObIArray<uint64_t> &table_size);
// interface to get each tablet_size in the table
/*
* in param:
* table_schema -- the target table we need to calc size.
* out param:
* each tablet's size.
*/
static int get_each_tablet_size(ObMySQLProxy *sql_proxy,
const ObTableSchema *table_schema,
ObIArray<std::pair<ObTabletID, uint64_t>> &tablet_size);
private:
static int parse_ddl_sql(ObExecContext &ctx,
const ObString &ddl_sql,
ObCreateIndexStmt *&stmt);
static int extract_info_from_stmt(ObExecContext &ctx,
ObCreateIndexStmt *stmt,
IndexCostInfo &info);
static int get_index_column_ids(const share::schema::ObTableSchema *table_schema,
const obrpc::ObCreateIndexArg &arg,
IndexCostInfo &info);
static int get_optimizer_stats(const IndexCostInfo &info,
OptStats &opt_stats);
static int calc_index_size(OptStats &opt_stats,
IndexCostInfo &info,
ObObjParam &actual_size,
ObObjParam &alloc_size);
static int get_compressed_ratio(ObExecContext &ctx,
IndexCostInfo &info);
static int inner_get_compressed_ratio(ObMySQLProxy *sql_proxy,
IndexCostInfo &info);
static int extract_total_compression_ratio(const sqlclient::ObMySQLResult *result,
double &compression_ratio);
static int inner_calc_index_size(const ObOptTableStat &table_stat,
const ObIArray<ObOptColumnStatHandle> &column_stats,
const IndexCostInfo &info,
uint64_t &actual_size,
uint64_t &alloc_size);
static int estimate_index_table_size_by_opt_stats(ObMySQLProxy *sql_proxy,
const OptStats &opt_stats,
IndexCostInfo &info,
ObIArray<uint64_t> &table_size);
static int estimate_index_table_size_default(ObMySQLProxy *sql_proxy,
const ObTableSchema *table_schema,
IndexCostInfo &info,
ObIArray<uint64_t> &table_size);
static int inner_calc_index_size_by_default(IndexCostInfo &info,
TabletInfoList &tablet_infos,
ObIArray<uint64_t> &table_size);
static int fill_tablet_infos(const ObTableSchema *table_schema,
TabletInfoList &tablet_infos);
static int extract_tablet_size(const sqlclient::ObMySQLResult *result,
TabletInfoList &tablet_infos);
static int extract_tablet_size(const sqlclient::ObMySQLResult *result,
ObIArray<std::pair<ObTabletID, uint64_t>> &tablet_size);
static int get_each_tablet_size(ObMySQLProxy *sql_proxy,
TabletInfoList &tablet_infos,
IndexCostInfo &info);
static const TabletInfo* get_tablet_info_by_tablet_id(const TabletInfoList &tablet_infos,
const ObTabletID tablet_id);
static const TabletInfo* get_tablet_info_by_part_id(const TabletInfoList &tablet_infos,
const ObObjectID partition_id);
static int set_tablet_info_by_tablet_id(const ObTabletID tablet_id,
const double row_len,
const uint64_t row_count,
const double compression_ratio,
TabletInfoList &tablet_infos);
static int get_default_index_column_len(const ObTableSchema *table_schema,
const TabletInfoList &table_infos,
IndexCostInfo &info);
static int check_stats_valid(const OptStats &opt_stats, bool &is_valid);
static int get_svr_info_from_schema(const ObTableSchema *table_schema,
ObIArray<ObAddr> &addr_list,
ObIArray<ObTabletID> &tablet_list,
uint64_t &tenant_id);
static int generate_part_key_str(ObSqlString &target_str,
const ObIArray<ObAddr> &addr_list);
static int generate_tablet_predicate_str(ObSqlString &target_str,
const ObIArray<ObTabletID> &tablet_list);
};
}
}
#endif

2
src/rootserver/CMakeLists.txt
View File

@ -101,6 +101,7 @@ ob_set_subtarget(ob_rootserver common
ob_lob_meta_builder.cpp
ob_ls_recovery_stat_handler.cpp
ob_shrink_expand_resource_pool_checker.cpp
ob_split_partition_helper.cpp
ob_transfer_partition_command.cpp
ob_partition_exchange.cpp
ob_service_name_command.cpp
@ -131,6 +132,7 @@ ob_set_subtarget(ob_rootserver ddl_task
ddl_task/ob_fts_index_build_task.cpp
ddl_task/ob_modify_autoinc_task.cpp
ddl_task/ob_table_redefinition_task.cpp
ddl_task/ob_partition_split_task.cpp
ddl_task/ob_recover_restore_table_task.cpp
ddl_task/ob_ddl_tablet_scheduler.cpp
ddl_task/ob_rebuild_index_task.cpp

101
src/rootserver/backup/ob_backup_data_set_task_mgr.cpp
View File

@ -36,7 +36,6 @@
#include "share/ob_tenant_info_proxy.h"
#include "observer/ob_inner_sql_connection.h"
#include "share/backup/ob_backup_server_mgr.h"
#include "share/backup/ob_backup_tablet_reorganize_helper.h"
#include "rootserver/backup/ob_backup_table_list_mgr.h"
using namespace oceanbase;
@ -1189,6 +1188,10 @@ int ObBackupSetTaskMgr::change_turn_(
ObIArray<storage::ObBackupDataTabletToLSInfo> &tablets_to_ls)
{
int ret = OB_SUCCESS;
ROOTSERVICE_EVENT_INSTANCE.sync_add_event("rs_backup", "rs_before_change_turn",
"tenant_id", set_task_attr_.tenant_id_,
"task_id", set_task_attr_.task_id_);
DEBUG_SYNC(RS_CHANGE_TURN_DEBUG_SYNC);
ObTimeoutCtx ctx;
const int64_t DEFAULT_TIMEOUT = 60_s;
const int64_t INNER_SQL_TIMEOUT = GCONF.internal_sql_execute_timeout;
@ -1558,10 +1561,7 @@ int ObBackupSetTaskMgr::do_get_change_turn_tablets_(
ObHashMap<ObLSID, ObArray<ObTabletReorganizeInfo>> tablet_reorganize_ls_map;
ObHashSet<ObLSID> ls_id_set;
ObHashSet<ObLSID> new_ls_id_set;
ObLSID transfer_ls_id;
ObLSID split_ls_id;
ObLSID final_ls_id;
int64_t transfer_seq = 0;
const int64_t OB_BACKUP_MAX_LS_BUCKET = 1024;
if (OB_ISNULL(job_attr_)) {
ret = OB_ERR_UNEXPECTED;
@ -1584,35 +1584,8 @@ int ObBackupSetTaskMgr::do_get_change_turn_tablets_(
for (auto iter = skipped_tablets.begin(); OB_SUCC(ret) && iter != skipped_tablets.end(); ++iter) {
const ObBackupSkipTabletAttr &skip_tablet = iter->first;
descendent_list.reset();
if (OB_FAIL(ObBackupTabletToLSOperator::get_ls_of_tablet(*sql_proxy_, job_attr_->tenant_id_, skip_tablet.tablet_id_, transfer_ls_id, transfer_seq))) {
if (OB_ENTRY_NOT_EXIST == ret) {
ret = OB_SUCCESS;
bool has_reorganized = false;
ObArray<ObTabletReorganizeInfo> *tablet_reorganize_array = NULL;
if (OB_FAIL(ObBackupTabletReorganizeHelper::check_tablet_has_reorganized(
*sql_proxy_, job_attr_->tenant_id_, skip_tablet.tablet_id_, split_ls_id, has_reorganized))) {
LOG_WARN("failed to check tablet has reorganized", K(ret), K(skip_tablet));
} else if (!has_reorganized) {
LOG_WARN("[DATA_BACKUP]deleted tablet", K(skip_tablet));
} else if (OB_ISNULL(tablet_reorganize_array = tablet_reorganize_ls_map.get(split_ls_id))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("do not exist", K(ret), K(split_ls_id));
} else if (OB_FAIL(ObBackupTabletReorganizeHelper::get_leaf_children_from_history(
job_attr_->tenant_id_, *tablet_reorganize_array, skip_tablet.tablet_id_, descendent_list))) {
LOG_WARN("failed to get lead children from history", K(ret));
} else {
final_ls_id = split_ls_id;
}
} else {
LOG_WARN("[DATA_BACKUP]failed to get tablet", K(ret), K(skip_tablet));
}
} else if (OB_FAIL(descendent_list.push_back(skip_tablet.tablet_id_))) {
LOG_WARN("failed to push back", K(ret));
} else {
final_ls_id = transfer_ls_id;
}
if (OB_FAIL(ret)) {
// do nothing
if (OB_FAIL(decide_tablet_final_ls_(skip_tablet, tablet_reorganize_ls_map, descendent_list, final_ls_id))) {
LOG_WARN("failed to decide tablet final ls", K(ret), K(skip_tablet), K(descendent_list));
} else if (!final_ls_id.is_valid()) {
// do nothing
} else {
@ -1670,6 +1643,68 @@ int ObBackupSetTaskMgr::do_get_change_turn_tablets_(
return ret;
}
int ObBackupSetTaskMgr::decide_tablet_final_ls_(const share::ObBackupSkipTabletAttr &skip_tablet,
common::hash::ObHashMap<share::ObLSID, common::ObArray<ObTabletReorganizeInfo>> &tablet_reorganize_ls_map,
common::ObArray<common::ObTabletID> &descendent_list,
share::ObLSID &final_ls_id)
{
int ret = OB_SUCCESS;
final_ls_id.reset();
ObLSID transfer_ls_id;
ObLSID split_ls_id;
int64_t transfer_seq = 0;
const int64_t OB_BACKUP_MAX_LS_BUCKET = 1024;
if (!skip_tablet.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get invalid args", K(ret), K(skip_tablet));
} else if (OB_ISNULL(sql_proxy_) || OB_ISNULL(job_attr_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sql proxy or job attr should not be null", K(ret), KP_(sql_proxy), KP_(job_attr));
} else if (OB_FAIL(ObBackupTabletToLSOperator::get_ls_of_tablet(*sql_proxy_, job_attr_->tenant_id_, skip_tablet.tablet_id_, transfer_ls_id, transfer_seq))) {
if (OB_ENTRY_NOT_EXIST != ret) {
LOG_WARN("[DATA_BACKUP]failed to get tablet", K(ret), K(skip_tablet));
} else {
ret = OB_SUCCESS;
bool has_reorganized = false;
ObArray<ObTabletReorganizeInfo> *tablet_reorganize_array = NULL;
if (OB_FAIL(ObBackupTabletReorganizeHelper::check_tablet_has_reorganized(
*sql_proxy_, job_attr_->tenant_id_, skip_tablet.tablet_id_, split_ls_id, has_reorganized))) {
LOG_WARN("failed to check tablet has reorganized", K(ret), K(skip_tablet));
} else if (!has_reorganized) {
LOG_INFO("[DATA_BACKUP]deleted tablet", K(skip_tablet));
} else if (OB_ISNULL(tablet_reorganize_array = tablet_reorganize_ls_map.get(split_ls_id))) {
ObHashSet<ObLSID> split_ls_id_set;
if (OB_FAIL(split_ls_id_set.create(OB_BACKUP_MAX_LS_BUCKET))) {
LOG_WARN("[DATA_BACKUP]failed to create set", K(ret));
} else if (OB_FAIL(split_ls_id_set.set_refactored(split_ls_id))) {
LOG_WARN("failed to set ls id", K(ret), K(split_ls_id));
} else if (OB_FAIL(ObBackupTabletReorganizeHelper::get_ls_to_tablet_reorganize_info_map(
*sql_proxy_, job_attr_->tenant_id_, split_ls_id_set, tablet_reorganize_ls_map))) {
LOG_WARN("failed to get tablet reorganize hash map", K(ret), KPC_(job_attr));
} else if (OB_ISNULL(tablet_reorganize_array = tablet_reorganize_ls_map.get(split_ls_id))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("split ls id has no tablet reoragnize array", K(ret), K(split_ls_id));
} else {
LOG_INFO("get ls to tablet reorganize info map", K(split_ls_id));
}
}
if (!has_reorganized) {
// do nothing
} else if (FAILEDx(ObBackupTabletReorganizeHelper::get_leaf_children_from_history(
job_attr_->tenant_id_, *tablet_reorganize_array, skip_tablet.tablet_id_, descendent_list))) {
LOG_WARN("failed to get lead children from history", K(ret));
} else {
final_ls_id = split_ls_id;
}
}
} else if (OB_FAIL(descendent_list.push_back(skip_tablet.tablet_id_))) {
LOG_WARN("failed to push back", K(ret));
} else {
final_ls_id = transfer_ls_id;
}
return ret;
}
int ObBackupSetTaskMgr::deduplicate_array_(const common::ObIArray<common::ObTabletID> &tablet_ids,
common::ObIArray<common::ObTabletID> &deduplicated_ids)
{

5
src/rootserver/backup/ob_backup_data_set_task_mgr.h
View File

@ -14,6 +14,7 @@
#define OCEANBASE_SHARE_OB_BACKUP_DATA_SET_TASK_MGR_H_
#include "ob_backup_data_scheduler.h"
#include "share/backup/ob_backup_tablet_reorganize_helper.h"
namespace oceanbase
{
@ -99,6 +100,10 @@ private:
int do_get_change_turn_tablets_(const ObIArray<share::ObBackupLSTaskAttr> &ls_tasks,
const common::hash::ObHashSet<ObBackupSkipTabletAttr> &skip_tablets,
ObIArray<storage::ObBackupDataTabletToLSInfo> &tablet_to_ls);
int decide_tablet_final_ls_(const share::ObBackupSkipTabletAttr &skip_tablet,
common::hash::ObHashMap<share::ObLSID, common::ObArray<ObTabletReorganizeInfo>> &tablet_reorganize_ls_map,
common::ObArray<common::ObTabletID> &descendent_list,
share::ObLSID &final_ls_id);
int deduplicate_array_(const common::ObIArray<common::ObTabletID> &tablet_ids,
common::ObIArray<common::ObTabletID> &deduplicated_ids);
int construct_cur_ls_set_(const ObIArray<share::ObBackupLSTaskAttr> &ls_tasks,

48
src/rootserver/ddl_task/ob_column_redefinition_task.cpp
View File

@ -22,6 +22,7 @@
#include "rootserver/ob_root_service.h"
#include "rootserver/ddl_task/ob_ddl_redefinition_task.h"
#include "storage/tablelock/ob_table_lock_service.h"
#include "storage/ob_partition_pre_split.h"
using namespace oceanbase::lib;
using namespace oceanbase::common;
@ -152,6 +153,7 @@ int ObColumnRedefinitionTask::init(const ObDDLTaskRecord &task_record)
// update sstable complement status for all leaders
int ObColumnRedefinitionTask::update_complete_sstable_job_status(const common::ObTabletID &tablet_id,
const ObAddr &addr,
const int64_t snapshot_version,
const int64_t execution_id,
const int ret_code,
@ -167,14 +169,16 @@ int ObColumnRedefinitionTask::update_complete_sstable_job_status(const common::O
// by pass, may be network delay
} else if (snapshot_version != snapshot_version_) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("snapshot version not match", K(ret), K(snapshot_version), K(snapshot_version_));
LOG_WARN("snapshot version not match", K(ret), K(addr), K(snapshot_version), K(snapshot_version_));
} else if (execution_id < execution_id_) {
LOG_INFO("receive a mismatch execution result, ignore", K(ret_code), K(execution_id), K(execution_id_));
} else if (OB_FAIL(replica_builder_.set_partition_task_status(tablet_id,
ret_code,
addition_info.row_scanned_,
addition_info.row_inserted_))) {
LOG_WARN("fail to set partition task status", K(ret));
LOG_INFO("receive a mismatch execution result, ignore", K(addr), K(ret_code), K(execution_id), K(execution_id_));
} else if (OB_FAIL(replica_builder_.update_build_progress(tablet_id,
addr,
ret_code,
addition_info.row_scanned_,
addition_info.row_inserted_,
addition_info.physical_row_count_))) {
LOG_WARN("fail to set update replica build progress", K(ret), K(addr));
}
return ret;
}
@ -227,10 +231,11 @@ int ObColumnRedefinitionTask::copy_table_indexes()
}
LOG_INFO("indexes schema are already built", K(index_ids));
} else {
// if there is no indexes in new tables, we need to rebuild indexes in new table
int64_t rpc_timeout = 0;
int64_t all_tablet_count = 0;
if (OB_FAIL(get_orig_all_index_tablet_count(schema_guard, all_tablet_count))) {
if (OB_FAIL(generate_rebuild_index_arg_list(tenant_id_, object_id_, schema_guard, alter_table_arg_))) { // for pre split index
LOG_WARN("fail to generate rebuild index arg list", K(ret), K(tenant_id_), K(object_id_));
} else if (OB_FAIL(get_orig_all_index_tablet_count(schema_guard, all_tablet_count))) {
LOG_WARN("get all tablet count failed", K(ret));
} else if (OB_FAIL(ObDDLUtil::get_ddl_rpc_timeout(all_tablet_count, rpc_timeout))) {
LOG_WARN("get ddl rpc timeout failed", K(ret));
@ -721,17 +726,32 @@ int ObColumnRedefinitionTask::collect_longops_stat(ObLongopsValue &value)
break;
}
case ObDDLTaskStatus::REDEFINITION: {
int64_t row_scanned = 0;
int64_t row_inserted = 0;
if (OB_FAIL(replica_builder_.get_progress(row_scanned, row_inserted))) {
int64_t physical_row_count_ = 0;
double percent = 0.0;
bool initializing = false;
{
TCRLockGuard guard(lock_);
initializing = !is_sstable_complete_task_submitted_;
}
if (initializing) {
if (OB_FAIL(databuff_printf(stat_info_.message_,
MAX_LONG_OPS_MESSAGE_LENGTH,
pos,
"STATUS: REPLICA BUILD, PARALLELISM: %ld, INITIALIZING",
ObDDLUtil::get_real_parallelism(parallelism_, false/*is mv refresh*/)))) {
LOG_WARN("failed to print", K(ret));
}
} else if (OB_FAIL(replica_builder_.get_progress(row_inserted, physical_row_count_, percent))) {
LOG_WARN("failed to gather redefinition stats", K(ret));
} else if (OB_FAIL(databuff_printf(stat_info_.message_,
MAX_LONG_OPS_MESSAGE_LENGTH,
pos,
"STATUS: REPLICA BUILD, PARALLELISM: %ld, ROW_SCANNED: %ld, ROW_INSERTED: %ld",
"STATUS: REPLICA BUILD, PARALLELISM: %ld, ESTIMATED_TOTAL_ROWS: %ld, ROW_PROCESSED: %ld, PROGRESS: %0.2lf%%",
ObDDLUtil::get_real_parallelism(parallelism_, false/*is mv refresh*/),
row_scanned,
row_inserted))) {
physical_row_count_,
row_inserted,
percent))) {
LOG_WARN("failed to print", K(ret));
}
break;

1
src/rootserver/ddl_task/ob_column_redefinition_task.h
View File

@ -46,6 +46,7 @@ public:
virtual int process() override;
virtual int update_complete_sstable_job_status(
const common::ObTabletID &tablet_id,
const ObAddr &addr,
const int64_t snapshot_version,
const int64_t execution_id,
const int ret_code,

6
src/rootserver/ddl_task/ob_constraint_task.cpp
View File

@ -57,6 +57,7 @@ int ObCheckConstraintValidationTask::process()
const ObDatabaseSchema *database_schema = nullptr;
int tmp_ret = OB_SUCCESS;
ObTabletID unused_tablet_id;
ObAddr unused_addr;
ObDDLTaskKey task_key(tenant_id_, target_object_id_, schema_version_);
if (OB_FAIL(ObMultiVersionSchemaService::get_instance().get_tenant_schema_guard(tenant_id_, schema_guard))) {
LOG_WARN("get tenant schema guard failed", K(ret), K(tenant_id_));
@ -160,7 +161,7 @@ int ObCheckConstraintValidationTask::process()
}
}
ObDDLTaskInfo info;
if (OB_SUCCESS != (tmp_ret = root_service->get_ddl_scheduler().on_sstable_complement_job_reply(unused_tablet_id, task_key, 1L/*unused snapshot version*/, 1L/*unused execution id*/, ret, info))) {
if (OB_SUCCESS != (tmp_ret = root_service->get_ddl_scheduler().on_sstable_complement_job_reply(unused_tablet_id, unused_addr, task_key, 1L/*unused snapshot version*/, 1L/*unused execution id*/, ret, info))) {
LOG_WARN("fail to finish check constraint task", K(ret), K(tmp_ret));
}
char table_id_buffer[256];
@ -215,13 +216,14 @@ int ObForeignKeyConstraintValidationTask::process()
LOG_WARN("ddl sim failure", K(ret), K(tenant_id_), K(task_id_));
} else {
ObTabletID unused_tablet_id;
ObAddr unused_addr;
ObDDLTaskKey task_key(tenant_id_, foregin_key_id_, schema_version_);
ObDDLTaskInfo info;
int tmp_ret = OB_SUCCESS;
if (OB_FAIL(check_fk_by_send_sql())) {
LOG_WARN("failed to check fk", K(ret));
}
if (OB_SUCCESS != (tmp_ret = root_service->get_ddl_scheduler().on_sstable_complement_job_reply(unused_tablet_id, task_key, 1L/*unused snapshot version*/, 1L/*unused execution id*/, ret, info))) {
if (OB_SUCCESS != (tmp_ret = root_service->get_ddl_scheduler().on_sstable_complement_job_reply(unused_tablet_id, unused_addr, task_key, 1L/*unused snapshot version*/, 1L/*unused execution id*/, ret, info))) {
LOG_WARN("fail to finish check constraint task", K(ret));
}
LOG_INFO("execute check foreign key task finish", K(ret), "ddl_event_info", ObDDLEventInfo(), K(task_key), K(data_table_id_), K(foregin_key_id_));

2
src/rootserver/ddl_task/ob_constraint_task.h
View File

@ -108,7 +108,6 @@ public:
virtual int serialize_params_to_message(char *buf, const int64_t buf_size, int64_t &pos) const override;
virtual int deserialize_params_from_message(const uint64_t tenant_id, const char *buf, const int64_t buf_size, int64_t &pos) override;
virtual int64_t get_serialize_param_size() const override;
virtual int cleanup_impl() override;
private:
int hold_snapshot(const int64_t snapshot_version);
int release_snapshot(const int64_t snapshot_version);
@ -116,6 +115,7 @@ private:
int validate_constraint_valid();
int fail();
int succ();
virtual int cleanup_impl() override;
int send_check_constraint_request();
int send_fk_constraint_request();
int set_foreign_key_constraint_validated();

130
src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp
View File

@ -26,6 +26,7 @@
#include "storage/tablelock/ob_table_lock_rpc_client.h"
#include "share/scn.h"
#include "pl/sys_package/ob_dbms_stats.h"
#include "storage/ob_partition_pre_split.h"
using namespace oceanbase::lib;
using namespace oceanbase::common;
@ -94,6 +95,7 @@ int ObDDLRedefinitionSSTableBuildTask::process()
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
ObTabletID unused_tablet_id;
ObAddr unused_addr;
ObTraceIdGuard trace_id_guard(trace_id_);
ObSqlString sql_string;
ObSchemaGetterGuard schema_guard;
@ -216,7 +218,7 @@ int ObDDLRedefinitionSSTableBuildTask::process()
}
}
}
if (OB_SUCCESS != (tmp_ret = root_service_->get_ddl_scheduler().on_sstable_complement_job_reply(unused_tablet_id, task_key, snapshot_version_, execution_id_, ret, info))) {
if (OB_SUCCESS != (tmp_ret = root_service_->get_ddl_scheduler().on_sstable_complement_job_reply(unused_tablet_id, unused_addr, task_key, snapshot_version_, execution_id_, ret, info))) {
LOG_WARN("fail to finish sstable complement", K(ret), "ddl_event_info", ObDDLEventInfo());
}
add_event_info(ret, "ddl redefinition sstable build task finish");
@ -545,31 +547,48 @@ int ObDDLRedefinitionTask::send_build_single_replica_request()
ret = OB_NOT_INIT;
LOG_WARN("ObColumnRedefinitionTask has not been inited", K(ret));
} else {
ObDDLSingleReplicaExecutorParam param;
param.tenant_id_ = tenant_id_;
param.dest_tenant_id_ = dst_tenant_id_;
param.type_ = task_type_;
param.source_table_id_ = object_id_;
param.dest_table_id_ = target_object_id_;
param.schema_version_ = schema_version_;
param.dest_schema_version_ = dst_schema_version_;
param.snapshot_version_ = snapshot_version_;
param.task_id_ = task_id_;
param.parallelism_ = std::max(alter_table_arg_.parallelism_, 1L);
param.execution_id_ = execution_id_;
param.data_format_version_ = data_format_version_;
param.consumer_group_id_ = alter_table_arg_.consumer_group_id_;
if (OB_FAIL(ObDDLUtil::get_tablets(tenant_id_, object_id_, param.source_tablet_ids_))) {
LOG_WARN("fail to get tablets", K(ret), K(tenant_id_), K(object_id_));
} else if (OB_FAIL(ObDDLUtil::get_tablets(dst_tenant_id_, target_object_id_, param.dest_tablet_ids_))) {
LOG_WARN("fail to get tablets", K(ret), K(tenant_id_), K(target_object_id_));
} else if (OB_FAIL(replica_builder_.build(param))) {
LOG_WARN("fail to send build single replica", K(ret));
} else {
LOG_INFO("start to build single replica", K(target_object_id_));
is_sstable_complete_task_submitted_ = true;
sstable_complete_request_time_ = ObTimeUtility::current_time();
}
SMART_VAR(ObDDLReplicaBuildExecutorParam, param) {
param.tenant_id_ = tenant_id_;
param.dest_tenant_id_ = dst_tenant_id_;
param.ddl_type_ = task_type_;
param.snapshot_version_ = snapshot_version_;
param.task_id_ = task_id_;
param.parallelism_ = std::max(alter_table_arg_.parallelism_, 1L);
param.execution_id_ = execution_id_;
param.data_format_version_ = data_format_version_;
param.consumer_group_id_ = alter_table_arg_.consumer_group_id_;
if (OB_FAIL(ObDDLUtil::get_tablets(tenant_id_, object_id_, param.source_tablet_ids_))) {
LOG_WARN("fail to get tablets", K(ret), K(tenant_id_), K(object_id_));
} else if (OB_FAIL(ObDDLUtil::get_tablets(dst_tenant_id_, target_object_id_, param.dest_tablet_ids_))) {
LOG_WARN("fail to get tablets", K(ret), K(dst_tenant_id_), K(target_object_id_));
}
const int64_t src_tablet_cnt = param.source_tablet_ids_.count();
for (int64_t i = 0; OB_SUCC(ret) && i < src_tablet_cnt; ++i) {
if (OB_FAIL(param.source_table_ids_.push_back(object_id_))) {
LOG_WARN("failed to push back src table id", K(ret));
} else if (OB_FAIL(param.source_schema_versions_.push_back(schema_version_))) {
LOG_WARN("failed to push back src schema version", K(ret));
}
}
const int64_t dest_tablet_cnt = param.dest_tablet_ids_.count();
for (int64_t i = 0; OB_SUCC(ret) && i < dest_tablet_cnt; ++i) {
if (OB_FAIL(param.dest_table_ids_.push_back(target_object_id_))) {
LOG_WARN("failed to push back dest table id", K(ret));
} else if (OB_FAIL(param.dest_schema_versions_.push_back(dst_schema_version_))) {
LOG_WARN("failed to push back dest schema version", K(ret));
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(replica_builder_.build(param))) {
LOG_WARN("fail to send build single replica", K(ret));
} else {
TCWLockGuard guard(lock_);
is_sstable_complete_task_submitted_ = true;
sstable_complete_request_time_ = ObTimeUtility::current_time();
LOG_INFO("start to build single replica", K(target_object_id_));
}
}
} // smart_var
}
return ret;
}
@ -586,6 +605,7 @@ int ObDDLRedefinitionTask::check_build_single_replica(bool &is_end)
LOG_WARN("fail to check build end", K(ret));
} else if (!is_end) {
if (sstable_complete_request_time_ + ObDDLUtil::calc_inner_sql_execute_timeout() < ObTimeUtility::current_time()) { // timeout, retry
TCWLockGuard guard(lock_);
is_sstable_complete_task_submitted_ = false;
sstable_complete_request_time_ = 0;
}
@ -1203,7 +1223,11 @@ int ObDDLRedefinitionTask::finish()
ObSArray<uint64_t> objs;
int64_t rpc_timeout = 0;
int64_t all_orig_index_tablet_count = 0;
alter_table_arg_.ddl_task_type_ = share::CLEANUP_GARBAGE_TASK;
if (ObDDLType::DDL_PARTITION_SPLIT_RECOVERY_TABLE_REDEFINITION == task_type_) {
alter_table_arg_.ddl_task_type_ = share::PARTITION_SPLIT_RECOVERY_CLEANUP_GARBAGE_TASK;
} else {
alter_table_arg_.ddl_task_type_ = share::CLEANUP_GARBAGE_TASK;
}
alter_table_arg_.table_id_ = object_id_;
alter_table_arg_.hidden_table_id_ = target_object_id_;
alter_table_arg_.task_id_ = task_id_;
@ -2293,6 +2317,55 @@ int ObDDLRedefinitionTask::check_need_check_table_empty(bool &need_check_table_e
return ret;
}
int ObDDLRedefinitionTask::generate_rebuild_index_arg_list(
const int64_t tenant_id,
const int64_t table_id,
ObSchemaGetterGuard &schema_guard,
obrpc::ObAlterTableArg &alter_table_arg)
{
int ret = OB_SUCCESS;
const ObTableSchema *table_schema = nullptr;
ObRootService *root_service = GCTX.root_service_;
if (tenant_id == OB_INVALID_TENANT_ID || table_id == OB_INVALID_ID) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(tenant_id), K(table_id));
} else if (OB_ISNULL(root_service)) {
ret = OB_ERR_SYS;
LOG_WARN("error sys, root service must not be nullptr", K(ret));
} else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, table_id, table_schema))) {
LOG_WARN("fail to get table schema", K(ret), K(tenant_id), K(table_id));
} else if (OB_ISNULL(table_schema)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("error unexpected, table schema must not be nullptr", K(ret), K(table_id));
} else {
const common::ObIArray<ObAuxTableMetaInfo> &index_infos = table_schema->get_simple_index_infos();
if (index_infos.count() > 0) {
AlterTableSchema &alter_table_schema = const_cast<AlterTableSchema &>(alter_table_arg.alter_table_schema_);
const ObString database_name = alter_table_schema.get_origin_database_name();
ObPartitionPreSplit pre_split(root_service->get_ddl_service());
for (int64_t i = 0; OB_SUCC(ret) && i < index_infos.count(); ++i) {
const ObTableSchema *index_schema = nullptr;
ObTableSchema new_index_schema;
const int64_t index_id = index_infos.at(i).table_id_;
if (OB_FAIL(schema_guard.get_table_schema(tenant_id, index_id, index_schema))) {
LOG_WARN("fail to get index table schema", K(ret), K(tenant_id), K(index_id));
} else if (OB_ISNULL(index_schema)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected schema nullptr", K(ret), KP(index_schema));
} else if (!index_schema->is_global_index_table()) { // skip
} else if (OB_FAIL(new_index_schema.assign(*index_schema))) {
LOG_WARN("fail to assign schema", K(ret));
} else if (OB_FAIL(pre_split.do_table_pre_split_if_need(database_name, ObDDLType::DDL_CREATE_INDEX, false, *table_schema, *index_schema, new_index_schema))) {
LOG_WARN("fail to pre split index partition", K(ret), K(index_id));
} else if (OB_FAIL(alter_table_arg.rebuild_index_arg_list_.push_back(new_index_schema))) {
LOG_WARN("fail to push back index schema", K(ret));
}
}
}
}
return ret;
}
int ObDDLRedefinitionTask::get_child_task_ids(char *buf, int64_t len)
{
int ret = OB_SUCCESS;
@ -2666,7 +2739,8 @@ int ObDDLRedefinitionTask::reap_old_replica_build_task(bool &need_exec_new_inner
LOG_WARN("failed to check and wait old complement task", K(ret));
}
} else if (!need_exec_new_inner_sql) {
if (OB_FAIL(update_complete_sstable_job_status(unused_tablet_id, snapshot_version_, old_execution_id, old_ret_code, unused_addition_info))) {
ObAddr unused_addr;
if (OB_FAIL(update_complete_sstable_job_status(unused_tablet_id, unused_addr, snapshot_version_, old_execution_id, old_ret_code, unused_addition_info))) {
LOG_WARN("failed to wait and complete old task finished!", K(ret));
}
}

11
src/rootserver/ddl_task/ob_ddl_redefinition_task.h
View File

@ -140,6 +140,7 @@ public:
virtual int process() = 0;
virtual int update_complete_sstable_job_status(
const common::ObTabletID &tablet_id,
const ObAddr &addr,
const int64_t snapshot_version,
const int64_t execution_id,
const int ret_code,
@ -148,7 +149,6 @@ public:
const uint64_t child_task_key,
const int ret_code) override;
int notify_update_autoinc_finish(const uint64_t autoinc_val, const int ret_code);
virtual int cleanup_impl() override;
int reap_old_replica_build_task(bool &need_exec_new_inner_sql);
INHERIT_TO_STRING_KV("ObDDLTask", ObDDLTask,
K(wait_trans_ctx_), K(sync_tablet_autoinc_seq_ctx_), K(build_replica_request_time_),
@ -258,7 +258,14 @@ protected:
int get_child_task_ids(char *buf, int64_t len);
int get_estimated_timeout(const share::schema::ObTableSchema *dst_table_schema, int64_t &estimated_timeout);
int get_orig_all_index_tablet_count(ObSchemaGetterGuard &schema_guard, int64_t &all_tablet_count);
int generate_rebuild_index_arg_list(const int64_t tenant_id,
const int64_t table_id,
ObSchemaGetterGuard &schema_guard,
obrpc::ObAlterTableArg &alter_table_arg);
int64_t get_build_replica_request_time();
private:
virtual int cleanup_impl() override;
protected:
static const int64_t MAP_BUCKET_NUM = 1024;
struct DependTaskStatus final
@ -291,7 +298,7 @@ protected:
int64_t check_table_empty_job_time_;
bool is_sstable_complete_task_submitted_;
int64_t sstable_complete_request_time_;
ObDDLSingleReplicaExecutor replica_builder_;
ObDDLReplicaBuildExecutor replica_builder_;
common::hash::ObHashMap<common::ObTabletID, common::ObTabletID> check_dag_exit_tablets_map_; // for dag complement data ddl only.
int64_t check_dag_exit_retry_cnt_;
};

2
src/rootserver/ddl_task/ob_ddl_retry_task.h
View File

@ -47,7 +47,6 @@ public:
common::ObMySQLTransaction &trans,
const uint64_t tenant_id,
const int64_t task_id);
virtual int cleanup_impl() override;
private:
int check_health();
int prepare(const share::ObDDLTaskStatus next_task_status);
@ -55,6 +54,7 @@ private:
int wait_alter_table(const share::ObDDLTaskStatus next_task_status);
int succ();
int fail();
virtual int cleanup_impl() override;
int deep_copy_ddl_arg(common::ObIAllocator &allocator, const share::ObDDLType &ddl_type, const obrpc::ObDDLArg *source_arg);
int init_compat_mode(const share::ObDDLType &ddl_type, const obrpc::ObDDLArg *source_arg);
int get_forward_user_message(const obrpc::ObRpcResultCode &rcode);

242
src/rootserver/ddl_task/ob_ddl_scheduler.cpp
View File

@ -40,6 +40,7 @@
#include "share/scheduler/ob_sys_task_stat.h"
#include "share/ob_ddl_sim_point.h"
#include "share/restore/ob_import_util.h"
#include "share/scheduler/ob_partition_auto_split_helper.h"
namespace oceanbase
{
@ -359,10 +360,12 @@ int ObDDLTaskQueue::update_task_ret_code(const ObDDLTaskID &task_id, const int r
LOG_WARN("ddl_task is null", K(ret));
} else {
const ObTabletID unused_tablet_id;
const ObAddr unused_addr;
const int64_t unused_snapshot_version = 0;
const int64_t unused_execution_id = 0;
const ObDDLTaskInfo unused_task_info;
ret = table_redefinition_task->update_complete_sstable_job_status(unused_tablet_id, unused_snapshot_version,
ret = table_redefinition_task->update_complete_sstable_job_status(
unused_tablet_id, unused_addr, unused_snapshot_version,
unused_execution_id, ret_code, unused_task_info);
}
return ret;
@ -665,10 +668,12 @@ int ObUpdateSSTableCompleteStatusCallback::update_redef_task_info(ObTableRedefin
{
int ret = OB_SUCCESS;
const ObTabletID unused_tablet_id;
const ObAddr unused_addr;
const int64_t unused_snapshot_version = 0;
const int64_t unused_execution_id = 0;
const ObDDLTaskInfo unused_task_info;
ret = redef_task.update_complete_sstable_job_status(unused_tablet_id, unused_snapshot_version,
ret = redef_task.update_complete_sstable_job_status(
unused_tablet_id, unused_addr, unused_snapshot_version,
unused_execution_id, ret_code_, unused_task_info);
return ret;
}
@ -976,6 +981,7 @@ int ObDDLScheduler::create_ddl_task(const ObCreateDDLTaskParam &param,
const obrpc::ObAlterTableArg *alter_table_arg = nullptr;
const obrpc::ObCreateIndexArg *create_index_arg = nullptr;
const obrpc::ObDropIndexArg *drop_index_arg = nullptr;
const obrpc::ObPartitionSplitArg *partition_split_arg = nullptr;
const obrpc::ObRebuildIndexArg *rebuild_index_arg = nullptr;
const obrpc::ObMViewCompleteRefreshArg *mview_complete_refresh_arg = nullptr;
ObRootService *root_service = GCTX.root_service_;
@ -1115,6 +1121,7 @@ int ObDDLScheduler::create_ddl_task(const ObCreateDDLTaskParam &param,
case DDL_ALTER_COLUMN_GROUP:
case DDL_MVIEW_COMPLETE_REFRESH:
case DDL_MODIFY_AUTO_INCREMENT_WITH_REDEFINITION:
case DDL_PARTITION_SPLIT_RECOVERY_TABLE_REDEFINITION:
if (OB_FAIL(create_table_redefinition_task(proxy,
param.type_,
param.src_table_schema_,
@ -1243,6 +1250,22 @@ int ObDDLScheduler::create_ddl_task(const ObCreateDDLTaskParam &param,
LOG_WARN("fail to create modify autoinc task", K(ret));
}
break;
case DDL_AUTO_SPLIT_BY_RANGE:
case DDL_AUTO_SPLIT_NON_RANGE:
case DDL_MANUAL_SPLIT_BY_RANGE:
case DDL_MANUAL_SPLIT_NON_RANGE:
partition_split_arg = static_cast<const obrpc::ObPartitionSplitArg *>(param.ddl_arg_);
if (OB_FAIL(create_partition_split_task(proxy,
param.src_table_schema_,
param.parallelism_,
param.parent_task_id_,
param.task_id_,
partition_split_arg,
*param.allocator_,
task_record))) {
LOG_WARN("fail to create partition split task", K(ret));
}
break;
case DDL_DROP_DATABASE:
case DDL_DROP_TABLE:
case DDL_TRUNCATE_TABLE:
@ -1361,6 +1384,109 @@ int ObDDLScheduler::prepare_alter_table_arg(const ObPrepareAlterTableArgParam &p
return ret;
}
int ObDDLScheduler::cache_auto_split_task(const obrpc::ObAutoSplitTabletBatchArg &arg,
obrpc::ObAutoSplitTabletBatchRes &res)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!arg.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(arg));
} else {
ObRsAutoSplitScheduler &split_task_scheduler = ObRsAutoSplitScheduler::get_instance();
ObArray<ObAutoSplitTask> task_array;
ObAutoSplitTask task;
const ObSArray<obrpc::ObAutoSplitTabletArg> &single_arg_array = arg.args_;
res.suggested_next_valid_time_ = OB_INVALID_TIMESTAMP;
res.rets_.reuse();
for (int64_t i = 0; OB_SUCC(ret) && i < single_arg_array.size(); ++i) {
const obrpc::ObAutoSplitTabletArg &single_arg = single_arg_array.at(i);
task.reset();
task.auto_split_tablet_size_ = single_arg.auto_split_tablet_size_;
task.ls_id_ = single_arg.ls_id_;
task.tablet_id_ = single_arg.tablet_id_;
task.tenant_id_ = single_arg.tenant_id_;
task.used_disk_space_ = single_arg.used_disk_space_;
task.retry_times_ = 0;
if (OB_FAIL(task_array.push_back(task))) {
LOG_WARN("fail to push back task", K(ret) ,K(task), K(task_array));
} else if (OB_FAIL(res.rets_.push_back(OB_SUCCESS))) {
LOG_WARN("fail to push back ret", K(ret));
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(split_task_scheduler.push_tasks(task_array))) {
LOG_WARN("fail to push tasks into auto_split_task_tree_", K(ret), K(task_array));
} else {
int64_t cur_time = ObTimeUtility::current_time();
bool is_busy = split_task_scheduler.is_busy();
res.suggested_next_valid_time_ = cur_time + (is_busy ? ObServerAutoSplitScheduler::OB_SERVER_DELAYED_TIME : 0);
}
}
}
return ret;
}
int ObDDLScheduler::schedule_auto_split_task()
{
int ret = OB_SUCCESS;
ObRsAutoSplitScheduler &split_task_scheduler = ObRsAutoSplitScheduler::get_instance();
ObArray<ObAutoSplitTask> task_array;
if (OB_FAIL(split_task_scheduler.pop_tasks(task_array))) {
LOG_WARN("fail to pop tasks from auto_split_task_tree");
} else if (task_array.count() == 0) {
//do nothing
} else {
ObAutoSplitArgBuilder split_helper;
ObArray<ObAutoSplitTask> failed_task;
obrpc::ObAlterTableRes unused_res;
common::ObMalloc allocator(common::ObMemAttr(OB_SERVER_TENANT_ID, "split_sched"));
for (int64_t i = 0; OB_SUCC(ret) && i < task_array.count(); ++i) {
int tmp_ret = OB_SUCCESS;
unused_res.reset();
ObAutoSplitTask &task = task_array.at(i);
void *buf = nullptr;
obrpc::ObAlterTableArg *single_arg = nullptr;
bool is_ls_migrating = false;
if (OB_FAIL(ObRsAutoSplitScheduler::check_ls_migrating(task.tenant_id_, task.tablet_id_, is_ls_migrating))) {
LOG_WARN("check ls migrating failed", K(ret), K(task));
} else if (is_ls_migrating) {
LOG_TRACE("ls migrating, delay auto split", K(task));
} else if (OB_ISNULL(buf = allocator.alloc(sizeof(obrpc::ObAlterTableArg)))) {
//ignore ret
tmp_ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("allocate memory failed", K(tmp_ret), K(task));
} else if (FALSE_IT(single_arg = new (buf) obrpc::ObAlterTableArg())) {
} else if (OB_TMP_FAIL(split_helper.build_arg(task.tenant_id_, task.ls_id_, task.tablet_id_,
task.auto_split_tablet_size_, task.used_disk_space_, *single_arg))) {
LOG_WARN("fail to build arg", K(tmp_ret), K(task));
} else if (!single_arg->is_auto_split_partition()) {
//do nothing
} else if (single_arg->alter_table_schema_.is_global_index_table()
&& OB_TMP_FAIL(root_service_->get_common_rpc_proxy().to(GCTX.self_addr()).timeout(GCONF._ob_ddl_timeout).split_global_index_tablet(*single_arg))) {
LOG_WARN("split global index failed", K(tmp_ret), K(single_arg));
} else if (!single_arg->alter_table_schema_.is_global_index_table()
&& OB_TMP_FAIL(root_service_->get_common_rpc_proxy().to(GCTX.self_addr()).timeout(GCONF._ob_ddl_timeout).alter_table(*single_arg, unused_res))) {
LOG_WARN("alter table failed", K(tmp_ret), K(single_arg), K(unused_res));
}
if (OB_TMP_FAIL(tmp_ret) && split_task_scheduler.can_retry(task, tmp_ret)) {
failed_task.reuse();
task.increment_retry_times();
if (OB_TMP_FAIL(failed_task.push_back(task))) {
LOG_WARN("fail to push back into failed task", K(tmp_ret), K(task));
} else if (OB_TMP_FAIL(split_task_scheduler.push_tasks(failed_task))) {
LOG_WARN("fail to push tasks", K(tmp_ret), K(failed_task));
}
}
if (OB_NOT_NULL(single_arg)) {
single_arg->~ObAlterTableArg();
allocator.free(single_arg);
single_arg = nullptr;
}
}
}
return ret;
}
int ObDDLScheduler::get_task_record(const ObDDLTaskID &task_id,
ObISQLClient &trans,
ObDDLTaskRecord &task_record,
@ -2021,7 +2147,7 @@ int ObDDLScheduler::create_constraint_task(
} else if (OB_UNLIKELY(nullptr == table_schema || OB_INVALID_ID == constraint_id || schema_version <= 0
|| nullptr == arg || !arg->is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(table_schema), K(constraint_id), K(schema_version), K(arg));
LOG_WARN("invalid argument", K(ret), KPC(table_schema), K(constraint_id), K(schema_version), K(arg));
} else if (OB_FAIL(ObDDLTask::fetch_new_task_id(root_service_->get_sql_proxy(), table_schema->get_tenant_id(), task_id))) {
LOG_WARN("fetch new task id failed", K(ret));
} else if (OB_FAIL(constraint_task.init(task_id, table_schema, constraint_id, ddl_type, schema_version, *arg, consumer_group_id, sub_task_trace_id, parent_task_id))) {
@ -2246,6 +2372,44 @@ int ObDDLScheduler::create_ddl_retry_task(
return ret;
}
int ObDDLScheduler::create_partition_split_task(
common::ObISQLClient &proxy,
const share::schema::ObTableSchema *table_schema,
const int64_t parallelism,
const int64_t parent_task_id,
const int64_t task_id,
const obrpc::ObPartitionSplitArg *partition_split_arg,
ObIAllocator &allocator,
ObDDLTaskRecord &task_record)
{
int ret = OB_SUCCESS;
SMART_VAR(ObPartitionSplitTask, split_task) {
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_ISNULL(partition_split_arg) || OB_ISNULL(table_schema) || OB_UNLIKELY(0 == task_id)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(task_id), KPC(partition_split_arg), KPC(table_schema));
} else if (OB_FAIL(split_task.init(table_schema->get_tenant_id(),
task_id,
table_schema->get_table_id(),
table_schema->get_schema_version(),
parallelism,
*partition_split_arg,
table_schema->get_tablet_size(),
parent_task_id))) {
LOG_WARN("init global index task failed", K(ret), KPC(table_schema));
} else if (OB_FAIL(split_task.set_trace_id(*ObCurTraceId::get_trace_id()))) {
LOG_WARN("set trace id failed", K(ret));
} else if (OB_FAIL(insert_task_record(proxy, split_task, allocator, task_record))) {
LOG_WARN("fail to insert task record", K(ret));
}
LOG_INFO("ddl_scheduler create partition split task finished", K(ret), K(split_task));
}
return ret;
}
int ObDDLScheduler::create_recover_restore_table_task(
common::ObISQLClient &proxy,
const share::ObDDLType &type,
@ -2340,6 +2504,7 @@ int ObDDLScheduler::recover_task()
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else {
schedule_auto_split_task(); //ignore schedule auto split task error
ObSqlString sql_string;
ObArray<ObDDLTaskRecord> task_records;
ObArray<uint64_t> primary_tenant_ids;
@ -2498,6 +2663,7 @@ int ObDDLScheduler::schedule_ddl_task(const ObDDLTaskRecord &record)
case DDL_ALTER_COLUMN_GROUP:
case DDL_MVIEW_COMPLETE_REFRESH:
case DDL_MODIFY_AUTO_INCREMENT_WITH_REDEFINITION:
case DDL_PARTITION_SPLIT_RECOVERY_TABLE_REDEFINITION:
ret = schedule_table_redefinition_task(record);
break;
case DDL_CREATE_MVIEW:
@ -2527,6 +2693,14 @@ int ObDDLScheduler::schedule_ddl_task(const ObDDLTaskRecord &record)
case DDL_TRUNCATE_SUB_PARTITION:
ret = schedule_ddl_retry_task(record);
break;
case DDL_AUTO_SPLIT_BY_RANGE:
case DDL_AUTO_SPLIT_NON_RANGE:
case DDL_MANUAL_SPLIT_BY_RANGE:
case DDL_MANUAL_SPLIT_NON_RANGE:
if (OB_FAIL(schedule_partition_split_task(record))) {
LOG_WARN("schedule partition split task failed", K(ret));
}
break;
case DDL_TABLE_RESTORE:
ret = schedule_recover_restore_table_task(record);
break;
@ -2805,6 +2979,40 @@ int ObDDLScheduler::schedule_ddl_retry_task(const ObDDLTaskRecord &task_record)
return ret;
}
int ObDDLScheduler::schedule_partition_split_task(
const ObDDLTaskRecord &task_record)
{
int ret = OB_SUCCESS;
ObPartitionSplitTask *split_task = nullptr;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_FAIL(alloc_ddl_task(split_task))) {
LOG_WARN("alloc ddl task failed", K(ret));
} else {
if (OB_FAIL(split_task->init(task_record))) {
LOG_WARN("init partition split task failed", K(ret), K(task_record));
} else if (OB_FAIL(split_task->set_trace_id(task_record.trace_id_))) {
LOG_WARN("set trace id failed", K(ret));
} else if (OB_FAIL(inner_schedule_ddl_task(split_task, task_record))) {
if (OB_ENTRY_EXIST != ret) {
LOG_WARN("inner schedule task failed", K(ret), K(*split_task));
}
} else {
LOG_INFO("scheduler partition split task successfully", K(*split_task));
}
}
if (OB_FAIL(ret) && nullptr != split_task) {
split_task->~ObPartitionSplitTask();
allocator_.free(split_task);
split_task = nullptr;
}
if (OB_ENTRY_EXIST == ret) {
ret = OB_SUCCESS;
}
return ret;
}
int ObDDLScheduler::schedule_constraint_task(const ObDDLTaskRecord &task_record)
{
int ret = OB_SUCCESS;
@ -3222,6 +3430,7 @@ int ObDDLScheduler::on_column_checksum_calc_reply(
int ObDDLScheduler::on_sstable_complement_job_reply(
const common::ObTabletID &tablet_id,
const ObAddr &svr,
const ObDDLTaskKey &task_key,
const int64_t snapshot_version,
const int64_t execution_id,
@ -3249,18 +3458,18 @@ int ObDDLScheduler::on_sstable_complement_job_reply(
if (OB_FAIL(modify_redef_task(task_id, callback))) {
LOG_WARN("fail to modify redef task", K(ret), K(task_id));
}
} else if (OB_FAIL(task_queue_.modify_task(task_key, [&tablet_id, &snapshot_version, &execution_id, &ret_code, &addition_info](ObDDLTask &task) -> int {
} else if (OB_FAIL(task_queue_.modify_task(task_key, [&tablet_id, &svr, &snapshot_version, &execution_id, &ret_code, &addition_info](ObDDLTask &task) -> int {
int ret = OB_SUCCESS;
const int64_t task_type = task.get_task_type();
switch (task_type) {
case ObDDLType::DDL_CREATE_INDEX:
case ObDDLType::DDL_CREATE_PARTITIONED_LOCAL_INDEX:
if (OB_FAIL(static_cast<ObIndexBuildTask *>(&task)->update_complete_sstable_job_status(tablet_id, snapshot_version, execution_id, ret_code, addition_info))) {
if (OB_FAIL(static_cast<ObIndexBuildTask *>(&task)->update_complete_sstable_job_status(tablet_id, svr, snapshot_version, execution_id, ret_code, addition_info))) {
LOG_WARN("update complete sstable job status failed", K(ret));
}
break;
case ObDDLType::DDL_DROP_PRIMARY_KEY:
if (OB_FAIL(static_cast<ObDropPrimaryKeyTask *>(&task)->update_complete_sstable_job_status(tablet_id, snapshot_version, execution_id, ret_code, addition_info))) {
if (OB_FAIL(static_cast<ObDropPrimaryKeyTask *>(&task)->update_complete_sstable_job_status(tablet_id, svr, snapshot_version, execution_id, ret_code, addition_info))) {
LOG_WARN("update complete sstable job status", K(ret));
}
break;
@ -3273,12 +3482,13 @@ int ObDDLScheduler::on_sstable_complement_job_reply(
case ObDDLType::DDL_ALTER_COLUMN_GROUP:
case ObDDLType::DDL_MVIEW_COMPLETE_REFRESH:
case ObDDLType::DDL_MODIFY_AUTO_INCREMENT_WITH_REDEFINITION:
if (OB_FAIL(static_cast<ObTableRedefinitionTask *>(&task)->update_complete_sstable_job_status(tablet_id, snapshot_version, execution_id, ret_code, addition_info))) {
case ObDDLType::DDL_PARTITION_SPLIT_RECOVERY_TABLE_REDEFINITION:
if (OB_FAIL(static_cast<ObTableRedefinitionTask *>(&task)->update_complete_sstable_job_status(tablet_id, svr, snapshot_version, execution_id, ret_code, addition_info))) {
LOG_WARN("update complete sstable job status", K(ret));
}
break;
case ObDDLType::DDL_TABLE_RESTORE:
if (OB_FAIL(static_cast<ObRecoverRestoreTableTask *>(&task)->update_complete_sstable_job_status(tablet_id, snapshot_version, execution_id, ret_code, addition_info))) {
if (OB_FAIL(static_cast<ObRecoverRestoreTableTask *>(&task)->update_complete_sstable_job_status(tablet_id, svr, snapshot_version, execution_id, ret_code, addition_info))) {
LOG_WARN("update complete sstable job status", K(ret));
}
break;
@ -3292,12 +3502,24 @@ int ObDDLScheduler::on_sstable_complement_job_reply(
case ObDDLType::DDL_DROP_COLUMN:
case ObDDLType::DDL_ADD_COLUMN_OFFLINE:
case ObDDLType::DDL_COLUMN_REDEFINITION:
if (OB_FAIL(static_cast<ObColumnRedefinitionTask *>(&task)->update_complete_sstable_job_status(tablet_id, snapshot_version, execution_id, ret_code, addition_info))) {
if (OB_FAIL(static_cast<ObColumnRedefinitionTask *>(&task)->update_complete_sstable_job_status(tablet_id, svr, snapshot_version, execution_id, ret_code, addition_info))) {
LOG_WARN("update complete sstable job status", K(ret), K(tablet_id), K(snapshot_version), K(ret_code));
}
break;
case DDL_AUTO_SPLIT_BY_RANGE:
case DDL_AUTO_SPLIT_NON_RANGE:
case DDL_MANUAL_SPLIT_BY_RANGE:
case DDL_MANUAL_SPLIT_NON_RANGE:
if (OB_FAIL(static_cast<ObPartitionSplitTask *>(&task)->update_complete_sstable_job_status(tablet_id,
svr,
execution_id,
ret_code,
addition_info))) {
LOG_WARN("update partition split task tastus", K(ret));
}
break;
case ObDDLType::DDL_DROP_VEC_INDEX:
if (OB_FAIL(static_cast<ObDropVecIndexTask *>(&task)->update_drop_lob_meta_row_job_status(tablet_id, snapshot_version, execution_id, ret_code, addition_info))) {
if (OB_FAIL(static_cast<ObDropVecIndexTask *>(&task)->update_drop_lob_meta_row_job_status(tablet_id, svr, snapshot_version, execution_id, ret_code, addition_info))) {
LOG_WARN("update complete sstable job status", K(ret), K(tablet_id), K(snapshot_version), K(ret_code));
}
break;

16
src/rootserver/ddl_task/ob_ddl_scheduler.h
View File

@ -24,6 +24,7 @@
#include "rootserver/ddl_task/ob_index_build_task.h"
#include "rootserver/ddl_task/ob_modify_autoinc_task.h"
#include "rootserver/ddl_task/ob_table_redefinition_task.h"
#include "rootserver/ddl_task//ob_partition_split_task.h"
#include "rootserver/ob_thread_idling.h"
#include "lib/hash/ob_hashmap.h"
#include "lib/profile/ob_trace_id.h"
@ -260,6 +261,7 @@ public:
int on_sstable_complement_job_reply(
const common::ObTabletID &tablet_id,
const ObAddr &svr,
const ObDDLTaskKey &task_key,
const int64_t snapshot_version,
const int64_t execution_id,
@ -300,6 +302,9 @@ public:
int prepare_alter_table_arg(const ObPrepareAlterTableArgParam &param,
const ObTableSchema *target_table_schema,
obrpc::ObAlterTableArg &alter_table_arg);
int cache_auto_split_task(const obrpc::ObAutoSplitTabletBatchArg &arg,
obrpc::ObAutoSplitTabletBatchRes &res);
int schedule_auto_split_task();
private:
class DDLIdling : public ObThreadIdling
{
@ -530,6 +535,16 @@ private:
ObIAllocator &allocator,
ObDDLTaskRecord &task_record);
int create_partition_split_task(
common::ObISQLClient &proxy,
const share::schema::ObTableSchema *table_schema,
const int64_t parallelism,
const int64_t parent_task_id,
const int64_t task_id,
const obrpc::ObPartitionSplitArg *partition_split_arg,
ObIAllocator &allocator,
ObDDLTaskRecord &task_record);
int create_recover_restore_table_task(
common::ObISQLClient &proxy,
const share::ObDDLType &type,
@ -561,6 +576,7 @@ private:
int schedule_rebuild_index_task(const ObDDLTaskRecord &task_record);
int schedule_drop_fts_index_task(const ObDDLTaskRecord &task_record);
int schedule_ddl_retry_task(const ObDDLTaskRecord &task_record);
int schedule_partition_split_task(const ObDDLTaskRecord &task_record);
int schedule_recover_restore_table_task(const ObDDLTaskRecord &task_record);
int add_sys_task(ObDDLTask *task);
int remove_sys_task(ObDDLTask *task);

1032
src/rootserver/ddl_task/ob_ddl_single_replica_executor.cpp
View File

File diff suppressed because it is too large Load Diff

288
src/rootserver/ddl_task/ob_ddl_single_replica_executor.h
View File

@ -16,125 +16,259 @@
#include "lib/container/ob_array.h"
#include "common/ob_tablet_id.h"
#include "share/ob_ddl_common.h"
#include "share/ob_rpc_struct.h"
namespace oceanbase
{
namespace rootserver
{
struct ObDDLSingleReplicaExecutorParam final
struct ObDDLReplicaBuildExecutorParam final
{
public:
ObDDLSingleReplicaExecutorParam()
: tenant_id_(common::OB_INVALID_TENANT_ID),
dest_tenant_id_(common::OB_INVALID_TENANT_ID),
type_(share::DDL_INVALID),
ObDDLReplicaBuildExecutorParam ()
: tenant_id_(OB_INVALID_TENANT_ID),
dest_tenant_id_(OB_INVALID_TENANT_ID),
ddl_type_(share::DDL_INVALID),
source_tablet_ids_(),
dest_tablet_ids_(),
source_table_id_(common::OB_INVALID_ID),
dest_table_id_(common::OB_INVALID_ID),
schema_version_(0),
dest_schema_version_(0),
source_table_ids_(OB_INVALID_ID),
dest_table_ids_(OB_INVALID_ID),
source_schema_versions_(0),
dest_schema_versions_(0),
snapshot_version_(0),
task_id_(0),
parallelism_(0),
execution_id_(-1),
data_format_version_(0),
consumer_group_id_(0)
consumer_group_id_(0),
compaction_scns_(),
lob_col_idxs_(),
can_reuse_macro_blocks_(),
parallel_datum_rowkey_list_()
{}
~ObDDLSingleReplicaExecutorParam() = default;
~ObDDLReplicaBuildExecutorParam () = default;
bool is_valid() const {
return common::OB_INVALID_TENANT_ID != tenant_id_ && common::OB_INVALID_TENANT_ID != dest_tenant_id_
&& share::DDL_INVALID != type_ && source_tablet_ids_.count() > 0 && dest_tablet_ids_.count() > 0
&& common::OB_INVALID_ID != source_table_id_ && common::OB_INVALID_ID != dest_table_id_
&& schema_version_ > 0 && dest_schema_version_ > 0 && snapshot_version_ > 0 && task_id_ > 0
&& execution_id_ >= 0 && data_format_version_ > 0 && consumer_group_id_ >= 0;
bool is_valid = tenant_id_ != OB_INVALID_TENANT_ID &&
dest_tenant_id_ != OB_INVALID_TENANT_ID &&
ddl_type_ != share::DDL_INVALID &&
source_tablet_ids_.count() > 0 &&
dest_tablet_ids_.count() == source_tablet_ids_.count() &&
source_table_ids_.count() == source_tablet_ids_.count() &&
dest_table_ids_.count() == source_tablet_ids_.count() &&
source_schema_versions_.count() == source_tablet_ids_.count() &&
dest_schema_versions_.count() == source_tablet_ids_.count() &&
snapshot_version_ > 0 &&
task_id_ > 0 &&
execution_id_ >= 0 &&
data_format_version_ > 0 &&
consumer_group_id_ >= 0;
if (is_tablet_split(ddl_type_)) {
is_valid = is_valid && compaction_scns_.count() == source_tablet_ids_.count()
&& can_reuse_macro_blocks_.count() == source_tablet_ids_.count();
} else {
is_valid = (is_valid && compaction_scns_.count() == 0);
}
return is_valid;
}
TO_STRING_KV(K_(tenant_id), K_(dest_tenant_id), K_(type), K_(source_tablet_ids), K_(dest_tablet_ids),
K_(source_table_id), K_(dest_table_id), K_(schema_version), K_(dest_schema_version),
K_(snapshot_version), K_(task_id), K_(parallelism), K_(execution_id),
K_(data_format_version), K_(consumer_group_id));
TO_STRING_KV(K_(tenant_id), K_(dest_tenant_id), K_(ddl_type), K_(source_tablet_ids),
K_(dest_tablet_ids), K_(source_table_ids), K_(dest_table_ids),
K_(source_schema_versions), K_(dest_schema_versions), K_(snapshot_version),
K_(task_id), K_(parallelism), K_(execution_id),
K_(data_format_version), K_(consumer_group_id), K_(can_reuse_macro_blocks),
K_(parallel_datum_rowkey_list));
public:
uint64_t tenant_id_;
uint64_t dest_tenant_id_;
share::ObDDLType type_;
common::ObArray<common::ObTabletID> source_tablet_ids_;
common::ObArray<common::ObTabletID> dest_tablet_ids_;
int64_t source_table_id_;
int64_t dest_table_id_;
int64_t schema_version_;
int64_t dest_schema_version_;
share::ObDDLType ddl_type_;
ObArray<ObTabletID> source_tablet_ids_;
ObSArray<ObTabletID> dest_tablet_ids_;
ObSArray<uint64_t> source_table_ids_;
ObSArray<uint64_t> dest_table_ids_;
ObSArray<uint64_t> source_schema_versions_;
ObSArray<uint64_t> dest_schema_versions_;
int64_t snapshot_version_;
int64_t task_id_;
int64_t parallelism_;
int64_t execution_id_;
int64_t data_format_version_;
int64_t consumer_group_id_;
ObSArray<int64_t> compaction_scns_;
ObSArray<uint64_t> lob_col_idxs_;
ObSArray<bool> can_reuse_macro_blocks_;
common::ObSEArray<common::ObSEArray<blocksstable::ObDatumRowkey, 8>, 8> parallel_datum_rowkey_list_;
};
class ObDDLSingleReplicaExecutor
enum class ObReplicaBuildStat
{
BUILD_INIT = 0,
BUILD_REQUESTED = 1,
BUILD_SUCCEED = 2,
BUILD_RETRY = 3,
BUILD_FAILED = 4
};
struct ObSingleReplicaBuildCtx final
{
public:
int build(const ObDDLSingleReplicaExecutorParam &param);
static const int64_t REPLICA_BUILD_HEART_BEAT_TIME = 10 * 1000 * 1000;
ObSingleReplicaBuildCtx()
: is_inited_(false),
addr_(),
ddl_type_(share::DDL_INVALID),
src_table_id_(OB_INVALID_ID),
dest_table_id_(OB_INVALID_ID),
src_schema_version_(0),
dest_schema_version_(0),
tablet_task_id_(0),
compaction_scn_(0),
src_tablet_id_(ObTabletID::INVALID_TABLET_ID),
dest_tablet_id_(),
can_reuse_macro_block_(false),
parallel_datum_rowkey_list_(),
stat_(ObReplicaBuildStat::BUILD_INIT),
ret_code_(OB_SUCCESS),
heart_beat_time_(0),
row_inserted_(0),
row_scanned_(0),
physical_row_count_(0)
{ }
~ObSingleReplicaBuildCtx() = default;
int init(const ObAddr& addr,
const share::ObDDLType ddl_type,
const int64_t src_table_id,
const int64_t dest_table_id,
const int64_t src_schema_version,
const int64_t dest_schema_version,
const int64_t tablet_task_id,
const int64_t compaction_scn,
const ObTabletID &src_tablet_id,
const ObTabletID &dest_tablet_ids,
const bool can_reuse_macro_block,
const ObIArray<blocksstable::ObDatumRowkey> &parallel_datum_rowkey_list);
void reset_build_stat();
bool is_valid() const;
int assign(const ObSingleReplicaBuildCtx &other);
int check_need_schedule(bool &need_schedule) const;
TO_STRING_KV(K(is_inited_), K(addr_), K(ddl_type_), K(src_table_id_),
K(src_schema_version_), K(dest_schema_version_),
K(dest_table_id_), K(tablet_task_id_), K(compaction_scn_),
K(src_tablet_id_), K(dest_tablet_id_), K_(can_reuse_macro_block),
K(parallel_datum_rowkey_list_), K(stat_), K(ret_code_),
K(heart_beat_time_), K(row_inserted_), K(row_scanned_), K(physical_row_count_));
public:
bool is_inited_;
ObAddr addr_;
share::ObDDLType ddl_type_;
int64_t src_table_id_;
int64_t dest_table_id_;
int64_t src_schema_version_;
int64_t dest_schema_version_;
int64_t tablet_task_id_;
int64_t compaction_scn_;
ObTabletID src_tablet_id_;
ObTabletID dest_tablet_id_;
bool can_reuse_macro_block_;
common::ObSEArray<blocksstable::ObDatumRowkey, 8> parallel_datum_rowkey_list_;
ObReplicaBuildStat stat_;
int64_t ret_code_;
int64_t heart_beat_time_;
int64_t row_inserted_;
int64_t row_scanned_;
int64_t physical_row_count_;
};
class ObDDLReplicaBuildExecutor
{
public:
ObDDLReplicaBuildExecutor()
: is_inited_(false),
tenant_id_(OB_INVALID_TENANT_ID),
dest_tenant_id_(OB_INVALID_TENANT_ID),
ddl_type_(share::ObDDLType::DDL_INVALID),
ddl_task_id_(0),
snapshot_version_(0),
parallelism_(0),
execution_id_(0),
data_format_version_(0),
consumer_group_id_(0),
lob_col_idxs_(),
src_tablet_ids_(),
dest_tablet_ids_(),
replica_build_ctxs_(),
lock_()
{}
~ObDDLReplicaBuildExecutor() = default;
int build(const ObDDLReplicaBuildExecutorParam &param);
int check_build_end(const bool need_checksum, bool &is_end, int64_t &ret_code);
int set_partition_task_status(const common::ObTabletID &tablet_id,
const int ret_code,
const int64_t row_scanned,
const int64_t row_inserted);
int get_progress(int64_t &row_scanned, int64_t &row_inserted);
int update_build_progress(const ObTabletID &tablet_id,
const ObAddr &addr,
const int ret_code,
const int64_t row_scanned,
const int64_t row_inserted,
const int64_t physical_row_count);
int get_progress(int64_t &row_inserted, int64_t &physical_row_count_, double& percent);
TO_STRING_KV(K(is_inited_), K(tenant_id_), K(dest_tenant_id_), K(ddl_type_),
K(ddl_task_id_), K(snapshot_version_), K(parallelism_),
K(execution_id_), K(data_format_version_), K(consumer_group_id_),
K(lob_col_idxs_), K(src_tablet_ids_), K(dest_tablet_ids_),
K(replica_build_ctxs_));
private:
int schedule_task();
int process_rpc_results(
const ObArray<ObTabletID> &tablet_ids,
const ObArray<ObAddr> addrs,
const ObIArray<const obrpc::ObDDLBuildSingleReplicaRequestResult *> &result_array,
const ObArray<int> &ret_array);
int construct_rpc_arg(
const ObSingleReplicaBuildCtx &replica_build_ctx,
obrpc::ObDDLBuildSingleReplicaRequestArg &arg) const;
int construct_replica_build_ctxs(
const ObDDLReplicaBuildExecutorParam &param,
ObArray<ObSingleReplicaBuildCtx> &replica_build_ctxs) const;
int get_refreshed_replica_addrs(
const bool send_to_all_replicas,
ObArray<ObTabletID> &replica_tablet_ids,
ObArray<ObAddr> &replica_addrs) const;
int refresh_replica_build_ctxs(
const ObArray<ObTabletID> &replica_tablet_ids,
const ObArray<ObAddr> &replica_addrs);
int update_build_ctx(
ObSingleReplicaBuildCtx &build_ctx,
const obrpc::ObDDLBuildSingleReplicaRequestResult *result,
const int ret_code);
int update_replica_build_ctx(
ObSingleReplicaBuildCtx &build_ctx,
const int64_t ret_code,
const int64_t row_scanned,
const int64_t row_inserted,
const int64_t row_count,
const bool is_rpc_request,
const bool is_observer_report);
int get_replica_build_ctx(
const ObTabletID &tablet_id,
const ObAddr &addr,
ObSingleReplicaBuildCtx *&replica_build_ctx,
bool &is_found);
private:
enum class ObPartitionBuildStat
{
BUILD_INIT = 0,
BUILD_REQUESTED = 1,
BUILD_SUCCEED = 2,
BUILD_RETRY = 3,
BUILD_FAILED = 4
};
struct ObPartitionBuildInfo final
{
public:
static const int64_t PARTITION_BUILD_HEART_BEAT_TIME = 10 * 1000 * 1000;
ObPartitionBuildInfo()
: ret_code_(common::OB_SUCCESS), stat_(ObPartitionBuildStat::BUILD_INIT), heart_beat_time_(0),
row_inserted_(0), row_scanned_(0)
{}
~ObPartitionBuildInfo() = default;
bool need_schedule() const {
return ObPartitionBuildStat::BUILD_INIT == stat_
|| ObPartitionBuildStat::BUILD_RETRY == stat_
|| (ObPartitionBuildStat::BUILD_REQUESTED == stat_
&& ObTimeUtility::current_time() - heart_beat_time_ > PARTITION_BUILD_HEART_BEAT_TIME);
}
TO_STRING_KV(K_(ret_code), K_(stat), K_(heart_beat_time), K_(row_inserted), K_(row_scanned));
public:
int64_t ret_code_;
ObPartitionBuildStat stat_;
int64_t heart_beat_time_;
int64_t row_inserted_;
int64_t row_scanned_;
};
private:
bool is_inited_;
uint64_t tenant_id_;
uint64_t dest_tenant_id_;
share::ObDDLType type_;
common::ObArray<common::ObTabletID> source_tablet_ids_;
common::ObArray<common::ObTabletID> dest_tablet_ids_;
common::ObArray<int64_t> tablet_task_ids_;
int64_t source_table_id_;
int64_t dest_table_id_;
int64_t schema_version_;
int64_t dest_schema_version_;
share::ObDDLType ddl_type_;
int64_t ddl_task_id_;
int64_t snapshot_version_;
int64_t task_id_;
int64_t parallelism_;
int64_t execution_id_;
int64_t data_format_version_;
int64_t consumer_group_id_;
common::ObArray<ObPartitionBuildInfo> partition_build_stat_;
common::ObSpinLock lock_;
ObSArray<uint64_t> lob_col_idxs_;
ObArray<ObTabletID> src_tablet_ids_;
ObSArray<ObTabletID> dest_tablet_ids_;
ObArray<ObSingleReplicaBuildCtx> replica_build_ctxs_; // NOTE hold lock before access
ObSpinLock lock_; // NOTE keep rpc send out of lock scope
};
} // end namespace rootserver

101
src/rootserver/ddl_task/ob_ddl_task.cpp
View File

@ -337,6 +337,7 @@ trace::ObSpanCtx* ObDDLTracing::begin_task_span()
case DDL_CONVERT_TO_CHARACTER:
case DDL_TABLE_REDEFINITION:
case DDL_ALTER_COLUMN_GROUP:
case DDL_PARTITION_SPLIT_RECOVERY_TABLE_REDEFINITION:
span = FLT_BEGIN_SPAN(ddl_table_redefinition);
break;
case DDL_DROP_PRIMARY_KEY:
@ -421,6 +422,7 @@ trace::ObSpanCtx* ObDDLTracing::restore_task_span()
case DDL_CONVERT_TO_CHARACTER:
case DDL_TABLE_REDEFINITION:
case DDL_ALTER_COLUMN_GROUP:
case DDL_PARTITION_SPLIT_RECOVERY_TABLE_REDEFINITION:
span = FLT_RESTORE_DDL_SPAN(ddl_table_redefinition, task_span_id_, task_start_ts_);
break;
case DDL_DROP_PRIMARY_KEY:
@ -927,6 +929,9 @@ int ObDDLTask::get_ddl_type_str(const int64_t ddl_type, const char *&ddl_type_st
case DDL_MODIFY_AUTO_INCREMENT_WITH_REDEFINITION:
ddl_type_str = "modify auto increment column with redefinition";
break;
case DDL_PARTITION_SPLIT_RECOVERY_TABLE_REDEFINITION:
ddl_type_str = "partition split recovery table redefinition";
break;
default:
ret = OB_ERR_UNEXPECTED;
}
@ -1169,7 +1174,7 @@ int ObDDLTask::switch_status(const ObDDLTaskStatus new_status, const bool enable
ObDDLTaskStatus real_new_status = new_status;
const ObDDLTaskStatus old_status = task_status_;
const bool error_need_retry = OB_SUCCESS != ret_code && is_error_need_retry(ret_code);
if (OB_TMP_FAIL(check_ddl_task_is_cancel(trace_id_, is_cancel))) {
if (!is_partition_split_recovery_table_redefinition(task_type_) && OB_TMP_FAIL(check_ddl_task_is_cancel(trace_id_, is_cancel))) {
LOG_WARN("check ddl task is cancel failed", K(tmp_ret), K(task_id_), K(parent_task_id_), K_(trace_id));
}
if (is_cancel) {
@ -1183,7 +1188,11 @@ int ObDDLTask::switch_status(const ObDDLTaskStatus new_status, const bool enable
real_ret_code = OB_SUCCESS;
}
ret_code_ = OB_SUCCESS == ret_code_ ? real_ret_code : ret_code_;
real_new_status = OB_SUCCESS != real_ret_code ? FAIL : real_new_status;
if (is_tablet_split(task_type_)) {
real_new_status = OB_SUCCESS != real_ret_code ? WAIT_PARTITION_SPLIT_RECOVERY_TASK_FINISH : real_new_status;
} else {
real_new_status = OB_SUCCESS != real_ret_code ? FAIL : real_new_status;
}
ObMySQLTransaction trans;
ObRootService *root_service = nullptr;
if (OB_ISNULL(root_service = GCTX.root_service_)) {
@ -1945,6 +1954,41 @@ int ObDDLWaitTransEndCtx::init(
return ret;
}
int ObDDLWaitTransEndCtx::init(
const uint64_t tenant_id,
const int64_t ddl_task_id,
const uint64_t table_id,
const ObIArray<ObTabletID> &tablet_ids,
const WaitTransType wait_trans_type,
const int64_t wait_version)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(is_inited_)) {
ret = OB_INIT_TWICE;
LOG_WARN("init twice", K(ret), K(is_inited_));
} else if (OB_UNLIKELY(OB_INVALID_ID == tenant_id
|| ddl_task_id <= 0
|| table_id <= 0
|| tablet_ids.count() <= 0
|| !is_wait_trans_type_valid(wait_trans_type)
|| wait_version <= 0)) {
LOG_WARN("invalid argument", K(ret), K(tenant_id), K(ddl_task_id), K(table_id), K(tablet_ids.count()), K(wait_trans_type), K(wait_version));
} else if (OB_FAIL(tablet_ids_.assign(tablet_ids))) {
LOG_WARN("failed to assign", K(ret));
} else if (OB_FAIL(snapshot_array_.prepare_allocate(tablet_ids_.count()))) {
LOG_WARN("failed to prepare allocate", K(ret));
} else {
tenant_id_ = tenant_id;
ddl_task_id_ = ddl_task_id;
table_id_ = table_id;
wait_type_ = wait_trans_type;
wait_version_ = wait_version;
is_trans_end_ = false;
is_inited_ = true;
}
return ret;
}
void ObDDLWaitTransEndCtx::reset()
{
is_inited_ = false;
@ -3627,6 +3671,7 @@ int ObDDLTaskRecordOperator::check_has_conflict_ddl(
case ObDDLType::DDL_DIRECT_LOAD_INSERT:
case ObDDLType::DDL_ALTER_COLUMN_GROUP:
case ObDDLType::DDL_MVIEW_COMPLETE_REFRESH:
case ObDDLType::DDL_PARTITION_SPLIT_RECOVERY_TABLE_REDEFINITION:
has_conflict_ddl = true;
break;
default:
@ -4204,6 +4249,58 @@ int ObDDLTaskRecordOperator::kill_inner_sql(
return ret;
}
int ObDDLTaskRecordOperator::get_partition_split_task_ids(
common::ObISQLClient &proxy,
const uint64_t tenant_id,
const ObIArray<uint64_t> &table_ids,
ObIArray<int64_t> &task_ids)
{
int ret = OB_SUCCESS;
ObSqlString sql_table_ids;
ObSqlString sql_string;
task_ids.reset();
if (OB_UNLIKELY(tenant_id <= 0 || table_ids.count() < 1)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(tenant_id), K(table_ids.count()));
} else {
SMART_VAR(ObMySQLProxy::MySQLResult, res) {
sqlclient::ObMySQLResult *result = NULL;
for (int64_t i = 0; OB_SUCC(ret) && i < table_ids.count(); i++) {
if (OB_FAIL(sql_table_ids.append_fmt("%s %lu", 0 == i ? "" : ", ", table_ids.at(i)))) {
LOG_WARN("append sql failed", K(ret));
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(sql_string.assign_fmt("SELECT task_id FROM %s WHERE object_id in (%.*s) AND ddl_type >= %u AND ddl_type <= %u ",
OB_ALL_DDL_TASK_STATUS_TNAME, static_cast<int>(sql_table_ids.length()), sql_table_ids.ptr(), ObDDLType::DDL_AUTO_SPLIT_BY_RANGE, ObDDLType::DDL_MANUAL_SPLIT_NON_RANGE))) {
LOG_WARN("assign sql string failed", K(ret));
} else if (OB_FAIL(proxy.read(res, tenant_id, sql_string.ptr()))) {
LOG_WARN("query task_ids task failed", K(ret), K(sql_string));
} else if (OB_UNLIKELY(NULL == (result = res.get_result()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("fail to get sql result", K(ret));
} else {
while (OB_SUCC(ret) && OB_SUCC(result->next())) {
uint64_t task_id = OB_INVALID_ID;
EXTRACT_INT_FIELD_MYSQL(*result, "task_id", task_id, uint64_t);
if (OB_SUCC(ret)) {
if (OB_FAIL(task_ids.push_back(task_id))) {
LOG_WARN("fail to push back task id", K(ret), K(task_id));
}
}
}
if (OB_ITER_END == ret) {
ret = OB_SUCCESS;
} else {
LOG_WARN("fail to iter result", K(ret));
}
}
}
}
}
return ret;
}
int ObDDLTaskRecordOperator::kill_task_inner_sql(
common::ObMySQLProxy &proxy,
const common::ObCurTraceId::TraceId &trace_id,

24
src/rootserver/ddl_task/ob_ddl_task.h
View File

@ -106,12 +106,13 @@ public:
struct ObDDLTaskInfo final
{
public:
ObDDLTaskInfo() : row_scanned_(0), row_inserted_(0) {}
ObDDLTaskInfo() : row_scanned_(0), row_inserted_(0), physical_row_count_(0) {}
~ObDDLTaskInfo() {}
TO_STRING_KV(K_(row_scanned), K_(row_inserted), K_(ls_id), K_(ls_leader_addr), K_(partition_ids));
TO_STRING_KV(K_(row_scanned), K_(row_inserted), K_(physical_row_count), K_(ls_id), K_(ls_leader_addr), K_(partition_ids));
public:
int64_t row_scanned_;
int64_t row_inserted_;
int64_t physical_row_count_;
share::ObLSID ls_id_;
common::ObAddr ls_leader_addr_;
ObArray<ObTabletID> partition_ids_;
@ -467,6 +468,11 @@ public:
common::ObIAllocator &allocator,
common::ObIArray<ObString> &records);
static int get_partition_split_task_ids(
common::ObISQLClient &proxy,
const uint64_t tenant_id,
const ObIArray<uint64_t> &table_ids,
ObIArray<int64_t> &task_ids);
static int check_rebuild_vec_index_task_exist(
const uint64_t tenant_id,
const uint64_t data_table_id,
@ -515,6 +521,13 @@ public:
const uint64_t table_id,
const WaitTransType wait_trans_type,
const int64_t wait_version);
int init(
const uint64_t tenant_id,
const int64_t ddl_task_id,
const uint64_t table_id,
const common::ObIArray<common::ObTabletID> &tablet_ids,
const WaitTransType wait_trans_type,
const int64_t wait_version);
void reset();
bool is_inited() const { return is_inited_; }
int try_wait(bool &is_trans_end, int64_t &snapshot_version, const bool need_wait_trans_end = true);
@ -758,7 +771,6 @@ public:
static bool check_is_load_data(share::ObDDLType task_type);
virtual bool support_longops_monitoring() const { return false; }
int cleanup();
virtual int cleanup_impl() { return OB_NOT_SUPPORTED; }
int update_task_record_status_and_msg(common::ObISQLClient &proxy, const share::ObDDLTaskStatus real_new_status);
#ifdef ERRSIM
@ -806,10 +818,12 @@ protected:
|| MAX_ERR_TOLERANCE_CNT > ++err_code_occurence_cnt_));
}
int init_ddl_task_monitor_info(const uint64_t target_table_id);
virtual bool task_can_retry() const { return true; }
virtual bool is_ddl_retryable() const { return true; }
private:
void clear_old_status_context();
virtual int cleanup_impl() { return OB_NOT_SUPPORTED; }
virtual bool task_can_retry() const { return true; }
protected:
virtual void clear_old_status_context();
protected:
static const int64_t TASK_EXECUTE_TIME_THRESHOLD = 3 * 24 * 60 * 60 * 1000000L; // 3 days
common::TCRWLock lock_;

2
src/rootserver/ddl_task/ob_drop_index_task.h
View File

@ -43,7 +43,6 @@ public:
virtual int deserialize_params_from_message(const uint64_t tenant_id, const char *buf, const int64_t buf_size, int64_t &pos) override;
virtual int64_t get_serialize_param_size() const override;
INHERIT_TO_STRING_KV("ObDDLTask", ObDDLTask, KP_(root_service));
virtual int cleanup_impl() override;
private:
int check_switch_succ();
@ -55,6 +54,7 @@ private:
int drop_index(const share::ObDDLTaskStatus new_status);
int succ();
int fail();
virtual int cleanup_impl() override;
int deep_copy_index_arg(common::ObIAllocator &allocator,
const obrpc::ObDropIndexArg &src_index_arg,
obrpc::ObDropIndexArg &dst_index_arg);

39
src/rootserver/ddl_task/ob_drop_vec_index_task.cpp
View File

@ -925,14 +925,10 @@ int ObDropVecIndexTask::send_build_single_replica_request()
ret = OB_NOT_INIT;
LOG_WARN("ObColumnRedefinitionTask has not been inited", K(ret));
} else {
ObDDLSingleReplicaExecutorParam param;
ObDDLReplicaBuildExecutorParam param;
param.tenant_id_ = tenant_id_;
param.dest_tenant_id_ = dst_tenant_id_;
param.type_ = task_type_;
param.source_table_id_ = vec_index_snapshot_data_.table_id_;
param.dest_table_id_ = target_object_id_;
param.schema_version_ = schema_version_;
param.dest_schema_version_ = dst_schema_version_;
param.ddl_type_ = task_type_;
param.snapshot_version_ = snapshot_version_; // should > 0, but = 0
param.task_id_ = task_id_;
param.parallelism_ = std::max(parallelism_, 1L);
@ -944,6 +940,26 @@ int ObDropVecIndexTask::send_build_single_replica_request()
LOG_WARN("fail to get tablets", K(ret), K(tenant_id_), K(object_id_));
} else if (OB_FAIL(ObDDLUtil::get_tablets(dst_tenant_id_, vec_index_snapshot_data_.table_id_, param.dest_tablet_ids_))) {
LOG_WARN("fail to get tablets", K(ret), K(tenant_id_), K(target_object_id_));
}
const int64_t src_tablet_cnt = param.source_tablet_ids_.count();
for (int64_t i = 0; OB_SUCC(ret) && i < src_tablet_cnt; ++i) {
if (OB_FAIL(param.source_table_ids_.push_back(vec_index_snapshot_data_.table_id_))) {
LOG_WARN("failed to push back src table id", K(ret));
} else if (OB_FAIL(param.source_schema_versions_.push_back(schema_version_))) {
LOG_WARN("failed to push back src schema version", K(ret));
}
}
const int64_t dest_tablet_cnt = param.dest_tablet_ids_.count();
for (int64_t i = 0; OB_SUCC(ret) && i < dest_tablet_cnt; ++i) {
if (OB_FAIL(param.dest_table_ids_.push_back(target_object_id_))) {
LOG_WARN("failed to push back dest table id", K(ret));
} else if (OB_FAIL(param.dest_schema_versions_.push_back(dst_schema_version_))) {
LOG_WARN("failed to push back dest schema version", K(ret));
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(replica_builder_.build(param))) {
LOG_WARN("fail to send build single replica", K(ret), K(param));
} else {
@ -975,6 +991,7 @@ int ObDropVecIndexTask::check_build_single_replica(bool &is_end)
// update sstable complement status for all leaders
int ObDropVecIndexTask::update_drop_lob_meta_row_job_status(const common::ObTabletID &tablet_id,
const ObAddr &addr,
const int64_t snapshot_version,
const int64_t execution_id,
const int ret_code,
@ -991,10 +1008,12 @@ int ObDropVecIndexTask::update_drop_lob_meta_row_job_status(const common::ObTabl
LOG_WARN("snapshot version not match", K(ret), K(snapshot_version), K(snapshot_version_));
} else if (execution_id < execution_id_) {
LOG_INFO("receive a mismatch execution result, ignore", K(ret_code), K(execution_id), K(execution_id_));
} else if (OB_FAIL(replica_builder_.set_partition_task_status(tablet_id,
ret_code,
addition_info.row_scanned_,
addition_info.row_inserted_))) {
} else if (OB_FAIL(replica_builder_.update_build_progress(tablet_id,
addr,
ret_code,
addition_info.row_scanned_,
addition_info.row_inserted_,
addition_info.physical_row_count_))) {
LOG_WARN("fail to set partition task status", K(ret));
}
return ret;

3
src/rootserver/ddl_task/ob_drop_vec_index_task.h
View File

@ -54,6 +54,7 @@ public:
virtual int64_t get_serialize_param_size() const override;
virtual int on_child_task_finish(const uint64_t child_task_key, const int ret_code) override { return OB_SUCCESS; }
int update_drop_lob_meta_row_job_status(const common::ObTabletID &tablet_id,
const ObAddr &addr,
const int64_t snapshot_version,
const int64_t execution_id,
const int ret_code,
@ -109,7 +110,7 @@ private:
ObVecIndexDDLChildTaskInfo vec_index_id_;
ObVecIndexDDLChildTaskInfo vec_index_snapshot_data_;
obrpc::ObDropIndexArg drop_index_arg_;
ObDDLSingleReplicaExecutor replica_builder_;
ObDDLReplicaBuildExecutor replica_builder_;
common::hash::ObHashMap<common::ObTabletID, common::ObTabletID> check_dag_exit_tablets_map_; // for delete lob meta row data ddl only.
ObDDLWaitTransEndCtx wait_trans_ctx_;
int64_t delte_lob_meta_request_time_;

17
src/rootserver/ddl_task/ob_index_build_task.cpp
View File

@ -81,6 +81,7 @@ int ObIndexSSTableBuildTask::process()
const ObSysVariableSchema *sys_variable_schema = NULL;
bool oracle_mode = false;
ObTabletID unused_tablet_id;
ObAddr unused_addr;
const ObTableSchema *data_schema = nullptr;
const ObTableSchema *index_schema = nullptr;
bool need_padding = false;
@ -196,9 +197,9 @@ int ObIndexSSTableBuildTask::process()
}
}
ObDDLTaskKey task_key(tenant_id_, dest_table_id_, schema_version_);
int tmp_ret = 0;
tmp_ret = root_service_->get_ddl_scheduler().on_sstable_complement_job_reply(
unused_tablet_id, task_key, snapshot_version_, execution_id_, ret, addition_info_);
ObDDLTaskInfo info;
int tmp_ret = root_service_->get_ddl_scheduler().on_sstable_complement_job_reply(
unused_tablet_id, unused_addr, task_key, snapshot_version_, execution_id_, ret, addition_info_);
if (OB_SUCCESS != tmp_ret) {
LOG_WARN("report build finish failed", K(ret), K(tmp_ret));
ret = OB_SUCCESS == ret ? tmp_ret : ret;
@ -851,7 +852,8 @@ int ObIndexBuildTask::reap_old_replica_build_task(bool &need_exec_new_inner_sql)
LOG_WARN("failed to check and wait old complement task", K(ret));
}
} else if (!need_exec_new_inner_sql) {
if (OB_FAIL(update_complete_sstable_job_status(unused_tablet_id, snapshot_version_, old_execution_id, old_ret_code, unused_addition_info))) {
ObAddr unused_addr;
if (OB_FAIL(update_complete_sstable_job_status(unused_tablet_id, unused_addr, snapshot_version_, old_execution_id, old_ret_code, unused_addition_info))) {
LOG_INFO("succ to wait and complete old task finished!", K(ret));
}
}
@ -1387,6 +1389,7 @@ int ObIndexBuildTask::update_column_checksum_calc_status(
int ObIndexBuildTask::update_complete_sstable_job_status(
const common::ObTabletID &tablet_id,
const ObAddr &addr,
const int64_t snapshot_version,
const int64_t execution_id,
const int ret_code,
@ -1399,7 +1402,7 @@ int ObIndexBuildTask::update_complete_sstable_job_status(
LOG_WARN("not init", K(ret));
} else if (OB_UNLIKELY(snapshot_version <= 0)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(snapshot_version), K(ret_code));
LOG_WARN("invalid argument", K(ret), K(snapshot_version), K(addr), K(ret_code));
} else if (OB_FAIL(DDL_SIM(tenant_id_, task_id_, UPDATE_COMPLETE_SSTABLE_FAILED))) {
LOG_WARN("ddl sim failure", K(tenant_id_), K(task_id_));
} else if (ObDDLTaskStatus::REDEFINITION != task_status_) {
@ -1407,7 +1410,7 @@ int ObIndexBuildTask::update_complete_sstable_job_status(
LOG_INFO("not waiting data complete, may finished", K(task_status_));
} else if (snapshot_version != snapshot_version_) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("snapshot version not match", K(ret), K(snapshot_version), K(snapshot_version_));
LOG_WARN("snapshot version not match", K(ret), K(addr), K(snapshot_version), K(snapshot_version_));
} else {
if (is_create_partitioned_local_index()) {
if (OB_UNLIKELY(addition_info.partition_ids_.count() < 1 || !addition_info.ls_id_.is_valid() || !addition_info.ls_leader_addr_.is_valid())) {
@ -1426,9 +1429,9 @@ int ObIndexBuildTask::update_complete_sstable_job_status(
complete_sstable_job_ret_code_ = ret_code;
sstable_complete_ts_ = ObTimeUtility::current_time();
execution_id_ = execution_id;
LOG_INFO("update complete sstable job return code", K(ret), K(target_object_id_), K(tablet_id), K(snapshot_version), K(ret_code), K(execution_id_), K(addition_info));
}
}
LOG_INFO("update complete sstable job return code", K(ret), K(addr), K(target_object_id_), K(tablet_id), K(snapshot_version), K(ret_code), K(execution_id_));
return ret;
}

3
src/rootserver/ddl_task/ob_index_build_task.h
View File

@ -111,12 +111,12 @@ public:
const int ret_code);
int update_complete_sstable_job_status(
const common::ObTabletID &tablet_id,
const ObAddr &addr,
const int64_t snapshot_version,
const int64_t execution_id,
const int ret_code,
const ObDDLTaskInfo &addition_info);
virtual int process() override;
virtual int cleanup_impl() override;
virtual bool is_valid() const override;
virtual int collect_longops_stat(share::ObLongopsValue &value) override;
virtual int serialize_params_to_message(char *buf, const int64_t buf_size, int64_t &pos) const override;
@ -136,6 +136,7 @@ private:
int enable_index();
int clean_on_failed();
int succ();
virtual int cleanup_impl() override;
int hold_snapshot(const int64_t snapshot);
int release_snapshot(const int64_t snapshot);
int update_index_status_in_schema(

2
src/rootserver/ddl_task/ob_modify_autoinc_task.cpp
View File

@ -313,7 +313,7 @@ int ObModifyAutoincTask::unlock_table()
} else if (OB_FAIL(owner_id.convert_from_value(ObLockOwnerType::DEFAULT_OWNER_TYPE,
task_id_))) {
LOG_WARN("convert owner id failed", K(ret), K(task_id_));
} else if (OB_FAIL(ObDDLLock::unlock_for_offline_ddl(tenant_id_, object_id_, owner_id, trans))) {
} else if (OB_FAIL(ObDDLLock::unlock_for_offline_ddl(tenant_id_, object_id_, nullptr/*hidden_tablet_ids_alone*/, owner_id, trans))) {
LOG_WARN("failed to unlock table", K(ret));
}

2
src/rootserver/ddl_task/ob_modify_autoinc_task.h
View File

@ -69,13 +69,13 @@ public:
virtual int deserialize_params_from_message(const uint64_t tenant_id, const char *buf, const int64_t buf_size, int64_t &pos) override;
virtual int64_t get_serialize_param_size() const override;
int notify_update_autoinc_finish(const uint64_t autoinc_val, const int ret_code);
virtual int cleanup_impl() override;
private:
int unlock_table();
int modify_autoinc();
int wait_trans_end();
int fail();
int success();
virtual int cleanup_impl() override;
int set_schema_available();
int rollback_schema();
int check_update_autoinc_end(bool &is_end);

2626
src/rootserver/ddl_task/ob_partition_split_task.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

258
src/rootserver/ddl_task/ob_partition_split_task.h Normal file
View File

@ -0,0 +1,258 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_ROOTSERVER_OB_PARTITION_SPLIT_TASK_H
#define OCEANBASE_ROOTSERVER_OB_PARTITION_SPLIT_TASK_H
#include "lib/net/ob_addr.h"
#include "rootserver/ddl_task/ob_ddl_task.h"
#include "share/ob_rpc_struct.h"
namespace oceanbase
{
namespace rootserver
{
enum ObCheckProgressStatus
{
NOT_STARTED = 0,
ONGOING = 1,
DONE,
};
enum ObPartitionSplitReplicaType
{
DATA_TABLET_REPLICA = 0,
LOCAL_INDEX_TABLET_REPLICA = 1,
LOB_TABLET_REPLICA = 2
};
template <typename T>
class ObCheckProgressKey final
{
public:
ObCheckProgressKey() = default;
ObCheckProgressKey(
const T &field,
const ObTabletID &tablet_id);
~ObCheckProgressKey() = default;
uint64_t hash() const;
int hash(uint64_t &hash_val) const
{
hash_val = hash(); return OB_SUCCESS;
}
bool operator==(const ObCheckProgressKey &other) const;
bool operator!=(const ObCheckProgressKey &other) const;
bool is_valid() const
{
return tablet_id_.is_valid();
}
int assign(const ObCheckProgressKey&other);
TO_STRING_KV(K(field_), K(tablet_id_));
public:
T field_;
ObTabletID tablet_id_;
};
class ObSplitFinishItem final
{
public:
ObSplitFinishItem()
: leader_addr_(), finish_arg_()
{}
~ObSplitFinishItem() {}
TO_STRING_KV(K(leader_addr_), K(finish_arg_));
public:
ObAddr leader_addr_;
obrpc::ObTabletSplitFinishArg finish_arg_;
};
// the process of partition split
class ObPartitionSplitTask final : public ObDDLTask
{
public:
ObPartitionSplitTask();
int init(
const uint64_t tenant_id,
const int64_t task_id,
const int64_t table_id,
const int64_t schema_version,
const int64_t parallelism,
const obrpc::ObPartitionSplitArg &partition_split_arg,
const int64_t tablet_size,
const int64_t parent_task_id = 0,
const int64_t task_status = share::ObDDLTaskStatus::PREPARE);
int init(const ObDDLTaskRecord &task_record);
virtual ~ObPartitionSplitTask();
virtual int process() override;
virtual int collect_longops_stat(share::ObLongopsValue &value) override;
virtual bool support_longops_monitoring() const { return true; }
int update_complete_sstable_job_status(
const ObTabletID &tablet_id,
const ObAddr &svr,
const int64_t execution_id,
const int ret_code,
const ObDDLTaskInfo &addition_info);
virtual int serialize_params_to_message(
char *buf,
const int64_t buf_size,
int64_t &pos) const override;
virtual int deserialize_params_from_message(
const uint64_t tenant_id,
const char *buf,
const int64_t buf_size,
int64_t &pos) override;
virtual int64_t get_serialize_param_size() const override;
virtual bool is_error_need_retry(const int ret_code) override
{
UNUSED(ret_code);
// we should always retry on partition split task
return true;
}
int get_src_tablet_ids(ObIArray<ObTabletID> &src_ids);
int get_dest_tablet_ids(ObIArray<ObTabletID> &dest_ids);
INHERIT_TO_STRING_KV("ObDDLTask", ObDDLTask,
K(partition_split_arg_), K(has_synced_stats_info_),
K(replica_build_task_submit_), K(replica_build_request_time_),
K(replica_build_ret_code_), K(all_src_tablet_ids_),
K(data_tablet_compaction_scn_), K(index_tablet_compaction_scns_),
K(lob_tablet_compaction_scns_), K(freeze_progress_status_inited_),
K(compact_progress_status_inited_), K(write_split_log_status_inited_),
K_(data_tablet_parallel_rowkey_list), K_(index_tablet_parallel_rowkey_list));
protected:
virtual void clear_old_status_context() override;
private:
int prepare(const share::ObDDLTaskStatus next_task_status);
int wait_freeze_end(const share::ObDDLTaskStatus next_task_status);
int wait_compaction_end(const share::ObDDLTaskStatus next_task_status);
int write_split_start_log(const share::ObDDLTaskStatus next_task_status);
int wait_data_tablet_split_end(const share::ObDDLTaskStatus next_task_status);
int wait_local_index_tablet_split_end(const share::ObDDLTaskStatus next_task_status);
int wait_lob_tablet_split_end(const share::ObDDLTaskStatus next_task_status);
int wait_trans_end(const share::ObDDLTaskStatus next_task_status);
int delete_stat_info(common::ObMySQLTransaction &trans,
const char *table_name,
const uint64_t table_id,
const uint64_t src_part_id);
int delete_src_part_stat_info(const uint64_t table_id,
const int64_t src_part_id,
const ObIArray<uint64_t> &local_index_table_ids,
const ObIArray<int64_t> &src_local_index_part_ids);
int take_effect(const share::ObDDLTaskStatus next_task_status);
int succ();
int wait_recovery_task_finish(const share::ObDDLTaskStatus next_task_status);
virtual int cleanup_impl() override;
virtual int clean_splitted_tablet();
int check_health();
int setup_src_tablet_ids_array();
int init_freeze_progress_map();
int init_compaction_scn_map();
int init_tablet_compaction_scn_array();
int update_tablet_compaction_scn_array();
int restore_compaction_scn_map();
int serialize_compaction_scn_to_task_record();
int init_data_complement_progress_map();
int init_send_finish_map();
int get_all_dest_tablet_ids(
const ObTabletID &source_tablet_id,
ObArray<ObTabletID> &dest_ids);
int setup_split_finish_items(
ObAddr &leader_addr,
ObIArray<obrpc::ObTabletSplitArg> &split_info_array);
int setup_lob_idxs_arr(ObSArray<uint64_t> &lob_col_idxs_);
int check_freeze_progress(
const ObSArray<ObTabletID> &tablet_ids,
bool &is_end);
int check_compaction_progress(
const share::ObLSID &ls_id,
const ObTabletID &tablet_id,
const ObIArray<ObAddr> &split_replica_addrs,
bool &is_end);
int send_split_request(
const ObPartitionSplitReplicaType replica_type);
int check_split_finished(bool &is_end);
int check_local_index_checksum();
int send_split_rpc(const bool is_split_start);
bool check_need_sync_stats();
int sync_stats_info();
void reset_replica_build_stat();
int init_sync_stats_info(const ObTableSchema* const table_schema,
ObSchemaGetterGuard &schema_guard,
int64_t &src_partition_id, /* OUTPUT */
ObSArray<int64_t> &src_local_index_partition_ids /* OUTPUT */);
int update_message_row_progress_(const oceanbase::share::ObDDLTaskStatus status,
const bool task_submitted,
int64_t &pos);
int update_message_tablet_progress_(const oceanbase::share::ObDDLTaskStatus status,
int64_t &pos);
int get_waiting_tablet_ids_(const hash::ObHashMap<ObCheckProgressKey<common::ObAddr>, ObCheckProgressStatus> &tablet_hash,
ObIArray<uint64_t> &waiting_tablets /* OUT */);
int get_waiting_tablet_ids_(const hash::ObHashMap<ObCheckProgressKey<uint64_t>, ObCheckProgressStatus> &tablet_hash,
ObIArray<uint64_t> &waiting_tablets /* OUT */);
int batch_insert_reorganize_history();
int check_can_reuse_macro_block(
ObSchemaGetterGuard &schema_guard,
const uint64_t tenant_id,
const ObIArray<uint64_t> &table_ids,
ObSArray<bool> &can_reuse_macro_blocks);
int check_src_tablet_exist(
const uint64_t tenant_id,
const int64_t table_id,
const ObTabletID &src_tablet_id,
bool &is_src_tablet_exist);
int prepare_tablet_split_ranges(
ObSEArray<ObSEArray<blocksstable::ObDatumRowkey, 8>, 8> &parallel_datum_rowkey_list);
int prepare_tablet_split_infos(
const share::ObLSID &ls_id,
const ObAddr &leader_addr,
ObIArray<obrpc::ObTabletSplitArg> &split_info_array);
int update_task_message();
private:
static const int64_t OB_PARTITION_SPLIT_TASK_VERSION = 1;
using ObDDLTask::is_inited_;
using ObDDLTask::task_status_;
using ObDDLTask::snapshot_version_;
using ObDDLTask::tenant_id_;
using ObDDLTask::dst_tenant_id_;
using ObDDLTask::object_id_;
using ObDDLTask::schema_version_;
using ObDDLTask::dst_schema_version_;
using ObDDLTask::task_type_;
ObRootService *root_service_;
obrpc::ObPartitionSplitArg partition_split_arg_;
bool has_synced_stats_info_;
bool replica_build_task_submit_;
int64_t replica_build_request_time_;
int64_t replica_build_ret_code_;
ObSArray<ObTabletID> all_src_tablet_ids_; // src data tablet, src local index tablet, src lob tablet
int64_t data_tablet_compaction_scn_;
ObSArray<int64_t> index_tablet_compaction_scns_;
ObSArray<int64_t> lob_tablet_compaction_scns_;
hash::ObHashMap<ObTabletID, int64_t> tablet_compaction_scn_map_;
hash::ObHashMap<ObCheckProgressKey<uint64_t>, ObCheckProgressStatus> freeze_progress_map_;
hash::ObHashMap<ObCheckProgressKey<common::ObAddr>, ObCheckProgressStatus> compaction_progress_map_;
hash::ObHashMap<ObCheckProgressKey<uint64_t>, ObCheckProgressStatus> send_finish_map_;
bool freeze_progress_status_inited_;
bool compact_progress_status_inited_;
bool write_split_log_status_inited_;
ObDDLReplicaBuildExecutor replica_builder_;
ObDDLWaitTransEndCtx wait_trans_ctx_;
int64_t tablet_size_;
common::ObSEArray<blocksstable::ObDatumRowkey, 8> data_tablet_parallel_rowkey_list_; // data table
common::ObSEArray<common::ObSEArray<blocksstable::ObDatumRowkey, 8>, 8> index_tablet_parallel_rowkey_list_; // index table.
};
} // end namespace rootserver
} // end namespace oceanbase
#endif // OCEANBASE_ROOTSERVER_OB_PARTITION_SPLIT_TASK_H

11
src/rootserver/ddl_task/ob_recover_restore_table_task.cpp
View File

@ -131,6 +131,7 @@ int ObRecoverRestoreTableTask::obtain_snapshot(const ObDDLTaskStatus next_task_s
// update sstable complement status for all leaders
int ObRecoverRestoreTableTask::update_complete_sstable_job_status(const common::ObTabletID &tablet_id,
const ObAddr &addr,
const int64_t snapshot_version,
const int64_t execution_id,
const int ret_code,
@ -147,10 +148,12 @@ int ObRecoverRestoreTableTask::update_complete_sstable_job_status(const common::
LOG_WARN("snapshot version not match", K(ret), K(snapshot_version), K(snapshot_version_));
} else if (execution_id < execution_id_) {
LOG_INFO("receive a mismatch execution result, ignore", K(ret_code), K(execution_id), K(execution_id_));
} else if (OB_FAIL(replica_builder_.set_partition_task_status(tablet_id,
ret_code,
addition_info.row_scanned_,
addition_info.row_inserted_))) {
} else if (OB_FAIL(replica_builder_.update_build_progress(tablet_id,
addr,
ret_code,
addition_info.row_scanned_,
addition_info.row_inserted_,
addition_info.physical_row_count_))) {
LOG_WARN("fail to set partition task status", K(ret));
}
return ret;

1
src/rootserver/ddl_task/ob_recover_restore_table_task.h
View File

@ -47,6 +47,7 @@ public:
virtual int process() override;
virtual int update_complete_sstable_job_status(
const common::ObTabletID &tablet_id,
const ObAddr &addr,
const int64_t snapshot_version,
const int64_t execution_id,
const int ret_code,

12
src/rootserver/ddl_task/ob_table_redefinition_task.cpp
View File

@ -196,6 +196,7 @@ int ObTableRedefinitionTask::init(const ObDDLTaskRecord &task_record)
}
int ObTableRedefinitionTask::update_complete_sstable_job_status(const common::ObTabletID &tablet_id,
const ObAddr &addr,
const int64_t snapshot_version,
const int64_t execution_id,
const int ret_code,
@ -218,20 +219,20 @@ int ObTableRedefinitionTask::update_complete_sstable_job_status(const common::Ob
case ObDDLType::DDL_DIRECT_LOAD_INSERT: {
complete_sstable_job_ret_code_ = ret_code;
ret_code_ = ret_code;
LOG_INFO("table redefinition task callback", K(complete_sstable_job_ret_code_));
LOG_INFO("table redefinition task callback", K(addr), K(complete_sstable_job_ret_code_));
break;
}
default : {
if (OB_UNLIKELY(snapshot_version_ != snapshot_version)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("error unexpected, snapshot version is not equal", K(ret), K(snapshot_version_), K(snapshot_version));
LOG_WARN("error unexpected, snapshot version is not equal", K(addr), K(ret), K(snapshot_version_), K(snapshot_version));
} else if (execution_id < execution_id_) {
ret = OB_TASK_EXPIRED;
LOG_WARN("receive a mismatch execution result, ignore", K(ret_code), K(execution_id), K(execution_id_));
LOG_WARN("receive a mismatch execution result, ignore", K(addr), K(ret_code), K(execution_id), K(execution_id_));
} else {
complete_sstable_job_ret_code_ = ret_code;
execution_id_ = execution_id; // update ObTableRedefinitionTask::execution_id_ from ObDDLRedefinitionSSTableBuildTask::execution_id_
LOG_INFO("table redefinition task callback", K(complete_sstable_job_ret_code_), K(execution_id_));
LOG_INFO("table redefinition task callback", K(addr), K(complete_sstable_job_ret_code_), K(execution_id_));
}
break;
}
@ -515,12 +516,13 @@ int ObTableRedefinitionTask::copy_table_indexes()
}
LOG_INFO("indexes schema are already built", K(index_ids));
} else {
// if there is no indexes in new tables, we need to rebuild indexes in new table
int64_t ddl_rpc_timeout = 0;
int64_t all_tablet_count = 0;
ObSchemaGetterGuard orig_schema_guard;
if (OB_FAIL(root_service->get_ddl_service().get_tenant_schema_guard_with_version_in_inner_table(tenant_id_, orig_schema_guard))) {
LOG_WARN("get schema guard failed", K(ret), K(tenant_id_));
} else if (OB_FAIL(generate_rebuild_index_arg_list(tenant_id_, object_id_, orig_schema_guard, alter_table_arg_))) {
LOG_WARN("fail to generate rebuild index arg list", K(ret), K(tenant_id_), K(object_id_));
} else if (OB_FAIL(get_orig_all_index_tablet_count(orig_schema_guard, all_tablet_count))) {
LOG_WARN("get all tablet count failed", K(ret));
} else if (OB_FAIL(ObDDLUtil::get_ddl_rpc_timeout(all_tablet_count, ddl_rpc_timeout))) {

1
src/rootserver/ddl_task/ob_table_redefinition_task.h
View File

@ -49,6 +49,7 @@ public:
virtual int process() override;
virtual int update_complete_sstable_job_status(
const common::ObTabletID &tablet_id,
const ObAddr &addr,
const int64_t snapshot_version,
const int64_t execution_id,
const int ret_code,

8
src/rootserver/ob_ddl_help.cpp
View File

@ -102,6 +102,10 @@ int ObTableGroupHelp::add_tables_to_tablegroup(ObMySQLTransaction &trans,
ret = OB_NOT_SUPPORTED;
LOG_WARN("alter tablegroup of materialized view log is not supported", KR(ret));
LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter tablegroup of materialized view log is");
} else if (table_schema->is_auto_partitioned_table()) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("not support to add auto-partitioned table to tablegroup", KR(ret), K(arg), KPC(table_schema));
LOG_USER_ERROR(OB_NOT_SUPPORTED, "add auto-partitioned table to tablegroup");
} else if (table_schema->is_external_table()) {
ret = OB_NOT_SUPPORTED;
LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter tablegroup of external table is");
@ -225,6 +229,10 @@ int ObTableGroupHelp::check_table_partition_in_tablegroup(const ObTableSchema *f
} else if (is_sys_tablegroup_id(tablegroup_id)) {
ret = OB_OP_NOT_ALLOW;
LOG_WARN("can not handle with sys tablegroup", KR(ret), K(tablegroup_id));
} else if (table.is_auto_partitioned_table()) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("not support to add auto-partitioned table to tablegroup", KR(ret), K(table));
LOG_USER_ERROR(OB_NOT_SUPPORTED, "add auto-partitioned table to tablegroup");
} else if (OB_FAIL(schema_guard.get_tablegroup_schema(tenant_id, tablegroup_id, tablegroup))) {
LOG_WARN("fail to get tablegroup schema", KR(ret), K(tenant_id), KT(tablegroup_id));
} else if (OB_ISNULL(tablegroup)) {

252
src/rootserver/ob_ddl_operator.cpp
View File

@ -2577,6 +2577,120 @@ int ObDDLOperator::update_boundary_schema_version(const uint64_t &tenant_id,
return ret;
}
int ObDDLOperator::inc_table_schema_version(ObMySQLTransaction &trans,
const uint64_t tenant_id,
const uint64_t table_id)
{
int ret = OB_SUCCESS;
ObSchemaService *schema_service = schema_service_.get_schema_service();
if (OB_INVALID_ID == table_id || OB_INVALID_ID == tenant_id) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(table_id), K(tenant_id));
} else if (OB_ISNULL(schema_service)) {
ret = OB_ERR_SYS;
LOG_WARN("schema service should not be NULL", K(ret));
} else if (OB_FAIL(schema_service->get_table_sql_service().
update_data_table_schema_version(trans,
tenant_id,
table_id,
false))) {
LOG_WARN("fail to update schema version", K(ret));
}
return ret;
}
// split_table_partitions() will modify __all_part and __all_table:
// 1. add split partitions' information into __all_part based on inc_table_schema
// which records the setting of split partitions
// 2. modify part_idx, partition_type of origin partition in __all_part
// based on upd_table_schema which records the setting of changed origin partitions
// 3. modify part_num of table in __all_table based on new_table_schema
// which records the setting of table
int ObDDLOperator::split_table_partitions(const ObTableSchema &orig_table_schema,
ObTableSchema &inc_table_schema,
ObTableSchema &new_table_schema,
ObTableSchema &upd_table_schema,
ObMySQLTransaction &trans)
{
int ret = OB_SUCCESS;
const uint64_t tenant_id = new_table_schema.get_tenant_id();
int64_t new_schema_version = OB_INVALID_VERSION;
ObSchemaService *schema_service = schema_service_.get_schema_service();
if (OB_ISNULL(schema_service)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("schema_service is NULL", KR(ret));
} else if (OB_FAIL(schema_service_.gen_new_schema_version(tenant_id, new_schema_version))) {
LOG_WARN("fail to gen new schema_version", KR(ret), K(tenant_id));
} else if (OB_FAIL(schema_service->get_table_sql_service().add_split_inc_part_info(trans,
orig_table_schema,
inc_table_schema,
new_schema_version))) {
LOG_WARN("add split inc part info failed", KR(ret),
K(new_table_schema),
K(inc_table_schema),
K(new_schema_version));
} else if (orig_table_schema.is_partitioned_table()) {
if (OB_FAIL(schema_service->get_table_sql_service().update_part_info(trans,
orig_table_schema,
upd_table_schema,
new_schema_version))) {
LOG_WARN("update split part info failed", KR(ret),
K(orig_table_schema),
K(upd_table_schema),
K(new_schema_version));
}
} else { // !orig_table_schema.is_partitioned_table()
if (orig_table_schema.is_user_table() || orig_table_schema.is_global_index_table()) {
const ObRowkeyInfo& partition_key_info = new_table_schema.get_partition_key_info();
if (partition_key_info.get_size() == 0) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid partition key numbers", KR(ret), K(orig_table_schema), K(new_table_schema));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < partition_key_info.get_size(); i++) {
const ObRowkeyColumn* partition_key_column = partition_key_info.get_column(i);
if (OB_ISNULL(partition_key_column)) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("partition_key_column is null", KR(ret), K(new_table_schema));
} else {
ObColumnSchemaV2* column_schema = new_table_schema.get_column_schema(
partition_key_column->column_id_);
if (OB_ISNULL(column_schema)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null", KR(ret), K(orig_table_schema), K(new_table_schema),
KPC(partition_key_column));
} else if (FALSE_IT(column_schema->set_schema_version(new_schema_version))) {
} else if (OB_FAIL(schema_service->get_table_sql_service()
.update_single_column(trans, orig_table_schema, new_table_schema,
*column_schema, false /* record_ddl_operation */))) {
LOG_WARN("update single column failed", KR(ret), K(orig_table_schema),
K(new_table_schema), KPC(column_schema));
}
}
} // end for
}
} else if (!orig_table_schema.is_index_local_storage() &&
!orig_table_schema.is_aux_lob_table()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("not supported table type", KR(ret), K(orig_table_schema));
} else {
// do nothing
}
}
if (OB_SUCC(ret)){
new_table_schema.set_schema_version(new_schema_version);
if (OB_FAIL(schema_service->get_table_sql_service()
.update_splitting_partition_option(trans, new_table_schema))) {
LOG_WARN("update splitting partition option failed", KR(ret), K(new_table_schema));
}
}
return ret;
}
int ObDDLOperator::truncate_table_partitions(const share::schema::ObTableSchema &orig_table_schema,
share::schema::ObTableSchema &inc_table_schema,
share::schema::ObTableSchema &del_table_schema,
@ -2862,6 +2976,40 @@ int ObDDLOperator::drop_table_partitions(const ObTableSchema &orig_table_schema,
return ret;
}
int ObDDLOperator::drop_table_splitted_partitions(const ObTableSchema &orig_table_schema,
ObTableSchema &inc_table_schema,
ObMySQLTransaction &trans)
{
int ret = OB_SUCCESS;
bool is_truncate_table = false;
bool is_truncate_partition = true; // in order to avoid logging operation in drop_inc_part_info()
const uint64_t tenant_id = orig_table_schema.get_tenant_id();
int64_t new_schema_version = OB_INVALID_VERSION;
ObSchemaService *schema_service = schema_service_.get_schema_service();
if (OB_ISNULL(schema_service)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("schema_service is NULL", K(ret));
} else if (OB_FAIL(schema_service_.gen_new_schema_version(tenant_id, new_schema_version))) {
LOG_WARN("fail to gen new schema_version", K(ret), K(tenant_id));
} else if (OB_FAIL(schema_service->get_table_sql_service().drop_inc_part_info(trans,
orig_table_schema,
inc_table_schema,
new_schema_version,
is_truncate_partition,
is_truncate_table))) {
LOG_WARN("delete inc part info failed", K(ret));
} else if (OB_FAIL(schema_service->get_table_sql_service().
update_data_table_schema_version(trans,
tenant_id,
orig_table_schema.get_table_id(),
false,
new_schema_version))) {
LOG_WARN("fail to update schema version", K(ret));
}
return ret;
}
int ObDDLOperator::drop_table_subpartitions(const ObTableSchema &orig_table_schema,
ObTableSchema &inc_table_schema,
ObTableSchema &new_table_schema,
@ -4421,6 +4569,87 @@ int ObDDLOperator::update_index_status(
return ret;
}
// "alter table ... partition by" clause need to call this function to modify index type
// when enable auto partitioning feature for non-partitioned table and it has global local index
int ObDDLOperator::update_index_type(const ObTableSchema &data_table_schema,
const uint64_t index_table_id,
const share::schema::ObIndexType index_type,
const common::ObString *ddl_stmt_str,
common::ObMySQLTransaction &trans)
{
int ret = OB_SUCCESS;
ObArray<uint64_t> index_table_ids;
ObArray<ObIndexType> index_types;
if (!data_table_schema.is_valid() ||
OB_INVALID_ID == index_table_id || index_type >= INDEX_TYPE_MAX) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument",
K(ret), K(data_table_schema), K(index_table_id), K(INDEX_TYPE_MAX));
} else if (OB_FAIL(index_table_ids.push_back(index_table_id))) {
LOG_WARN("fail to push back", K(ret), K(index_table_id));
} else if (OB_FAIL(index_types.push_back(index_type))) {
LOG_WARN("fail to push back", K(ret), K(index_type));
} else if (OB_FAIL(update_indexes_type(data_table_schema,
index_table_ids, index_types,
ddl_stmt_str,
trans))) {
LOG_WARN("update index type failed", K(ret), K(data_table_schema));
}
return ret;
}
int ObDDLOperator::update_indexes_type(const ObTableSchema &data_table_schema,
const ObIArray<uint64_t> &index_table_ids,
const ObIArray<ObIndexType> &index_types,
const common::ObString *ddl_stmt_str,
common::ObMySQLTransaction &trans)
{
int ret = OB_SUCCESS;
int64_t new_schema_version = OB_INVALID_VERSION;
ObSchemaService *schema_service = schema_service_.get_schema_service();
uint64_t tenant_id = data_table_schema.get_tenant_id();
uint64_t data_table_id = data_table_schema.get_table_id();
if (OB_INVALID_TENANT_ID == tenant_id || OB_INVALID_ID == data_table_id) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(tenant_id), K(data_table_id));
} else if (index_table_ids.count() != index_types.count()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(index_table_ids), K(index_types), K(data_table_schema));
} else if (OB_ISNULL(schema_service)) {
ret = OB_ERR_SYS;
LOG_WARN("schema service should not be NULL", K(ret));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < index_table_ids.count(); i++) {
uint64_t index_table_id = index_table_ids.at(i);
ObIndexType index_type = index_types.at(i);
if (OB_INVALID_ID == data_table_id || OB_INVALID_ID == index_table_id
|| index_type <= INDEX_TYPE_IS_NOT || index_type >= INDEX_TYPE_MAX) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(data_table_id), K(index_table_id), K(index_type));
} else if (OB_FAIL(schema_service_.gen_new_schema_version(tenant_id, new_schema_version))) {
LOG_WARN("fail to gen new schema_version", K(ret), K(tenant_id));
} else if (OB_FAIL(schema_service->get_table_sql_service().update_index_type(
data_table_schema, index_table_id,
index_type, new_schema_version,
ddl_stmt_str,
trans))) {
LOG_WARN("update index type failed", K(ret), K(data_table_schema));
}
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(schema_service->get_table_sql_service().update_data_table_schema_version(
trans, tenant_id,
data_table_id,
data_table_schema.get_in_offline_ddl_white_list()))) {
LOG_WARN("update data table schema version failed", K(ret));
}
return ret;
}
int ObDDLOperator::update_table_attribute(ObTableSchema &new_table_schema,
common::ObMySQLTransaction &trans,
const ObSchemaOperationType operation_type,
@ -4610,6 +4839,29 @@ int ObDDLOperator::update_partition_option(common::ObMySQLTransaction &trans,
return ret;
}
int ObDDLOperator::update_partition_option(common::ObMySQLTransaction &trans,
ObTableSchema &table_schema,
const ObString &ddl_stmt_str)
{
int ret = OB_SUCCESS;
const uint64_t tenant_id = table_schema.get_tenant_id();
int64_t new_schema_version = OB_INVALID_VERSION;
ObSchemaService *schema_service_impl = schema_service_.get_schema_service();
if (OB_ISNULL(schema_service_impl)) {
ret = OB_ERR_SYS;
RS_LOG(WARN, "schema_service_impl must not null", K(ret));
} else if (OB_FAIL(schema_service_.gen_new_schema_version(tenant_id, new_schema_version))) {
LOG_WARN("fail to gen new schema_version", K(ret), K(tenant_id));
} else {
table_schema.set_schema_version(new_schema_version);
if (OB_FAIL(schema_service_impl->get_table_sql_service().update_partition_option(
trans, table_schema, &ddl_stmt_str))) {
RS_LOG(WARN, "failed to update partition option", K(table_schema), K(ret));
}
}
return ret;
}
int ObDDLOperator::update_check_constraint_state(common::ObMySQLTransaction &trans,
const ObTableSchema &table_schema,
ObConstraint &cst)

24
src/rootserver/ob_ddl_operator.h
View File

@ -259,6 +259,9 @@ public:
int update_boundary_schema_version(const uint64_t &tenant_id,
const uint64_t &boundary_schema_version,
common::ObMySQLTransaction &trans);
int inc_table_schema_version(ObMySQLTransaction &trans,
const uint64_t tenant_id,
const uint64_t table_id);
int truncate_table_partitions(const share::schema::ObTableSchema &orig_table_schema,
share::schema::ObTableSchema &inc_table_schema,
share::schema::ObTableSchema &del_table_schema,
@ -275,6 +278,14 @@ public:
share::schema::ObTableSchema &inc_table_schema,
share::schema::ObTableSchema &new_table_schema,
common::ObMySQLTransaction &trans);
int split_table_partitions(const share::schema::ObTableSchema &orig_table_schema,
share::schema::ObTableSchema &inc_table_schema,
share::schema::ObTableSchema &new_table_schema,
share::schema::ObTableSchema &upd_table_schema,
common::ObMySQLTransaction &trans);
int drop_table_splitted_partitions(const share::schema::ObTableSchema &orig_table_schema,
share::schema::ObTableSchema &inc_table_schema,
common::ObMySQLTransaction &trans);
int drop_table_partitions(const share::schema::ObTableSchema &orig_table_schema,
share::schema::ObTableSchema &inc_table_schema,
share::schema::ObTableSchema &new_table_schema,
@ -552,6 +563,16 @@ public:
const bool in_offline_ddl_white_list,
common::ObMySQLTransaction &trans,
const common::ObString *ddl_stmt_str);
virtual int update_index_type(const ObTableSchema &data_table_schema,
const uint64_t index_table_id,
const share::schema::ObIndexType index_type,
const common::ObString *ddl_stmt_str,
common::ObMySQLTransaction &trans);
virtual int update_indexes_type(const ObTableSchema &data_table_schema,
const ObIArray<uint64_t> &index_table_ids,
const ObIArray<ObIndexType> &index_types,
const common::ObString *ddl_stmt_str,
common::ObMySQLTransaction &trans);
// tablespace
virtual int create_tablespace(share::schema::ObTablespaceSchema &tablespace_schema,
@ -1028,6 +1049,9 @@ public:
const ObColumnSchemaV2 &new_column_schema);
int update_partition_option(common::ObMySQLTransaction &trans,
share::schema::ObTableSchema &table_schema);
int update_partition_option(common::ObMySQLTransaction &trans,
share::schema::ObTableSchema &table_schema,
const common::ObString &ddl_stmt_str);
int update_check_constraint_state(common::ObMySQLTransaction &trans,
const share::schema::ObTableSchema &table,
share::schema::ObConstraint &cst);

1958
src/rootserver/ob_ddl_service.cpp
View File

File diff suppressed because it is too large Load Diff

75
src/rootserver/ob_ddl_service.h
View File

@ -163,6 +163,18 @@ public:
obrpc::ObAlterTableRes &res);
int clean_splitted_tablet(const obrpc::ObCleanSplittedTabletArg &arg);
int generate_splitted_schema_array(const obrpc::ObCleanSplittedTabletArg &arg,
ObArenaAllocator& allocator,
common::ObIArray<const share::schema::ObTableSchema*> &splitting_table_schemas,
common::ObIArray<share::schema::ObTableSchema*> &del_table_schemas,
int64_t &refreshed_schema_version);
int generate_splitted_schema_from_partitioned_table(
ObArenaAllocator& allocator,
const ObTabletID splitted_tablet_id,
const share::schema::ObTableSchema &splitting_table_schema,
share::schema::ObTableSchema *&del_table_schema);
int create_inner_expr_index(ObMySQLTransaction &trans,
const share::schema::ObTableSchema &orig_table_schema,
const uint64_t tenant_data_version,
@ -368,6 +380,7 @@ public:
share::schema::ObSchemaGetterGuard &schema_guard,
bool &need_redistribute_column_id);
int gen_alter_partition_new_table_schema_offline(
obrpc::ObAlterTableArg &alter_table_arg,
const share::schema::AlterTableSchema & alter_table_schema,
const share::schema::ObTableSchema &orig_table_schema,
share::schema::ObTableSchema &new_table_schema);
@ -429,12 +442,14 @@ public:
common::ObIArray<ObTableSchema*> &new_table_schemas,
common::ObIArray<AlterTableSchema*> &inc_table_schemas,
common::ObIArray<AlterTableSchema*> &del_table_schemas,
common::ObIArray<ObTableSchema*> &upd_table_schemas,
const ObTableSchema &orig_table_schema,
ObTableSchema &new_table_schema,
AlterTableSchema &inc_table_schema,
share::schema::ObSchemaGetterGuard &schema_guard,
ObArenaAllocator &allocator);
bool is_add_and_drop_partition(const obrpc::ObAlterTableArg::AlterPartitionType &op_type);
int split_global_index_partitions(obrpc::ObAlterTableArg &arg, obrpc::ObAlterTableRes &res);
// execute alter_table_partitions for some tables which are data table and its local indexes
//
// @param [in] op_type, modify part ddl op
@ -447,6 +462,7 @@ public:
common::ObIArray<ObTableSchema*> &new_table_schemas,
common::ObIArray<AlterTableSchema*> &inc_table_schemas,
common::ObIArray<AlterTableSchema*> &del_table_schemas,
ObIArray<ObTableSchema*> &upd_table_schemas,
ObDDLOperator &ddl_operator,
ObSchemaGetterGuard &schema_guard,
ObMySQLTransaction &trans);
@ -454,6 +470,7 @@ public:
const share::schema::ObTableSchema &orig_table_schema,
share::schema::AlterTableSchema &inc_table_schema,
share::schema::AlterTableSchema &del_table_schema,
ObTableSchema &upd_table_schema,
share::schema::ObTableSchema &new_table_schema,
ObDDLOperator &ddl_operator,
ObSchemaGetterGuard &schema_guard,
@ -658,6 +675,7 @@ public:
int cleanup_garbage(obrpc::ObAlterTableArg &alter_table_arg);
int modify_hidden_table_fk_state(obrpc::ObAlterTableArg &alter_table_arg);
int modify_hidden_table_not_null_column_state(const obrpc::ObAlterTableArg &alter_table_arg);
int restore_the_table_to_split_completed_state(obrpc::ObAlterTableArg &alter_table_arg);
int maintain_obj_dependency_info(const obrpc::ObDependencyObjDDLArg &arg);
int process_schema_object_dependency(
const uint64_t tenant_id,
@ -1966,6 +1984,12 @@ private:
const AlterColumnSchema &alter_column_schema,
lib::Worker::CompatMode compat_mode);
int modify_part_func_expr(const share::schema::ObTableSchema &orig_table_schema,
const share::schema::ObColumnSchemaV2 &orig_column_schema,
const share::schema::AlterColumnSchema &alter_column_schema,
share::schema::ObTableSchema &new_table_schema,
const common::ObTimeZoneInfo &tz_info,
common::ObIAllocator &allocator);
int modify_part_func_expr(const ObString &orig_column_name,
const ObString &alter_column_name,
share::schema::ObTableSchema &table_schema,
@ -2337,6 +2361,10 @@ public:
int ddl_rlock();
int ddl_wlock();
int ddl_unlock() { return ddl_lock_.unlock(); }
template<typename SCHEMA, typename ALTER_SCHEMA>
int fill_part_name(
const SCHEMA &orig_schema,
ALTER_SCHEMA &alter_schema);
private:
int generate_tenant_schema(
const obrpc::ObCreateTenantArg &arg,
@ -2669,9 +2697,6 @@ private:
const common::ObIArray<share::ObResourcePoolName> &short_pool_name_list,
common::ObIArray<share::ObResourcePoolName> &diff_pools);
template<typename SCHEMA, typename ALTER_SCHEMA>
int fill_part_name(const SCHEMA &orig_schema,
ALTER_SCHEMA &alter_schema);
template<typename SCHEMA, typename ALTER_SCHEMA>
int check_partition_name_valid(const SCHEMA &orig_schema,
const ALTER_SCHEMA &alter_schema,
const ObString part_name,
@ -2691,6 +2716,8 @@ private:
const bool is_truncate);
int check_alter_drop_subpartitions(const share::schema::ObTableSchema &orig_table_schema,
const obrpc::ObAlterTableArg &alter_table_arg);
int check_alter_split_partitions(const share::schema::ObTableSchema &orig_table_schema,
obrpc::ObAlterTableArg &alter_table_arg);
int check_alter_add_partitions(const share::schema::ObTableSchema &orig_table_schema,
obrpc::ObAlterTableArg &alter_table_arg);
int filter_out_duplicate_interval_part(const share::schema::ObTableSchema &orig_table_schema,
@ -2865,6 +2892,48 @@ private:
const share::schema::ObTenantSchema &orig_tenant_schema,
const share::schema::ObTenantSchema &new_tenant_schema);
int correct_source_tablet_id_for_inc_aux_table_schema_(
const obrpc::ObAlterTableArg::AlterPartitionType op_type,
const ObPartitionLevel target_part_level,
const ObTableSchema &table_schema,
const ObTableSchema &aux_table_schema,
const AlterTableSchema &inc_table_schema,
ObTableSchema &inc_aux_table_schema);
int generate_split_info_for_schemas_(const obrpc::ObAlterTableArg::AlterPartitionType type,
ObIArray<const ObTableSchema*>& ori_table_schemas,
ObIArray<AlterTableSchema*>& inc_table_schemas,
ObIArray<ObTableSchema*>& new_table_schemas,
ObIArray<ObTableSchema*>& upd_table_schemas);
int generate_split_info_for_schema_(const ObPartitionLevel target_part_level,
const ObTableSchema& ori_table_schema,
ObTableSchema& inc_table_schema,
ObTableSchema& new_table_schema,
ObTableSchema& upd_table_schema);
int generate_partition_info_from_non_partitioned_table_(const ObTableSchema& ori_table_schema,
ObTableSchema& inc_table_schema,
ObTableSchema& new_table_schema);
int mock_hidden_partition_for_non_partitioned_table_(const ObTableSchema& ori_table_schema,
ObTableSchema& inc_table_schema,
ObTableSchema& new_table_schema);
int generate_partition_info_from_partitioned_table_(const ObTableSchema& ori_table_schema,
ObTableSchema& inc_table_schema,
ObTableSchema& upd_table_schema);
int check_split_partition_val_(const share::schema::ObTableSchema &orig_table_schema,
const AlterTableSchema &alter_table_schema,
const ObPartitionLevel target_part_level,
const obrpc::ObAlterTableArg::AlterPartitionType type);
int check_split_partitions_from_same_source_(ObPartition **split_part_array,
const int64_t part_array_size,
const int64_t start, const int64_t end,
const share::schema::ObTableSchema &orig_table_schema,
const ObPartitionLevel target_part_level,
const obrpc::ObAlterTableArg::AlterPartitionType type);
int check_split_partition_name_(const share::schema::ObTableSchema &orig_table_schema,
const AlterTableSchema &alter_table_schema,
const ObPartitionLevel target_part_level);
int check_split_global_index_partition_(ObSchemaGetterGuard &schema_guard,
obrpc::ObAlterTableArg &arg,
const share::schema::ObTableSchema &orig_index_schema);
// this function is used for add extra tenant config init during create excepet data version
// The addition of new configuration items requires the addition or modification of related test cases to ensure their effectiveness.
int add_extra_tenant_init_config_(

124
src/rootserver/ob_index_builder.cpp
View File

@ -1215,9 +1215,6 @@ int ObIndexBuilder::do_create_index(
} else if (!arg.is_inner_ && table_schema->is_in_recyclebin()) {
ret = OB_ERR_OPERATION_ON_RECYCLE_OBJECT;
LOG_WARN("can not add index on table in recyclebin", K(ret), K(arg));
} else if (table_schema->is_in_splitting()) {
ret = OB_OP_NOT_ALLOW;
LOG_WARN("can not create index during splitting", K(ret), K(arg));
} else if (OB_FAIL(ddl_service_.check_restore_point_allow(tenant_id, *table_schema))) {
LOG_WARN("failed to check restore point allow.", K(ret), K(tenant_id), K(table_id));
} else if (table_schema->get_index_tid_count() >= OB_MAX_INDEX_PER_TABLE) {
@ -1240,8 +1237,12 @@ int ObIndexBuilder::do_create_index(
} else if (INDEX_TYPE_NORMAL_GLOBAL == arg.index_type_
|| INDEX_TYPE_UNIQUE_GLOBAL == arg.index_type_
|| INDEX_TYPE_SPATIAL_GLOBAL == arg.index_type_) {
if (!table_schema->is_partitioned_table() && !arg.index_schema_.is_partitioned_table()) {
if (!table_schema->is_partitioned_table()
&& !arg.index_schema_.is_partitioned_table()
&& !table_schema->is_auto_partitioned_table()) {
// create a global index with local storage when both the data table and index table are non-partitioned
// specifically, if the data table is auto-partitioned, we will create auto-partitioned global index rather
// than global local index.
if (OB_FAIL(do_create_local_index(schema_guard, arg, *table_schema, res))) {
LOG_WARN("fail to do create local index", K(ret));
}
@ -1462,6 +1463,18 @@ int ObIndexBuilder::generate_schema(
LOG_USER_ERROR(OB_NOT_SUPPORTED, "version is less than 4.2, functional index in mysql mode not supported");
}
}
if (OB_SUCC(ret) && data_schema.is_auto_partitioned_table()) {
if (arg.is_spatial_index()) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("not support to create spatial index for auto-partitioned table", KR(ret));
LOG_USER_ERROR(OB_NOT_SUPPORTED, "creating spatial index for auto-partitioned table is");
} else if (INDEX_TYPE_DOMAIN_CTXCAT_DEPRECATED == arg.index_type_) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("not support to create domain index for auto-partitioned table", KR(ret));
LOG_USER_ERROR(OB_NOT_SUPPORTED, "creating domain index for auto-partitioned table is");
}
}
}
if (OB_SUCC(ret)) {
@ -1478,20 +1491,22 @@ int ObIndexBuilder::generate_schema(
LOG_WARN("set_index_table_columns failed", K(arg), K(data_schema), K(ret));
} else if (OB_FAIL(set_index_table_options(arg, data_schema, schema))) {
LOG_WARN("set_index_table_options failed", K(arg), K(data_schema), K(ret));
} else if (schema.is_global_index_table() &&
OB_FAIL(set_global_index_auto_partition_infos(data_schema, schema))) {
LOG_WARN("fail to set auto partition infos", KR(ret), K(data_schema), K(schema));
} else {
if (!share::schema::is_built_in_vec_index(arg.index_type_)) {
// only delta_buffer_table set vector_index_param
schema.set_index_params(arg.index_schema_.get_index_params());
}
schema.set_name_generated_type(arg.index_schema_.get_name_generated_type());
LOG_INFO("finish generate index schema", K(schema));
LOG_INFO("finish generate index schema", K(schema), K(arg), K(need_generate_index_schema_column), K(global_index_without_column_info));
}
}
if (OB_FAIL(ret)) {
} else if (data_schema.get_part_level() > 0
&& is_index_local_storage(arg.index_type_)
&& OB_FAIL(schema.assign_partition_schema(data_schema))) {
LOG_WARN("fail to assign partition schema", K(schema), K(ret));
} else if (schema.is_index_local_storage() &&
OB_FAIL(set_local_index_partition_schema(data_schema, schema))) {
LOG_WARN("fail to assign partition schema", KR(ret), K(schema));
} else if (OB_FAIL(ddl_service_.try_format_partition_schema(schema))) {
LOG_WARN("fail to format partition schema", KR(ret), K(schema));
} else if (generate_id) {
@ -1683,6 +1698,81 @@ int ObIndexBuilder::set_basic_infos(const ObCreateIndexArg &arg,
return ret;
}
int ObIndexBuilder::set_global_index_auto_partition_infos(const share::schema::ObTableSchema &data_schema,
share::schema::ObTableSchema &schema)
{
int ret = OB_SUCCESS;
const ObPartitionOption& index_part_option = schema.get_part_option();
if (OB_UNLIKELY(!data_schema.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(data_schema), KR(ret));
} else if (OB_UNLIKELY(!schema.is_global_index_table())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid index type", K(schema), KR(ret));
} else if (data_schema.is_auto_partitioned_table()) {
// for global index, auto_part could be true only if it is valid for auto-partitioning
// and its data table enables auto-partitioning
bool enable_auto_split = true;
const int64_t auto_part_size = data_schema.get_part_option().get_auto_part_size();
if (schema.get_part_level() == PARTITION_LEVEL_ZERO) {
if (OB_UNLIKELY(!index_part_option.get_part_func_expr_str().empty())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("not allow to use auto-partition clause to"
"set presetting partition key for creating index",
KR(ret), K(schema), K(data_schema));
} else {
const ObRowkeyInfo &presetting_partition_keys = schema.get_index_info();
ObPartitionFuncType part_type = presetting_partition_keys.get_size() > 1 ?
ObPartitionFuncType::PARTITION_FUNC_TYPE_RANGE_COLUMNS :
ObPartitionFuncType::PARTITION_FUNC_TYPE_RANGE;
for (int64_t i = 0; enable_auto_split && OB_SUCC(ret) && i < presetting_partition_keys.get_size(); ++i) {
const ObRowkeyColumn *partition_column = presetting_partition_keys.get_column(i);
if (OB_ISNULL(partition_column)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("the partition column is NULL, ", KR(ret), K(i), K(presetting_partition_keys));
} else {
ObObjType type = partition_column->get_meta_type().get_type();
if (ObResolverUtils::is_partition_range_column_type(type)) {
/* case: create index idx1 on t1(c1) global, c1 is double type*/
part_type = ObPartitionFuncType::PARTITION_FUNC_TYPE_RANGE_COLUMNS;
}
enable_auto_split = ObResolverUtils::is_valid_partition_column_type(
partition_column->get_meta_type().get_type(), part_type, false);
}
}
}
} else if (schema.get_part_level() == PARTITION_LEVEL_ONE) {
if (OB_FAIL(schema.is_partition_key_match_rowkey_prefix(enable_auto_split))) {
LOG_WARN("fail to check whether matching", KR(ret));
} else if (enable_auto_split && !schema.is_valid_split_part_type()) {
enable_auto_split = false;
}
} else if (schema.get_part_level() == PARTITION_LEVEL_TWO) {
enable_auto_split = false;
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid part level", KR(ret), K(data_schema), K(schema));
}
uint64_t data_version = 0;
if (OB_FAIL(ret)) {
} else if (!enable_auto_split) {
schema.forbid_auto_partition();
} else if (OB_FAIL(GET_MIN_DATA_VERSION(data_schema.get_tenant_id(), data_version))) {
LOG_WARN("get tenant data version failed", K(ret));
} else if (data_version < DATA_VERSION_4_3_4_0){
ret = OB_NOT_SUPPORTED;
LOG_WARN("current data version doesn't support to split partition", KR(ret), K(data_version));
LOG_USER_ERROR(OB_NOT_SUPPORTED, "data version lower than 4.4 is");
} else if (OB_FAIL(schema.enable_auto_partition(auto_part_size))) {
LOG_WARN("fail to enable auto partition", KR(ret));
}
}
return ret;
}
int ObIndexBuilder::set_index_table_columns(const ObCreateIndexArg &arg,
const ObTableSchema &data_schema,
ObTableSchema &schema)
@ -1740,6 +1830,22 @@ bool ObIndexBuilder::is_final_index_status(const ObIndexStatus index_status) con
|| is_error_index_status(index_status));
}
int ObIndexBuilder::set_local_index_partition_schema(const share::schema::ObTableSchema &data_schema,
share::schema::ObTableSchema &index_schema)
{
int ret = OB_SUCCESS;
if (!index_schema.is_index_local_storage()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(index_schema));
} else if ((data_schema.is_partitioned_table() || data_schema.is_auto_partitioned_table())) {
if (OB_FAIL(index_schema.assign_partition_schema_without_auto_part_attr(data_schema))) {
LOG_WARN("fail to assign basic partition schema", KR(ret), K(index_schema));
}
}
return ret;
}
int ObIndexBuilder::check_has_none_shared_index_tables_for_fts_or_multivalue_index_(
const uint64_t tenant_id,
const uint64_t data_table_id,

5
src/rootserver/ob_index_builder.h
View File

@ -130,6 +130,8 @@ private:
int set_basic_infos(const obrpc::ObCreateIndexArg &arg,
const share::schema::ObTableSchema &data_schema,
share::schema::ObTableSchema &schema);
int set_global_index_auto_partition_infos(const share::schema::ObTableSchema &data_schema,
share::schema::ObTableSchema &schema);
int set_index_table_columns(const obrpc::ObCreateIndexArg &arg,
const share::schema::ObTableSchema &data_schema,
share::schema::ObTableSchema &schema);
@ -138,6 +140,9 @@ private:
share::schema::ObTableSchema &schema);
bool is_final_index_status(const share::schema::ObIndexStatus index_status) const;
int set_local_index_partition_schema(const share::schema::ObTableSchema &data_schema,
share::schema::ObTableSchema &index_schema);
int check_has_none_shared_index_tables_for_fts_or_multivalue_index_(
const uint64_t tenant_id,
const uint64_t data_table_id,

7
src/rootserver/ob_lob_meta_builder.cpp
View File

@ -84,8 +84,8 @@ int ObLobMetaBuilder::generate_aux_lob_meta_schema(
if (OB_FAIL(ret)) {
} else if (OB_FAIL(set_lob_table_column_store_if_need(aux_lob_meta_schema))) {
LOG_WARN("fail to set lob table column store if need", KR(ret));
} else if (data_schema.get_part_level() > 0 &&
OB_FAIL(aux_lob_meta_schema.assign_partition_schema(data_schema))) {
} else if ((data_schema.is_partitioned_table() || data_schema.is_auto_partitioned_table())
&& OB_FAIL(aux_lob_meta_schema.assign_partition_schema_without_auto_part_attr(data_schema))) {
LOG_WARN("fail to assign partition schema", K(aux_lob_meta_schema), K(ret));
} else if (need_generate_id) {
if (OB_FAIL(ddl_service_.generate_object_id_for_partition_schema(aux_lob_meta_schema))) {
@ -126,6 +126,9 @@ int ObLobMetaBuilder::set_basic_infos(
int ret = OB_SUCCESS;
aux_lob_meta_schema.set_table_type(AUX_LOB_META);
aux_lob_meta_schema.set_data_table_id(data_schema.get_table_id());
// reset tablet id
ObTabletID empty_id;
aux_lob_meta_schema.set_tablet_id(empty_id);
// reset tablet id to zero
// real tablet id will be generated by generate_tablet_id according to partition info

7
src/rootserver/ob_lob_piece_builder.cpp
View File

@ -84,8 +84,8 @@ int ObLobPieceBuilder::generate_aux_lob_piece_schema(
if (OB_FAIL(ret)) {
} else if (OB_FAIL(set_lob_table_column_store_if_need(aux_lob_piece_schema))) {
LOG_WARN("fail to set lob column store if need", KR(ret));
} else if (data_schema.get_part_level() > 0 &&
OB_FAIL(aux_lob_piece_schema.assign_partition_schema(data_schema))) {
} else if ((data_schema.is_partitioned_table() || data_schema.is_auto_partitioned_table())
&& OB_FAIL(aux_lob_piece_schema.assign_partition_schema_without_auto_part_attr(data_schema))) {
LOG_WARN("fail to assign partition schema", K(aux_lob_piece_schema), K(ret));
} else if (need_generate_id) {
if (OB_FAIL(ddl_service_.generate_object_id_for_partition_schema(aux_lob_piece_schema))) {
@ -125,6 +125,9 @@ int ObLobPieceBuilder::set_basic_infos(
{
int ret = OB_SUCCESS;
aux_lob_piece_schema.set_data_table_id(data_schema.get_table_id());
// reset tablet id
ObTabletID empty_id;
aux_lob_piece_schema.set_tablet_id(empty_id);
// reset tablet id to zero
// real tablet id will be generated by generate_tablet_id according to partition info

103
src/rootserver/ob_root_service.cpp
View File

@ -67,6 +67,7 @@
#include "share/backup/ob_backup_config.h"
#include "share/backup/ob_backup_helper.h"
#include "share/scheduler/ob_sys_task_stat.h"
#include "share/scheduler/ob_partition_auto_split_helper.h"
#include "sql/executor/ob_executor_rpc_proxy.h"
#include "sql/engine/cmd/ob_user_cmd_executor.h"
@ -915,6 +916,8 @@ int ObRootService::init(ObServerConfig &config,
FLOG_WARN("init THE_RS_JOB_TABLE failed", KR(ret));
} else if (OB_FAIL(ddl_scheduler_.init(this))) {
FLOG_WARN("init ddl task scheduler failed", KR(ret));
} else if (OB_FAIL(ObRsAutoSplitScheduler::get_instance().init())) {
FLOG_WARN("init auto split task scheduler failed", K(ret));
} else if (OB_FAIL(schema_history_recycler_.init(*schema_service_,
zone_manager_,
sql_proxy_))) {
@ -3496,7 +3499,8 @@ int ObRootService::create_table(const ObCreateTableArg &arg, ObCreateTableRes &r
//if we pass the table_schema argument, the create_index_arg can not set database_name
//and table_name, which will used from get data table schema in generate_schema
if (!index_arg.index_schema_.is_partitioned_table()
&& !table_schema.is_partitioned_table()) {
&& !table_schema.is_partitioned_table()
&& !table_schema.is_auto_partitioned_table()) {
if (INDEX_TYPE_NORMAL_GLOBAL == index_arg.index_type_) {
index_arg.index_type_ = INDEX_TYPE_NORMAL_GLOBAL_LOCAL_STORAGE;
} else if (INDEX_TYPE_UNIQUE_GLOBAL == index_arg.index_type_) {
@ -4163,7 +4167,8 @@ int ObRootService::execute_ddl_task(const obrpc::ObAlterTableArg &arg,
}
break;
}
case share::CLEANUP_GARBAGE_TASK: {
case share::CLEANUP_GARBAGE_TASK:
case share::PARTITION_SPLIT_RECOVERY_CLEANUP_GARBAGE_TASK: {
if (OB_FAIL(ddl_service_.cleanup_garbage(
const_cast<obrpc::ObAlterTableArg &>(arg)))) {
LOG_WARN("failed to cleanup garbage", K(ret));
@ -4209,6 +4214,12 @@ int ObRootService::execute_ddl_task(const obrpc::ObAlterTableArg &arg,
}
break;
}
case share::PARTITION_SPLIT_RECOVERY_TASK: {
if (OB_FAIL(ddl_service_.restore_the_table_to_split_completed_state(const_cast<ObAlterTableArg &>(arg)))) {
LOG_WARN("failed to restore the table to split completed state", K(ret));
}
break;
}
case share::SWITCH_VEC_INDEX_NAME_TASK: {
if (OB_FAIL(ddl_service_.switch_index_name_and_status_for_vec_index_table(const_cast<ObAlterTableArg &>(arg)))) {
LOG_WARN("make recovert restore task visible failed", K(ret), K(arg));
@ -5081,6 +5092,82 @@ int ObRootService::rebuild_index(const obrpc::ObRebuildIndexArg &arg, obrpc::ObA
return ret;
}
int ObRootService::send_auto_split_tablet_task_request(const obrpc::ObAutoSplitTabletBatchArg &arg,
obrpc::ObAutoSplitTabletBatchRes &res)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), K(inited_));
} else if (OB_UNLIKELY(!arg.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arg", K(ret), K(arg));
} else if (OB_FAIL(ddl_scheduler_.cache_auto_split_task(arg, res))) {
LOG_WARN("fail to cache auto split task", K(ret), K(arg), K(res));
}
return ret;
}
int ObRootService::split_global_index_tablet(const obrpc::ObAlterTableArg &arg)
{
int ret = OB_SUCCESS;
bool is_oracle_mode = false;
ObSchemaGetterGuard schema_guard;
const uint64_t tenant_id = arg.alter_table_schema_.get_tenant_id();
ObAlterTableArg &nonconst_arg = const_cast<ObAlterTableArg &>(arg);
obrpc::ObAlterTableRes res;
if (OB_UNLIKELY(!inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_UNLIKELY(!arg.is_valid()) || arg.is_add_to_scheduler_ || !arg.alter_table_schema_.is_global_index_table()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arg", K(ret), K(arg), K(arg.is_add_to_scheduler_), K(arg.alter_table_schema_.is_global_index_table()));
} else {
if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(tenant_id, schema_guard))) {
LOG_WARN("get schema guard in inner table failed", K(ret));
} else if (OB_FAIL(check_parallel_ddl_conflict(schema_guard, arg))) {
LOG_WARN("check parallel ddl conflict failed", K(ret));
} else if (OB_FAIL(table_allow_ddl_operation(arg))) {
LOG_WARN("table can't do ddl now", K(ret));
} else if (OB_FAIL(ddl_service_.split_global_index_partitions(nonconst_arg, res))) {
LOG_WARN("split global index failed", K(arg), K(ret));
}
}
char table_id_buffer[256];
snprintf(table_id_buffer, sizeof(table_id_buffer), "table_id:%ld, hidden_table_id:%ld",
arg.table_id_, arg.hidden_table_id_);
ROOTSERVICE_EVENT_ADD("ddl scheduler", "split global index",
K(tenant_id),
"ret", ret,
"trace_id", *ObCurTraceId::get_trace_id(),
"task_id", res.task_id_,
"table_id", table_id_buffer,
"schema_version", res.schema_version_);
LOG_INFO("finish split global index tablet ddl", K(ret), K(arg), K(res), "ddl_event_info", ObDDLEventInfo());
return ret;
}
int ObRootService::clean_splitted_tablet(const obrpc::ObCleanSplittedTabletArg &arg)
{
int ret = OB_SUCCESS;
uint64_t data_version = 0;
if (!inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret));
} else if (OB_FAIL(GET_MIN_DATA_VERSION(arg.tenant_id_, data_version))) {
LOG_WARN("failed to get min data version", KR(ret));
} else if (data_version < DATA_VERSION_4_3_4_0) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("current data version doesn't support to clean splitted tablet", K(ret), K(data_version));
} else if (!arg.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arg", KR(ret), K(arg));
} else if (OB_FAIL(ddl_service_.clean_splitted_tablet(arg))) {
LOG_WARN("ddl_service clean splitted tablet failed", KR(ret), K(arg));
}
return ret;
}
int ObRootService::flashback_index(const ObFlashBackIndexArg &arg) {
int ret = OB_SUCCESS;
if (!inited_) {
@ -10525,6 +10612,7 @@ int ObRootService::table_allow_ddl_operation(const obrpc::ObAlterTableArg &arg)
const ObString &origin_database_name = alter_table_schema.get_origin_database_name();
const ObString &origin_table_name = alter_table_schema.get_origin_table_name();
schema_guard.set_session_id(arg.session_id_);
bool is_index = arg.alter_table_schema_.is_index_table();
if (arg.is_refresh_sess_active_time()) {
//do nothing
} else if (!arg.is_valid()) {
@ -10533,17 +10621,12 @@ int ObRootService::table_allow_ddl_operation(const obrpc::ObAlterTableArg &arg)
} else if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(tenant_id, schema_guard))) {
LOG_WARN("get schema guard in inner table failed", K(ret));
} else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, origin_database_name,
origin_table_name, false, schema))) {
origin_table_name, is_index, schema))) {
LOG_WARN("fail to get table schema", K(ret), K(tenant_id), K(origin_database_name), K(origin_table_name));
} else if (OB_ISNULL(schema)) {
ret = OB_TABLE_NOT_EXIST;
LOG_WARN("invalid schema", K(ret));
LOG_USER_ERROR(OB_TABLE_NOT_EXIST, to_cstring(origin_database_name), to_cstring(origin_table_name));
} else if (schema->is_in_splitting()) {
//TODO ddl must not execute on splitting table due to split not unstable
ret = OB_OP_NOT_ALLOW;
LOG_WARN("table is physical or logical split can not split", K(ret), K(schema));
LOG_USER_ERROR(OB_OP_NOT_ALLOW, "table is in physial or logical split, ddl operation");
} else if (schema->is_ctas_tmp_table()) {
if (!alter_table_schema.alter_option_bitset_.has_member(ObAlterTableArg::SESSION_ID)) {
//to prevet alter table after failed to create table, the table is invisible.
@ -11619,10 +11702,10 @@ int ObRootService::drop_restore_point(const obrpc::ObDropRestorePointArg &arg)
int ObRootService::build_ddl_single_replica_response(const obrpc::ObDDLBuildSingleReplicaResponseArg &arg)
{
int ret = OB_SUCCESS;
LOG_INFO("receive build ddl single replica response", K(arg));
ObDDLTaskInfo info;
info.row_scanned_ = arg.row_scanned_;
info.row_inserted_ = arg.row_inserted_;
info.physical_row_count_ = arg.physical_row_count_;
if (OB_UNLIKELY(!inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not inited", K(ret));
@ -11632,7 +11715,7 @@ int ObRootService::build_ddl_single_replica_response(const obrpc::ObDDLBuildSing
} else if (OB_FAIL(DDL_SIM(arg.tenant_id_, arg.task_id_, PROCESS_BUILD_SSTABLE_RESPONSE_SLOW))) {
LOG_WARN("ddl sim failure: procesc build sstable response slow", K(ret));
} else if (OB_FAIL(ddl_scheduler_.on_sstable_complement_job_reply(
arg.tablet_id_/*source tablet id*/, ObDDLTaskKey(arg.dest_tenant_id_, arg.dest_schema_id_, arg.dest_schema_version_), arg.snapshot_version_, arg.execution_id_, arg.ret_code_, info))) {
arg.tablet_id_/*source tablet id*/, arg.server_addr_, ObDDLTaskKey(arg.dest_tenant_id_, arg.dest_schema_id_, arg.dest_schema_version_), arg.snapshot_version_, arg.execution_id_, arg.ret_code_, info))) {
LOG_WARN("handle column checksum calc response failed", K(ret), K(arg));
}
ROOTSERVICE_EVENT_ADD("ddl scheduler", "build ddl single replica response",

6
src/rootserver/ob_root_service.h
View File

@ -516,6 +516,8 @@ public:
int abort_redef_table(const obrpc::ObAbortRedefTableArg &arg);
int update_ddl_task_active_time(const obrpc::ObUpdateDDLTaskActiveTimeArg &arg);
int create_hidden_table(const obrpc::ObCreateHiddenTableArg &arg, obrpc::ObCreateHiddenTableRes &res);
int send_auto_split_tablet_task_request(const obrpc::ObAutoSplitTabletBatchArg &arg, obrpc::ObAutoSplitTabletBatchRes &res);
int split_global_index_tablet(const obrpc::ObAlterTableArg &arg);
/**
* For recover restore table ddl, data insert into the target table is selected from another tenant.
* The function is used to create a hidden target table without any change on the source table,
@ -545,6 +547,10 @@ public:
int rebuild_index(const obrpc::ObRebuildIndexArg &arg, obrpc::ObAlterTableRes &res);
int rebuild_vec_index(const obrpc::ObRebuildIndexArg &arg, obrpc::ObAlterTableRes &res);
int clone_tenant(const obrpc::ObCloneTenantArg &arg, obrpc::ObCloneTenantRes &res);
// the interface only for gc splitted source tablet
int clean_splitted_tablet(const obrpc::ObCleanSplittedTabletArg &arg);
//the interface only for switchover: execute skip check enable_ddl
int flashback_index(const obrpc::ObFlashBackIndexArg &arg);
int purge_index(const obrpc::ObPurgeIndexArg &arg);

Some files were not shown because too many files have changed in this diff Show More