diff --git a/mittest/simple_server/CMakeLists.txt b/mittest/simple_server/CMakeLists.txt index 1a8902eca..3e4dc00a0 100644 --- a/mittest/simple_server/CMakeLists.txt +++ b/mittest/simple_server/CMakeLists.txt @@ -103,6 +103,7 @@ ob_unittest_observer(test_arbitration_service_table_operator test_arbitration_se ob_unittest_observer(test_add_remove_replace_arbitration_service test_add_remove_replace_arbitration_service.cpp) ob_unittest_observer(test_create_tenant_with_arbitration_service test_create_tenant_with_arbitration_service.cpp) ob_unittest_observer(test_arbitration_service_replica_task_table_operator test_arbitration_service_replica_task_table_operator.cpp) +ob_unittest_observer(test_ob_parallel_migration_mode test_ob_parallel_migration_mode.cpp) ob_unittest_observer(test_change_arb_service_status test_change_arb_service_status.cpp) ob_unittest_observer(test_big_tx_data test_big_tx_data.cpp) ob_unittest_observer(test_fast_commit_report fast_commit_report.cpp) diff --git a/mittest/simple_server/test_ob_parallel_migration_mode.cpp b/mittest/simple_server/test_ob_parallel_migration_mode.cpp new file mode 100644 index 000000000..d77e67cb3 --- /dev/null +++ b/mittest/simple_server/test_ob_parallel_migration_mode.cpp @@ -0,0 +1,183 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SHARE + +#include +#include +#include "env/ob_simple_cluster_test_base.h" +#include "lib/ob_errno.h" +#include "rootserver/ob_disaster_recovery_task_mgr.h" +#include "rootserver/ob_disaster_recovery_task.h" + +namespace oceanbase +{ +using namespace unittest; +namespace share +{ +using ::testing::_; +using ::testing::Invoke; +using ::testing::Return; + +using namespace schema; +using namespace common; + +class TestParallelMigrationMode : public unittest::ObSimpleClusterTestBase +{ +public: + TestParallelMigrationMode() : unittest::ObSimpleClusterTestBase("test_ob_parallel_migration_mode") {} +}; + +TEST_F(TestParallelMigrationMode, test_parse_from_string) +{ + int ret = OB_SUCCESS; + + rootserver::ObParallelMigrationMode mode; + ObString auto_mode("auto"); + ObString on_mode("on"); + ObString off_mode("off"); + ObString not_valid_mode("onoff"); + ret = mode.parse_from_string(auto_mode); + ASSERT_EQ(true, mode.is_auto_mode()); + ASSERT_EQ(OB_SUCCESS, ret); + + mode.reset(); + ret = mode.parse_from_string(on_mode); + ASSERT_EQ(true, mode.is_on_mode()); + ASSERT_EQ(OB_SUCCESS, ret); + + mode.reset(); + ret = mode.parse_from_string(off_mode); + ASSERT_EQ(true, mode.is_off_mode()); + ASSERT_EQ(OB_SUCCESS, ret); + + mode.reset(); + ret = mode.parse_from_string(not_valid_mode); + ASSERT_EQ(OB_INVALID_ARGUMENT, ret); +} + +TEST_F(TestParallelMigrationMode, test_get_mode_str) +{ + int ret = OB_SUCCESS; + rootserver::ObParallelMigrationMode auto_mode(rootserver::ObParallelMigrationMode::ParallelMigrationMode::AUTO); + rootserver::ObParallelMigrationMode on_mode(rootserver::ObParallelMigrationMode::ParallelMigrationMode::ON); + rootserver::ObParallelMigrationMode off_mode(rootserver::ObParallelMigrationMode::ParallelMigrationMode::OFF); + rootserver::ObParallelMigrationMode max_mode(rootserver::ObParallelMigrationMode::ParallelMigrationMode::MAX); + ObString auto_str(auto_mode.get_mode_str()); + ASSERT_EQ("AUTO", std::string(auto_str.ptr(), auto_str.length())); + + ObString on_str(on_mode.get_mode_str()); + ASSERT_EQ("ON", std::string(on_str.ptr(), on_str.length())); + + ObString off_str(off_mode.get_mode_str()); + ASSERT_EQ("OFF", std::string(off_str.ptr(), off_str.length())); + + ObString max_str(max_mode.get_mode_str()); + ASSERT_EQ("", std::string(max_str.ptr(), max_str.length())); +} + +TEST_F(TestParallelMigrationMode, test_to_string) +{ + int ret = OB_SUCCESS; + rootserver::ObParallelMigrationMode auto_mode(rootserver::ObParallelMigrationMode::ParallelMigrationMode::AUTO); + rootserver::ObParallelMigrationMode on_mode(rootserver::ObParallelMigrationMode::ParallelMigrationMode::ON); + rootserver::ObParallelMigrationMode off_mode(rootserver::ObParallelMigrationMode::ParallelMigrationMode::OFF); + rootserver::ObParallelMigrationMode max_mode(rootserver::ObParallelMigrationMode::ParallelMigrationMode::MAX); + + char auto_buf[100]; + int auto_len = sizeof(auto_buf); + int pos1 = auto_mode.to_string(auto_buf, auto_len); + ASSERT_EQ("{mode:0, mode:\"AUTO\"}", std::string(auto_buf, pos1)); + + char on_buf[100]; + int on_len = sizeof(on_buf); + int pos2 = on_mode.to_string(on_buf, on_len); + ASSERT_EQ("{mode:1, mode:\"ON\"}", std::string(on_buf, pos2)); + + char off_buf[100]; + int off_len = sizeof(off_buf); + int pos3 = off_mode.to_string(off_buf, off_len); + ASSERT_EQ("{mode:2, mode:\"OFF\"}", std::string(off_buf, pos3)); + + char max_buf[100]; + int max_len = sizeof(max_buf); + int pos4 = max_mode.to_string(max_buf, max_len); + ASSERT_EQ("{mode:3, mode:null}", std::string(max_buf, pos4)); +} + +TEST_F(TestParallelMigrationMode, test_task_type) +{ + int ret = OB_SUCCESS; + ObString migrate("MIGRATE REPLICA"); + ObString add("ADD REPLICA"); + ObString build("BUILD ONLY IN MEMBER LIST"); + ObString transform("TYPE TRANSFORM"); + ObString remove_paxos("REMOVE PAXOS REPLICA"); + ObString remove_non_paxos("REMOVE NON PAXOS REPLICA"); + ObString modify("MODIFY PAXOS REPLICA NUMBER"); + ObString max("MAX_TYPE"); + ObString no_valid("NOT_VALID_TYPE"); + rootserver::ObDRTaskType task_type = rootserver::ObDRTaskType::MAX_TYPE; + ret = parse_disaster_recovery_task_type_from_string(migrate, task_type); + ASSERT_EQ(rootserver::ObDRTaskType::LS_MIGRATE_REPLICA, task_type); + ASSERT_EQ(OB_SUCCESS, ret); + task_type = rootserver::ObDRTaskType::MAX_TYPE; + + ret = parse_disaster_recovery_task_type_from_string(add, task_type); + ASSERT_EQ(rootserver::ObDRTaskType::LS_ADD_REPLICA, task_type); + ASSERT_EQ(OB_SUCCESS, ret); + task_type = rootserver::ObDRTaskType::MAX_TYPE; + + ret = parse_disaster_recovery_task_type_from_string(build, task_type); + ASSERT_EQ(rootserver::ObDRTaskType::LS_BUILD_ONLY_IN_MEMBER_LIST, task_type); + ASSERT_EQ(OB_SUCCESS, ret); + task_type = rootserver::ObDRTaskType::MAX_TYPE; + + ret = parse_disaster_recovery_task_type_from_string(transform, task_type); + ASSERT_EQ(rootserver::ObDRTaskType::LS_TYPE_TRANSFORM, task_type); + ASSERT_EQ(OB_SUCCESS, ret); + task_type = rootserver::ObDRTaskType::MAX_TYPE; + + ret = parse_disaster_recovery_task_type_from_string(remove_paxos, task_type); + ASSERT_EQ(rootserver::ObDRTaskType::LS_REMOVE_PAXOS_REPLICA, task_type); + ASSERT_EQ(OB_SUCCESS, ret); + task_type = rootserver::ObDRTaskType::MAX_TYPE; + + ret = parse_disaster_recovery_task_type_from_string(remove_non_paxos, task_type); + ASSERT_EQ(rootserver::ObDRTaskType::LS_REMOVE_NON_PAXOS_REPLICA, task_type); + ASSERT_EQ(OB_SUCCESS, ret); + task_type = rootserver::ObDRTaskType::MAX_TYPE; + + ret = parse_disaster_recovery_task_type_from_string(modify, task_type); + ASSERT_EQ(rootserver::ObDRTaskType::LS_MODIFY_PAXOS_REPLICA_NUMBER, task_type); + ASSERT_EQ(OB_SUCCESS, ret); + task_type = rootserver::ObDRTaskType::MAX_TYPE; + + ret = parse_disaster_recovery_task_type_from_string(max, task_type); + ASSERT_EQ(rootserver::ObDRTaskType::MAX_TYPE, task_type); + ASSERT_EQ(OB_SUCCESS, ret); + task_type = rootserver::ObDRTaskType::MAX_TYPE; + + ret = parse_disaster_recovery_task_type_from_string(no_valid, task_type); + ASSERT_EQ(OB_INVALID_ARGUMENT, ret); +} + +} // namespace share +} // namespace oceanbase + +int main(int argc, char **argv) +{ + oceanbase::unittest::init_log_and_gtest(argc, argv); + OB_LOGGER.set_log_level("INFO"); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/src/observer/ob_service.cpp b/src/observer/ob_service.cpp index 298ac32b7..44bb37289 100644 --- a/src/observer/ob_service.cpp +++ b/src/observer/ob_service.cpp @@ -2008,6 +2008,10 @@ int ObService::do_migrate_ls_replica(const obrpc::ObLSMigrateReplicaArg &arg) migration_op_arg.paxos_replica_number_ = arg.paxos_replica_number_; migration_op_arg.src_ = arg.src_; migration_op_arg.type_ = ObMigrationOpType::MIGRATE_LS_OP; + migration_op_arg.prioritize_same_zone_src_ = arg.prioritize_same_zone_src_; +#ifdef ERRSIM + migration_op_arg.prioritize_same_zone_src_ = GCONF.enable_parallel_migration; +#endif if (OB_FAIL(ls_service->create_ls_for_ha(arg.task_id_, migration_op_arg))) { LOG_WARN("failed to create ls for ha", KR(ret), K(arg), K(migration_op_arg)); } diff --git a/src/rootserver/ob_disaster_recovery_task.cpp b/src/rootserver/ob_disaster_recovery_task.cpp index 8cebeb2b5..ceaadf7ab 100644 --- a/src/rootserver/ob_disaster_recovery_task.cpp +++ b/src/rootserver/ob_disaster_recovery_task.cpp @@ -43,6 +43,95 @@ using namespace share::schema; namespace rootserver { +#define READ_TASK_FROM_SQL_RES_FOR_DISASTER_RECOVERY \ + uint64_t tenant_id = OB_INVALID_TENANT_ID; \ + int64_t ls_id = ObLSID::INVALID_LS_ID; \ + common::ObString task_type_str; \ + common::ObString task_id; \ + int64_t priority = 2; \ + common::ObString dest_ip; \ + int64_t dest_port = OB_INVALID_INDEX; \ + common::ObString src_ip; \ + int64_t src_port = OB_INVALID_INDEX; \ + int64_t src_paxos_replica_number = OB_INVALID_COUNT; \ + int64_t dest_paxos_replica_number = OB_INVALID_COUNT; \ + common::ObString src_type_str; \ + common::ObString dest_type_str; \ + ObReplicaType src_type_to_set = REPLICA_TYPE_INVALID; \ + ObReplicaType dest_type_to_set = REPLICA_TYPE_INVALID; \ + common::ObString execute_ip; \ + int64_t execute_port = OB_INVALID_INDEX; \ + int64_t schedule_time_us = 0; \ + int64_t generate_time_us = 0; \ + common::ObString comment; \ + int64_t data_source_port = 0; \ + common::ObString data_source_ip; \ + bool is_manual = false; \ + (void)GET_COL_IGNORE_NULL(res.get_int, "ls_id", ls_id); \ + EXTRACT_INT_FIELD_MYSQL(res, "tenant_id", tenant_id, uint64_t); \ + (void)GET_COL_IGNORE_NULL(res.get_varchar, "task_type", task_type_str); \ + (void)GET_COL_IGNORE_NULL(res.get_varchar, "task_id", task_id); \ + (void)GET_COL_IGNORE_NULL(res.get_int, "priority", priority); \ + (void)GET_COL_IGNORE_NULL(res.get_int, "source_paxos_replica_number", src_paxos_replica_number); \ + (void)GET_COL_IGNORE_NULL(res.get_int, "target_paxos_replica_number", dest_paxos_replica_number); \ + (void)GET_COL_IGNORE_NULL(res.get_varchar, "source_replica_type", src_type_str); \ + (void)GET_COL_IGNORE_NULL(res.get_varchar, "target_replica_type", dest_type_str); \ + (void)GET_COL_IGNORE_NULL(res.get_varchar, "source_replica_svr_ip", src_ip); \ + (void)GET_COL_IGNORE_NULL(res.get_int, "source_replica_svr_port", src_port); \ + (void)GET_COL_IGNORE_NULL(res.get_varchar, "target_replica_svr_ip", dest_ip); \ + (void)GET_COL_IGNORE_NULL(res.get_int, "target_replica_svr_port", dest_port); \ + (void)GET_COL_IGNORE_NULL(res.get_varchar, "task_exec_svr_ip", execute_ip); \ + (void)GET_COL_IGNORE_NULL(res.get_int, "task_exec_svr_port", execute_port); \ + { \ + ObTimeZoneInfoWrap tz_info_wrap; \ + ObTZMapWrap tz_map_wrap; \ + OZ(OTTZ_MGR.get_tenant_tz(tenant_id, tz_map_wrap)); \ + tz_info_wrap.set_tz_info_map(tz_map_wrap.get_tz_map()); \ + (void)GET_COL_IGNORE_NULL(res.get_timestamp, "generate_time", tz_info_wrap.get_time_zone_info(), generate_time_us); \ + (void)GET_COL_IGNORE_NULL(res.get_timestamp, "schedule_time", tz_info_wrap.get_time_zone_info(), schedule_time_us); \ + } \ + (void)GET_COL_IGNORE_NULL(res.get_varchar, "comment", comment); \ + EXTRACT_INT_FIELD_MYSQL_WITH_DEFAULT_VALUE(res, "data_source_svr_port", data_source_port, \ + int64_t, true/*skip null error*/, true/*skip column error*/, 0); \ + EXTRACT_VARCHAR_FIELD_MYSQL_WITH_DEFAULT_VALUE(res, "data_source_svr_ip", data_source_ip, \ + true/*skip null error*/, true/*skip column error*/, "0.0.0.0"); \ + EXTRACT_BOOL_FIELD_MYSQL_SKIP_RET(res, "is_manual", is_manual); \ + ObDRTaskKey task_key; \ + common::ObAddr execute_server_key; \ + common::ObZone execute_zone; \ + rootserver::ObDRTaskType task_type = rootserver::ObDRTaskType::MAX_TYPE; \ + ObSqlString comment_to_set; \ + ObSqlString task_id_sqlstring_format; \ + share::ObTaskId task_id_to_set; \ + rootserver::ObDRTaskPriority priority_to_set; \ + if (OB_FAIL(ret)) { \ + } else if (OB_FAIL(parse_disaster_recovery_task_type_from_string(task_type_str, task_type))) { \ + LOG_WARN("fail to parse task type", KR(ret), K(task_type_str)); \ + } else if (false == execute_server_key.set_ip_addr(execute_ip, static_cast(execute_port))) { \ + ret = OB_ERR_UNEXPECTED; \ + LOG_WARN("invalid server address", K(execute_ip), K(execute_port)); \ + } else if (OB_FAIL(SVR_TRACER.get_server_zone(execute_server_key, execute_zone))) { \ + LOG_WARN("get server zone failed", KR(ret), K(execute_server_key)); \ + } else if (OB_FAIL(task_key.init(tenant_id, share::ObLSID(ls_id), execute_zone, task_type))) { \ + LOG_WARN("fail to init a ObDRTaskKey", KR(ret), K(tenant_id), K(ls_id), K(execute_zone), K(task_type)); \ + } else if (OB_FAIL(comment_to_set.assign(comment))) { \ + LOG_WARN("fai to assign a ObString to ObSqlString", KR(ret), K(comment)); \ + } else if (OB_FAIL(task_id_sqlstring_format.assign(task_id))) { \ + LOG_WARN("fail to assign task id to ObSqlString format", KR(ret), K(task_id)); \ + } else if (OB_FAIL(task_id_to_set.set(task_id_sqlstring_format.ptr()))) { \ + LOG_WARN("fail to init a task_id", KR(ret), K(task_id_sqlstring_format)); \ + } else { \ + if (priority == 0) { \ + priority_to_set = ObDRTaskPriority::HIGH_PRI; \ + } else if (priority == 1) { \ + priority_to_set = ObDRTaskPriority::LOW_PRI; \ + } else { \ + priority_to_set = ObDRTaskPriority::MAX_PRI; \ + } \ + src_type_to_set = ObShareUtil::string_to_replica_type(src_type_str); \ + dest_type_to_set = ObShareUtil::string_to_replica_type(dest_type_str); \ + } \ + OB_SERIALIZE_MEMBER( ObDRLSReplicaTaskStatus, status_); @@ -255,89 +344,91 @@ const char *ob_disaster_recovery_task_type_strs(const rootserver::ObDRTaskType t return str; } -bool ObDRTaskKey::is_valid() const -{ - return key_type_ > ObDRTaskKeyType::INVALID - && key_type_ <= ObDRTaskKeyType::FORMAL_DR_KEY; -} - -bool ObDRTaskKey::operator==(const ObDRTaskKey &that) const -{ - return key_1_ == that.key_1_ - && key_2_ == that.key_2_ - && key_3_ == that.key_3_ - && key_4_ == that.key_4_ - && key_type_ == that.key_type_; -} - -ObDRTaskKey &ObDRTaskKey::operator=(const ObDRTaskKey &that) -{ - key_1_ = that.key_1_; - key_2_ = that.key_2_; - key_3_ = that.key_3_; - key_4_ = that.key_4_; - key_type_ = that.key_type_; - hash_value_ = that.hash_value_; - return (*this); -} - -uint64_t ObDRTaskKey::hash() const -{ - return hash_value_; -} - -int ObDRTaskKey::init( - const uint64_t key_1, - const uint64_t key_2, - const uint64_t key_3, - const uint64_t key_4, - const ObDRTaskKeyType key_type) +int parse_disaster_recovery_task_type_from_string( + const ObString &task_type_str, + rootserver::ObDRTaskType& task_type) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(key_type > ObDRTaskKeyType::FORMAL_DR_KEY - || key_type <= ObDRTaskKeyType::INVALID)) { + bool found = false; + STATIC_ASSERT(ARRAYSIZEOF(disaster_recovery_task_type_strs) == (int64_t)rootserver::ObDRTaskType::MAX_TYPE + 1, + "disaster_recovery_task_type_strs string array size mismatch enum ObDRTaskType count"); + for (int64_t i = 0; i < ARRAYSIZEOF(disaster_recovery_task_type_strs) && !found; i++) { + if (0 == task_type_str.case_compare(disaster_recovery_task_type_strs[i])) { + task_type = static_cast(i); + found = true; + } + } + if (!found) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(key_type)); - } else { - key_1_ = key_1; - key_2_ = key_2; - key_3_ = key_3; - key_4_ = key_4; - key_type_ = key_type; - hash_value_ = inner_hash(); + LOG_WARN("fail to parse task_type from string", KR(ret), K(task_type), K(task_type_str)); } return ret; } +void ObDRTaskKey::reset() +{ + tenant_id_ = OB_INVALID_TENANT_ID; + ls_id_.reset(); + task_execute_zone_.reset(); + task_type_ = ObDRTaskType::MAX_TYPE; +} + +bool ObDRTaskKey::is_valid() const +{ + return is_valid_tenant_id(tenant_id_) + && ls_id_.is_valid_with_tenant(tenant_id_) + && !task_execute_zone_.is_empty() + && ObDRTaskType::MAX_TYPE != task_type_; +} + +bool ObDRTaskKey::operator==(const ObDRTaskKey &that) const +{ + return tenant_id_ == that.tenant_id_ + && ls_id_ == that.ls_id_ + && task_execute_zone_ == that.task_execute_zone_ + && task_type_ == that.task_type_; +} + int ObDRTaskKey::init( + const uint64_t tenant_id, + const share::ObLSID &ls_id, + const common::ObZone &task_execute_zone, + const ObDRTaskType &task_type) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) + || !ls_id.is_valid_with_tenant(tenant_id) + || task_execute_zone.is_empty() + || ObDRTaskType::MAX_TYPE == task_type)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(tenant_id), K(ls_id), K(task_execute_zone), K(task_type)); + } else if (OB_FAIL(task_execute_zone_.assign(task_execute_zone))) { + LOG_WARN("task_execute_zone_ assign failed", KR(ret), K(task_execute_zone)); + } else { + tenant_id_ = tenant_id; + ls_id_ = ls_id; + task_type_ = task_type; + } + return ret; +} + +int ObDRTaskKey::assign( const ObDRTaskKey &that) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!that.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret)); + } else if (OB_FAIL(task_execute_zone_.assign(that.task_execute_zone_))) { + LOG_WARN("task_execute_zone_ assign failed", KR(ret), K(that.task_execute_zone_)); } else { - key_1_ = that.key_1_; - key_2_ = that.key_2_; - key_3_ = that.key_3_; - key_4_ = that.key_4_; - key_type_ = that.key_type_; - hash_value_ = inner_hash(); + tenant_id_ = that.tenant_id_; + ls_id_ = that.ls_id_; + task_type_ = that.task_type_; } return ret; } -uint64_t ObDRTaskKey::inner_hash() const -{ - uint64_t hash_val = 0; - hash_val = murmurhash(&key_1_, sizeof(key_1_), hash_val); - hash_val = murmurhash(&key_2_, sizeof(key_2_), hash_val); - hash_val = murmurhash(&key_3_, sizeof(key_3_), hash_val); - hash_val = murmurhash(&key_4_, sizeof(key_4_), hash_val); - hash_val = murmurhash(&key_type_, sizeof(key_type_), hash_val); - return hash_val; -} - int ObDRTask::fill_dml_splicer( share::ObDMLSqlSplicer &dml_splicer) const { @@ -418,31 +509,12 @@ int ObDRTask::set_task_key( const ObDRTaskKey &task_key) { int ret = OB_SUCCESS; - if (OB_FAIL(task_key_.init(task_key))) { + if (OB_FAIL(task_key_.assign(task_key))) { LOG_WARN("fail to init task", KR(ret), K(task_key)); } return ret; } -int ObDRTask::set_task_key( - const uint64_t key_1, - const uint64_t key_2, - const uint64_t key_3, - const uint64_t key_4, - const ObDRTaskKeyType key_type) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(task_key_.init( - key_1, - key_2, - key_3, - key_4, - key_type))) { - LOG_WARN("fail to init task", KR(ret), K(key_1), K(key_2), K(key_3), K(key_4), K(key_type)); - } - return ret; -} - void ObDRTask::set_schedule() { set_schedule_time(ObTimeUtility::current_time()); @@ -451,12 +523,10 @@ void ObDRTask::set_schedule() int ObDRTask::deep_copy(const ObDRTask &that) { int ret = OB_SUCCESS; - task_key_ = that.task_key_; tenant_id_ = that.tenant_id_; ls_id_ = that.ls_id_; cluster_id_ = that.cluster_id_; transmit_data_size_ = that.transmit_data_size_; - sibling_in_schedule_ = that.sibling_in_schedule_; invoked_source_ = that.invoked_source_; /* generated_time_ shall not be copied, * the generated_time_ is automatically set in the constructor func @@ -467,6 +537,8 @@ int ObDRTask::deep_copy(const ObDRTask &that) task_id_ = that.task_id_; if (OB_FAIL(set_comment(that.comment_.string()))) { LOG_WARN("fail to assign comment", KR(ret), K_(comment), K(that)); + } else if (OB_FAIL(task_key_.assign(that.task_key_))) { + LOG_WARN("fail to set task_key", KR(ret), K_(task_key), K(that)); } return ret; } @@ -653,7 +725,14 @@ int ObMigrateLSReplicaTask::execute( get_data_src_member(), get_paxos_replica_number(), false/*skip_change_member_list(not used)*/, - get_force_data_src_member()))) { + get_force_data_src_member(), + get_prioritize_same_zone_src()))) { + /* + During parallel migration, the same zone data source is preferred. + If multiple replica select the same server as the data source, + it will cause greater IO pressure on the corresponding server and affect normal function. + In addition, high IO pressure will also affect the speed of copying data and lose the advantage of parallel migration. + */ LOG_WARN("fail to init arg", KR(ret)); } else if (OB_FAIL(rpc_proxy.to(get_dst_server()) .by(get_tenant_id()).ls_migrate_replica(arg))) { @@ -808,6 +887,7 @@ int ObMigrateLSReplicaTask::clone( my_task->set_data_src_member(get_data_src_member()); my_task->set_force_data_src_member(get_force_data_src_member()); my_task->set_paxos_replica_number(get_paxos_replica_number()); + my_task->set_prioritize_same_zone_src(get_prioritize_same_zone_src()); output_task = my_task; } } @@ -890,6 +970,7 @@ int ObMigrateLSReplicaTask::simple_build( { int ret = OB_SUCCESS; ObDRTaskKey task_key; + common::ObZone zone; if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || !ls_id.is_valid_with_tenant(tenant_id) || !task_id.is_valid() @@ -899,9 +980,10 @@ int ObMigrateLSReplicaTask::simple_build( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id), K(task_id), K(dst_replica), K(src_member), K(paxos_replica_number)); - } else if (OB_FAIL(task_key.init(tenant_id, ls_id.id(), 0, 0, - ObDRTaskKeyType::FORMAL_DR_KEY))) { - LOG_WARN("fail to init task key", KR(ret), K(tenant_id), K(ls_id)); + } else if (OB_FAIL(SVR_TRACER.get_server_zone(dst_replica.get_server(), zone))) { + LOG_WARN("get server zone failed", KR(ret), K(dst_replica)); + } else if (OB_FAIL(task_key.init(tenant_id, ls_id, zone, ObDRTaskType::LS_MIGRATE_REPLICA))) { + LOG_WARN("fail to init task key", KR(ret), K(tenant_id), K(ls_id), K(zone)); } else if (OB_FAIL(ObDRTask::build( task_key, tenant_id, @@ -928,73 +1010,12 @@ int ObMigrateLSReplicaTask::build_task_from_sql_result( const sqlclient::ObMySQLResult &res) { int ret = OB_SUCCESS; - uint64_t tenant_id = OB_INVALID_TENANT_ID; - int64_t ls_id = ObLSID::INVALID_LS_ID; - common::ObString task_id; - int64_t priority = 2; - common::ObString src_ip; - int64_t src_port = OB_INVALID_INDEX; - common::ObString dest_ip; - int64_t dest_port = OB_INVALID_INDEX; - int64_t transmit_data_size = 0; - int64_t src_paxos_replica_number = OB_INVALID_COUNT; - int64_t schedule_time_us = 0; - int64_t generate_time_us = 0; - common::ObString comment; - int64_t data_source_port = 0; - common::ObString data_source_ip; - bool is_manual = false; - //STEP1_0: read certain members from sql result - EXTRACT_INT_FIELD_MYSQL(res, "tenant_id", tenant_id, uint64_t); - { - ObTimeZoneInfoWrap tz_info_wrap; - ObTZMapWrap tz_map_wrap; - OZ(OTTZ_MGR.get_tenant_tz(tenant_id, tz_map_wrap)); - tz_info_wrap.set_tz_info_map(tz_map_wrap.get_tz_map()); - (void)GET_COL_IGNORE_NULL(res.get_timestamp, "generate_time", tz_info_wrap.get_time_zone_info(), generate_time_us); - (void)GET_COL_IGNORE_NULL(res.get_timestamp, "schedule_time", tz_info_wrap.get_time_zone_info(), schedule_time_us); - } - (void)GET_COL_IGNORE_NULL(res.get_int, "ls_id", ls_id); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "task_id", task_id); - (void)GET_COL_IGNORE_NULL(res.get_int, "priority", priority); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "source_replica_svr_ip", src_ip); - (void)GET_COL_IGNORE_NULL(res.get_int, "source_replica_svr_port", src_port); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "target_replica_svr_ip", dest_ip); - (void)GET_COL_IGNORE_NULL(res.get_int, "target_replica_svr_port", dest_port); - (void)GET_COL_IGNORE_NULL(res.get_int, "source_paxos_replica_number", src_paxos_replica_number); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "comment", comment); - // TODO(cangming.zl): get src_replica_type and dest_replica_type from result - EXTRACT_INT_FIELD_MYSQL_WITH_DEFAULT_VALUE(res, "data_source_svr_port", data_source_port, - int64_t, true/*skip null error*/, true/*skip column error*/, 0); - EXTRACT_VARCHAR_FIELD_MYSQL_WITH_DEFAULT_VALUE(res, "data_source_svr_ip", data_source_ip, - true/*skip null error*/, true/*skip column error*/, "0.0.0.0"); - EXTRACT_BOOL_FIELD_MYSQL_SKIP_RET(res, "is_manual", is_manual); - //STEP2_0: make necessary members to build a task - ObDRTaskKey task_key; + READ_TASK_FROM_SQL_RES_FOR_DISASTER_RECOVERY common::ObAddr src_server; common::ObAddr dest_server; - rootserver::ObDRTaskPriority priority_to_set; - common::ObString zone; ObDstReplica dst_replica; - share::ObTaskId task_id_to_set; - ObSqlString comment_to_set; - ObSqlString task_id_sqlstring_format; common::ObAddr force_data_source; - if (OB_FAIL(ret)) { - } else if (OB_FAIL(comment_to_set.assign(comment))) { - LOG_WARN("fai to assign a ObString to ObSqlString", KR(ret), K(comment)); - } else if (OB_FAIL(task_id_sqlstring_format.assign(task_id))) { - LOG_WARN("fail to assign task id to ObSqlString format", KR(ret), K(task_id)); - } else if (OB_FAIL(task_id_to_set.set(task_id_sqlstring_format.ptr()))) { - LOG_WARN("fail to init a task_id", KR(ret), K(task_id_sqlstring_format)); - } else if (OB_FAIL(task_key.init( - tenant_id, - ls_id, - 0/* set to 0 */, - 0/* set to 0 */, - ObDRTaskKeyType::FORMAL_DR_KEY))) { - LOG_WARN("fail to init a ObDRTaskKey", KR(ret), K(tenant_id), K(ls_id)); } else if (false == src_server.set_ip_addr(src_ip, static_cast(src_port))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid server address", K(src_ip), K(src_port)); @@ -1004,33 +1025,17 @@ int ObMigrateLSReplicaTask::build_task_from_sql_result( } else if (false == force_data_source.set_ip_addr(data_source_ip, static_cast(data_source_port))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid server address", K(data_source_ip), K(data_source_port)); - } else if (OB_FAIL(dst_replica.assign( - 0/*unit id*/, - 0/*unit group id*/, - zone, - ObReplicaMember(dest_server, 0)))) { + } else if (OB_FAIL(dst_replica.assign(0/*unit id*/, 0/*unit group id*/, task_key.get_zone(), ObReplicaMember(dest_server, 0)))) { LOG_WARN("fail to init a ObDstReplica", KR(ret)); - } else { - //transform priority(int) -> priority_to_set(ObDRTaskPriority) - if (priority == 0) { - priority_to_set = ObDRTaskPriority::HIGH_PRI; - } else if (priority == 1) { - priority_to_set = ObDRTaskPriority::LOW_PRI; - } else { - priority_to_set = ObDRTaskPriority::MAX_PRI; - } - } - //STEP3_0: to build a task - if (OB_FAIL(ret)) { } else if (OB_FAIL(build( task_key, //(in used) - tenant_id, //(in used) - ObLSID(ls_id), //(in used) + task_key.get_tenant_id(), //(in used) + task_key.get_ls_id(), //(in used) task_id_to_set, //(in used) schedule_time_us, //(in used) generate_time_us, //(in used) GCONF.cluster_id, //(not used)cluster_id - transmit_data_size, //(not used) + 0/*transmit_data_size*/, //(not used) is_manual ? obrpc::ObAdminClearDRTaskArg::TaskType::MANUAL : obrpc::ObAdminClearDRTaskArg::TaskType::AUTO,//(not used)invoked_source priority_to_set, //(not used) comment_to_set.ptr(), //comment @@ -1401,6 +1406,7 @@ int ObAddLSReplicaTask::simple_build( { int ret = OB_SUCCESS; ObDRTaskKey task_key; + common::ObZone zone; if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || !ls_id.is_valid_with_tenant(tenant_id) || !task_id.is_valid() @@ -1410,9 +1416,10 @@ int ObAddLSReplicaTask::simple_build( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id), K(task_id), K(dst_replica), K(orig_paxos_replica_number), K(paxos_replica_number)); - } else if (OB_FAIL(task_key.init(tenant_id, ls_id.id(), 0, 0, - ObDRTaskKeyType::FORMAL_DR_KEY))) { - LOG_WARN("fail to init task key", KR(ret), K(tenant_id), K(ls_id)); + } else if (OB_FAIL(SVR_TRACER.get_server_zone(dst_replica.get_server(), zone))) { + LOG_WARN("get server zone failed", KR(ret), K(dst_replica)); + } else if (OB_FAIL(task_key.init(tenant_id, ls_id, zone, ObDRTaskType::LS_ADD_REPLICA))) { + LOG_WARN("fail to init task key", KR(ret), K(tenant_id), K(ls_id), K(zone)); } else if (OB_FAIL(ObDRTask::build( task_key, tenant_id, @@ -1440,75 +1447,12 @@ int ObAddLSReplicaTask::build_task_from_sql_result( const sqlclient::ObMySQLResult &res) { int ret = OB_SUCCESS; - uint64_t tenant_id = OB_INVALID_TENANT_ID; - int64_t ls_id = ObLSID::INVALID_LS_ID; - common::ObString task_id; - int64_t priority = 2; - common::ObString src_ip; - int64_t src_port = OB_INVALID_INDEX; - common::ObString dest_ip; - int64_t dest_port = OB_INVALID_INDEX; - int64_t transmit_data_size = 0; - int64_t src_paxos_replica_number = OB_INVALID_COUNT; - int64_t dest_paxos_replica_number = OB_INVALID_COUNT; - int64_t schedule_time_us = 0; - int64_t generate_time_us = 0; - common::ObString comment; - int64_t data_source_port = 0; - common::ObString data_source_ip; - bool is_manual = false; - //STEP1_0: read certain members from sql result - EXTRACT_INT_FIELD_MYSQL(res, "tenant_id", tenant_id, uint64_t); - { - ObTimeZoneInfoWrap tz_info_wrap; - ObTZMapWrap tz_map_wrap; - OZ(OTTZ_MGR.get_tenant_tz(tenant_id, tz_map_wrap)); - tz_info_wrap.set_tz_info_map(tz_map_wrap.get_tz_map()); - (void)GET_COL_IGNORE_NULL(res.get_timestamp, "generate_time", tz_info_wrap.get_time_zone_info(), generate_time_us); - (void)GET_COL_IGNORE_NULL(res.get_timestamp, "schedule_time", tz_info_wrap.get_time_zone_info(), schedule_time_us); - } - (void)GET_COL_IGNORE_NULL(res.get_int, "ls_id", ls_id); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "task_id", task_id); - (void)GET_COL_IGNORE_NULL(res.get_int, "priority", priority); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "source_replica_svr_ip", src_ip); - (void)GET_COL_IGNORE_NULL(res.get_int, "source_replica_svr_port", src_port); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "target_replica_svr_ip", dest_ip); - (void)GET_COL_IGNORE_NULL(res.get_int, "target_replica_svr_port", dest_port); - (void)GET_COL_IGNORE_NULL(res.get_int, "source_paxos_replica_number", src_paxos_replica_number); - (void)GET_COL_IGNORE_NULL(res.get_int, "target_paxos_replica_number", dest_paxos_replica_number); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "comment", comment); - // TODO(cangming.zl): get src_replica_type and dest_replica_type from result - EXTRACT_INT_FIELD_MYSQL_WITH_DEFAULT_VALUE(res, "data_source_svr_port", data_source_port, - int64_t, true/*skip null error*/, true/*skip column error*/, 0); - EXTRACT_VARCHAR_FIELD_MYSQL_WITH_DEFAULT_VALUE(res, "data_source_svr_ip", data_source_ip, - true/*skip null error*/, true/*skip column error*/, "0.0.0.0"); - EXTRACT_BOOL_FIELD_MYSQL_SKIP_RET(res, "is_manual", is_manual); - //STEP2_0: make necessary members to build a task - ObDRTaskKey task_key; + READ_TASK_FROM_SQL_RES_FOR_DISASTER_RECOVERY common::ObAddr src_server; common::ObAddr dest_server; - common::ObString zone; - rootserver::ObDRTaskPriority priority_to_set; ObDstReplica dst_replica; - share::ObTaskId task_id_to_set; - ObSqlString comment_to_set; - ObSqlString task_id_sqlstring_format; common::ObAddr force_data_source; - if (OB_FAIL(ret)) { - } else if (OB_FAIL(comment_to_set.assign(comment))) { - LOG_WARN("fai to assign a ObString to ObSqlString", KR(ret), K(comment)); - } else if (OB_FAIL(task_id_sqlstring_format.assign(task_id))) { - LOG_WARN("fail to assign task id to ObSqlString format", KR(ret), K(task_id)); - } else if (OB_FAIL(task_id_to_set.set(task_id_sqlstring_format.ptr()))) { - LOG_WARN("fail to init a task_id", KR(ret), K(task_id_sqlstring_format)); - } else if (OB_FAIL(task_key.init( - tenant_id, - ls_id, - 0/* set to 0 */, - 0/* set to 0 */, - ObDRTaskKeyType::FORMAL_DR_KEY))) { - LOG_WARN("fail to init a ObDRTaskKey", KR(ret), K(tenant_id), K(ls_id)); } else if (false == src_server.set_ip_addr(src_ip, static_cast(src_port))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid server address", K(src_ip), K(src_port)); @@ -1518,33 +1462,17 @@ int ObAddLSReplicaTask::build_task_from_sql_result( } else if (false == force_data_source.set_ip_addr(data_source_ip, static_cast(data_source_port))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid server address", K(data_source_ip), K(data_source_port)); - } else if (OB_FAIL(dst_replica.assign( - 0/*unit id*/, - 0/*unit group id*/, - zone, - ObReplicaMember(dest_server, 0)))) { + } else if (OB_FAIL(dst_replica.assign(0/*unit id*/, 0/*unit group id*/, task_key.get_zone(), ObReplicaMember(dest_server, 0)))) { LOG_WARN("fail to init a ObDstReplica", KR(ret)); - } else { - //transform priority(int) -> priority_to_set(ObDRTaskPriority) - if (priority == 0) { - priority_to_set = ObDRTaskPriority::HIGH_PRI; - } else if (priority == 1) { - priority_to_set = ObDRTaskPriority::LOW_PRI; - } else { - priority_to_set = ObDRTaskPriority::MAX_PRI; - } - } - //STEP3_0: to build a task - if (OB_FAIL(ret)) { } else if (OB_FAIL(build( task_key, //(in used) - tenant_id, //(in used) - ObLSID(ls_id), //(in used) + task_key.get_tenant_id(), //(in used) + task_key.get_ls_id(), //(in used) task_id_to_set, //(in used) schedule_time_us, generate_time_us, GCONF.cluster_id, //(not used)cluster_id - transmit_data_size, //(not used) + 0/*transmit_data_size*/, //(not used) is_manual ? obrpc::ObAdminClearDRTaskArg::TaskType::MANUAL : obrpc::ObAdminClearDRTaskArg::TaskType::AUTO,//(not used)invoked_source priority_to_set, //(not used) comment_to_set.ptr(), //comments @@ -1908,6 +1836,7 @@ int ObLSTypeTransformTask::simple_build( { int ret = OB_SUCCESS; ObDRTaskKey task_key; + common::ObZone zone; if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || !ls_id.is_valid_with_tenant(tenant_id) || !task_id.is_valid() @@ -1918,9 +1847,10 @@ int ObLSTypeTransformTask::simple_build( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id), K(task_id), K(dst_replica), K(src_member), K(orig_paxos_replica_number), K(paxos_replica_number)); - } else if (OB_FAIL(task_key.init(tenant_id, ls_id.id(), 0, 0, - ObDRTaskKeyType::FORMAL_DR_KEY))) { - LOG_WARN("fail to init task key", KR(ret), K(tenant_id), K(ls_id)); + } else if (OB_FAIL(SVR_TRACER.get_server_zone(dst_replica.get_server(), zone))) { + LOG_WARN("get server zone failed", KR(ret), K(dst_replica)); + } else if (OB_FAIL(task_key.init(tenant_id, ls_id, zone, ObDRTaskType::LS_TYPE_TRANSFORM))) { + LOG_WARN("fail to init task key", KR(ret), K(tenant_id), K(ls_id), K(zone)); } else if (OB_FAIL(ObDRTask::build( task_key, tenant_id, @@ -1947,73 +1877,11 @@ int ObLSTypeTransformTask::build_task_from_sql_result( const sqlclient::ObMySQLResult &res) { int ret = OB_SUCCESS; - uint64_t tenant_id = OB_INVALID_TENANT_ID; - int64_t ls_id = ObLSID::INVALID_LS_ID; - common::ObString task_id; - int64_t priority = 2; - common::ObString src_ip; - int64_t src_port = OB_INVALID_INDEX; - common::ObString dest_ip; - int64_t dest_port = OB_INVALID_INDEX; - int64_t transmit_data_size = 0; - int64_t src_paxos_replica_number = OB_INVALID_COUNT; - int64_t dest_paxos_replica_number = OB_INVALID_COUNT; - common::ObString src_type_str; - common::ObString dest_type_str; - int64_t schedule_time_us = 0; - int64_t generate_time_us = 0; - common::ObString comment; - bool is_manual = false; - //STEP1_0: read certain members from sql result - EXTRACT_INT_FIELD_MYSQL(res, "tenant_id", tenant_id, uint64_t); - { - ObTimeZoneInfoWrap tz_info_wrap; - ObTZMapWrap tz_map_wrap; - OZ(OTTZ_MGR.get_tenant_tz(tenant_id, tz_map_wrap)); - tz_info_wrap.set_tz_info_map(tz_map_wrap.get_tz_map()); - (void)GET_COL_IGNORE_NULL(res.get_timestamp, "generate_time", tz_info_wrap.get_time_zone_info(), generate_time_us); - (void)GET_COL_IGNORE_NULL(res.get_timestamp, "schedule_time", tz_info_wrap.get_time_zone_info(), schedule_time_us); - } - (void)GET_COL_IGNORE_NULL(res.get_int, "ls_id", ls_id); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "task_id", task_id); - (void)GET_COL_IGNORE_NULL(res.get_int, "priority", priority); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "source_replica_svr_ip", src_ip); - (void)GET_COL_IGNORE_NULL(res.get_int, "source_replica_svr_port", src_port); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "target_replica_svr_ip", dest_ip); - (void)GET_COL_IGNORE_NULL(res.get_int, "target_replica_svr_port", dest_port); - (void)GET_COL_IGNORE_NULL(res.get_int, "source_paxos_replica_number", src_paxos_replica_number); - (void)GET_COL_IGNORE_NULL(res.get_int, "target_paxos_replica_number", dest_paxos_replica_number); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "source_replica_type", src_type_str); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "target_replica_type", dest_type_str); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "comment", comment); - EXTRACT_BOOL_FIELD_MYSQL_SKIP_RET(res, "is_manual", is_manual); - //STEP2_0: make necessary members to build a task - ObDRTaskKey task_key; + READ_TASK_FROM_SQL_RES_FOR_DISASTER_RECOVERY common::ObAddr src_server; common::ObAddr dest_server; - common::ObString zone; - rootserver::ObDRTaskPriority priority_to_set; - ObReplicaType src_type_to_set = ObShareUtil::string_to_replica_type(src_type_str); - ObReplicaType dest_type_to_set = ObShareUtil::string_to_replica_type(dest_type_str); ObDstReplica dst_replica; - share::ObTaskId task_id_to_set; - ObSqlString comment_to_set; - ObSqlString task_id_sqlstring_format; - if (OB_FAIL(ret)) { - } else if (OB_FAIL(comment_to_set.assign(comment))) { - LOG_WARN("fai to assign a ObString to ObSqlString", KR(ret), K(comment)); - } else if (OB_FAIL(task_id_sqlstring_format.assign(task_id))) { - LOG_WARN("fail to assign task id to ObSqlString format", KR(ret), K(task_id)); - } else if (OB_FAIL(task_id_to_set.set(task_id_sqlstring_format.ptr()))) { - LOG_WARN("fail to init a task_id", KR(ret), K(task_id_sqlstring_format)); - } else if (OB_FAIL(task_key.init( - tenant_id, - ls_id, - 0/* set to 0 */, - 0/* set to 0 */, - ObDRTaskKeyType::FORMAL_DR_KEY))) { - LOG_WARN("fail to init a ObDRTaskKey", KR(ret), K(tenant_id), K(ls_id)); } else if (false == src_server.set_ip_addr(src_ip, static_cast(src_port))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid server address", K(src_ip), K(src_port)); @@ -2021,38 +1889,26 @@ int ObLSTypeTransformTask::build_task_from_sql_result( ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid server address", K(dest_ip), K(dest_port)); } else { - //transform priority(int) -> priority_to_set(ObDRTaskPriority) - if (priority == 0) { - priority_to_set = ObDRTaskPriority::HIGH_PRI; - } else if (priority == 1) { - priority_to_set = ObDRTaskPriority::LOW_PRI; - } else { - priority_to_set = ObDRTaskPriority::MAX_PRI; - } ObReplicaMember src_member; ObReplicaMember dest_member; if (OB_FAIL(src_member.init(src_server, 0, src_type_to_set))) { LOG_WARN("failed to init src_member", KR(ret), K(src_server), K(src_type_str), K(src_type_to_set)); } else if (OB_FAIL(dest_member.init(dest_server, 0, dest_type_to_set))) { LOG_WARN("failed to init dest_member", KR(ret), K(dest_server), K(dest_type_str), K(dest_type_to_set)); - } else if (OB_FAIL(dst_replica.assign( - 0/*unit id*/, - 0/*unit group id*/, - zone, - dest_member))) { - LOG_WARN("fail to init a ObDstReplica", KR(ret), K(zone), K(dest_member)); + } else if (OB_FAIL(dst_replica.assign(0/*unit id*/, 0/*unit group id*/, task_key.get_zone(), dest_member))) { + LOG_WARN("fail to init a ObDstReplica", KR(ret), K(task_key), K(dest_member)); } //STEP3_0: to build a task if (OB_FAIL(ret)) { } else if (OB_FAIL(build( task_key, //(in used) - tenant_id, //(in used) - ObLSID(ls_id), //(in used) + task_key.get_tenant_id(), //(in used) + task_key.get_ls_id(), //(in used) task_id_to_set, //(in used) schedule_time_us, generate_time_us, GCONF.cluster_id, //(not used)cluster_id - transmit_data_size, //(not used) + 0/*transmit_data_size*/, //(not used) is_manual ? obrpc::ObAdminClearDRTaskArg::TaskType::MANUAL : obrpc::ObAdminClearDRTaskArg::TaskType::AUTO,//(not used)invoked_source priority_to_set, //(not used) comment_to_set.ptr(), //comment @@ -2061,8 +1917,8 @@ int ObLSTypeTransformTask::build_task_from_sql_result( src_member, //(not used)data_src_server src_paxos_replica_number, //(in used) dest_paxos_replica_number))) { //(in used) - LOG_WARN("fail to build a ObLSTypeTransformTask", KR(ret), K(task_key), K(tenant_id), K(ls_id), - K(task_id_to_set), K(schedule_time_us), K(generate_time_us), K(transmit_data_size), + LOG_WARN("fail to build a ObLSTypeTransformTask", KR(ret), K(task_key), + K(task_id_to_set), K(schedule_time_us), K(generate_time_us), K(priority_to_set), K(dst_replica), K(src_member), K(comment_to_set), K(src_paxos_replica_number), K(dest_paxos_replica_number)); } else { @@ -2328,6 +2184,8 @@ int ObRemoveLSReplicaTask::simple_build( { int ret = OB_SUCCESS; ObDRTaskKey task_key; + ObDRTaskType task_type = ObDRTaskType::MAX_TYPE; + common::ObZone zone; if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || !ls_id.is_valid_with_tenant(tenant_id) || !task_id.is_valid() @@ -2340,9 +2198,16 @@ int ObRemoveLSReplicaTask::simple_build( LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id), K(task_id), K(leader), K(remove_server), K(orig_paxos_replica_number), K(paxos_replica_number), K(replica_type)); - } else if (OB_FAIL(task_key.init(tenant_id, ls_id.id(), 0, 0, - ObDRTaskKeyType::FORMAL_DR_KEY))) { - LOG_WARN("fail to init task key", KR(ret), K(tenant_id), K(ls_id)); + } else if (ObReplicaTypeCheck::is_paxos_replica_V2(replica_type)) { + task_type = ObDRTaskType::LS_REMOVE_PAXOS_REPLICA; + } else { + task_type = ObDRTaskType::LS_REMOVE_NON_PAXOS_REPLICA; + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(SVR_TRACER.get_server_zone(leader, zone))) { + LOG_WARN("get server zone failed", KR(ret), K(leader)); + } else if (OB_FAIL(task_key.init(tenant_id, ls_id, zone, task_type))) { + LOG_WARN("fail to init task key", KR(ret), K(tenant_id), K(ls_id), K(zone)); } else if (OB_FAIL(ObDRTask::build( task_key, tenant_id, @@ -2368,120 +2233,50 @@ int ObRemoveLSReplicaTask::build_task_from_sql_result( const sqlclient::ObMySQLResult &res) { int ret = OB_SUCCESS; - common::ObString task_type; - uint64_t tenant_id = OB_INVALID_TENANT_ID; - int64_t ls_id = ObLSID::INVALID_LS_ID; - common::ObString task_id; - int64_t priority = 2; - common::ObString dest_ip; - int64_t dest_port = OB_INVALID_INDEX; - common::ObString target_ip; - int64_t target_port = OB_INVALID_INDEX; - int64_t transmit_data_size = 0; - int64_t src_paxos_replica_number = OB_INVALID_COUNT; - int64_t dest_paxos_replica_number = OB_INVALID_COUNT; - int64_t schedule_time_us = 0; - int64_t generate_time_us = 0; - common::ObString comment; - common::ObString replica_type_str; - bool is_manual = false; - //STEP1_0: read certain members from sql result - EXTRACT_INT_FIELD_MYSQL(res, "tenant_id", tenant_id, uint64_t); - { - ObTimeZoneInfoWrap tz_info_wrap; - ObTZMapWrap tz_map_wrap; - OZ(OTTZ_MGR.get_tenant_tz(tenant_id, tz_map_wrap)); - tz_info_wrap.set_tz_info_map(tz_map_wrap.get_tz_map()); - (void)GET_COL_IGNORE_NULL(res.get_timestamp, "generate_time", tz_info_wrap.get_time_zone_info(), generate_time_us); - (void)GET_COL_IGNORE_NULL(res.get_timestamp, "schedule_time", tz_info_wrap.get_time_zone_info(), schedule_time_us); - } - (void)GET_COL_IGNORE_NULL(res.get_int, "ls_id", ls_id); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "task_id", task_id); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "task_type", task_type); - (void)GET_COL_IGNORE_NULL(res.get_int, "priority", priority); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "task_exec_svr_ip", dest_ip); - (void)GET_COL_IGNORE_NULL(res.get_int, "task_exec_svr_port", dest_port); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "target_replica_svr_ip", target_ip); - (void)GET_COL_IGNORE_NULL(res.get_int, "target_replica_svr_port", target_port); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "target_replica_type", replica_type_str); - (void)GET_COL_IGNORE_NULL(res.get_int, "source_paxos_replica_number", src_paxos_replica_number); - (void)GET_COL_IGNORE_NULL(res.get_int, "target_paxos_replica_number", dest_paxos_replica_number); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "comment", comment); - EXTRACT_BOOL_FIELD_MYSQL_SKIP_RET(res, "is_manual", is_manual); - //STEP2_0: make necessary members to build a task - ObDRTaskKey task_key; + READ_TASK_FROM_SQL_RES_FOR_DISASTER_RECOVERY common::ObAddr dest_server; - common::ObAddr target_server; - rootserver::ObDRTaskPriority priority_to_set; - share::ObTaskId task_id_to_set; - ObSqlString comment_to_set; - ObSqlString task_id_sqlstring_format; // for adding trailing null - + common::ObAddr execute_server; if (OB_FAIL(ret)) { - } else if (OB_FAIL(comment_to_set.assign(comment))) { - LOG_WARN("fai to assign a ObString to ObSqlString", KR(ret), K(comment)); - } else if (OB_FAIL(task_id_sqlstring_format.assign(task_id))) { - LOG_WARN("fail to assign task id to ObSqlString format", KR(ret), K(task_id)); - } else if (OB_FAIL(task_id_to_set.set(task_id_sqlstring_format.ptr()))) { - LOG_WARN("fail to init a task_id", KR(ret), K(task_id_sqlstring_format)); - } else if (OB_FAIL(task_key.init( - tenant_id, - ls_id, - 0/* set to 0 */, - 0/* set to 0 */, - ObDRTaskKeyType::FORMAL_DR_KEY))) { - LOG_WARN("fail to init a ObDRTaskKey", KR(ret), K(tenant_id), K(ls_id)); } else if (false == dest_server.set_ip_addr(dest_ip, static_cast(dest_port))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid server address", K(dest_ip), K(dest_port)); - } else if (false == target_server.set_ip_addr(target_ip, static_cast(target_port))) { + } else if (false == execute_server.set_ip_addr(execute_ip, static_cast(execute_port))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid server address", K(target_ip), K(target_port)); + LOG_WARN("invalid server address", K(execute_ip), K(execute_port)); } else { - //transform priority(int) -> priority_to_set(ObDRTaskPriority) - if (priority == 0) { - priority_to_set = ObDRTaskPriority::HIGH_PRI; - } else if (priority == 1) { - priority_to_set = ObDRTaskPriority::LOW_PRI; - } else { - priority_to_set = ObDRTaskPriority::MAX_PRI; - } - // get replica_type_ from replica_type_str and check whether or not match with task_type. - replica_type_ = ObShareUtil::string_to_replica_type(replica_type_str); - if (0 == task_type.case_compare(ob_disaster_recovery_task_type_strs(ObDRTaskType::LS_REMOVE_PAXOS_REPLICA))) { - if (OB_UNLIKELY(REPLICA_TYPE_FULL != replica_type_)) { + if (0 == task_type_str.case_compare(ob_disaster_recovery_task_type_strs(ObDRTaskType::LS_REMOVE_PAXOS_REPLICA))) { + if (OB_UNLIKELY(REPLICA_TYPE_FULL != dest_type_to_set)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("task_type and replica_type do not match", KR(ret), K(task_type), K(replica_type_)); + LOG_WARN("task_type and replica_type do not match", KR(ret), K(task_type_str), K(dest_type_to_set)); } - } else if (0 == task_type.case_compare(ob_disaster_recovery_task_type_strs(ObDRTaskType::LS_REMOVE_NON_PAXOS_REPLICA))) { - if (OB_UNLIKELY(! ObReplicaTypeCheck::is_non_paxos_replica(replica_type_))) { + } else if (0 == task_type_str.case_compare(ob_disaster_recovery_task_type_strs(ObDRTaskType::LS_REMOVE_NON_PAXOS_REPLICA))) { + if (OB_UNLIKELY(! ObReplicaTypeCheck::is_non_paxos_replica(dest_type_to_set))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("task_type and replica_type do not match", KR(ret), K(task_type), K(replica_type_)); + LOG_WARN("task_type and replica_type do not match", KR(ret), K(task_type_str), K(dest_type_to_set)); } } else { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected task_type", KR(ret), K(task_type)); + LOG_WARN("unexpected task_type", KR(ret), K(task_type_str)); } } - //STEP3_0: to build a task if (OB_FAIL(ret)) { } else if (OB_FAIL(build( task_key, //(in used) - tenant_id, //(in used) - ObLSID(ls_id), //(in used) + task_key.get_tenant_id(), //(in used) + task_key.get_ls_id(), //(in used) task_id_to_set, //(in used) schedule_time_us, generate_time_us, GCONF.cluster_id, //(not used)cluster_id - transmit_data_size, //(not used) + 0/*transmit_data_size*/, //(not used) is_manual ? obrpc::ObAdminClearDRTaskArg::TaskType::MANUAL : obrpc::ObAdminClearDRTaskArg::TaskType::AUTO,//(not used)invoked_source priority_to_set, //(not used) comment_to_set.ptr(), //comment - dest_server, //(in used)leader - ObReplicaMember(target_server, 0), //(in used)target_server + execute_server, //(in used)leader + ObReplicaMember(dest_server, 0), //(in used)dest_server src_paxos_replica_number, //(in used) dest_paxos_replica_number, //(in used) - replica_type_))) { //(in used) + dest_type_to_set))) { //(in used) LOG_WARN("fail to build a ObRemoveLSReplicaTask", KR(ret)); } else { LOG_INFO("success to build a ObRemoveLSReplicaTask", KPC(this), K(task_id), K(task_id_to_set), K(task_id_sqlstring_format)); @@ -2724,6 +2519,7 @@ int ObLSModifyPaxosReplicaNumberTask::simple_build( { int ret = OB_SUCCESS; ObDRTaskKey task_key; + common::ObZone zone; if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || !ls_id.is_valid_with_tenant(tenant_id) || !task_id.is_valid() @@ -2735,9 +2531,11 @@ int ObLSModifyPaxosReplicaNumberTask::simple_build( LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id), K(task_id), K(dst_server), K(orig_paxos_replica_number), K(paxos_replica_number), K(member_list)); - } else if (OB_FAIL(task_key.init(tenant_id, ls_id.id(), 0, 0, - ObDRTaskKeyType::FORMAL_DR_KEY))) { - LOG_WARN("fail to init task key", KR(ret), K(tenant_id), K(ls_id)); + } else if (OB_FAIL(SVR_TRACER.get_server_zone(dst_server, zone))) { + LOG_WARN("get server zone failed", KR(ret), K(dst_server)); + } else if (OB_FAIL(task_key.init(tenant_id, ls_id, zone, + ObDRTaskType::LS_MODIFY_PAXOS_REPLICA_NUMBER))) { + LOG_WARN("fail to init task key", KR(ret), K(tenant_id), K(ls_id), K(zone)); } else if (OB_FAIL(ObDRTask::build( task_key, tenant_id, @@ -2763,93 +2561,30 @@ int ObLSModifyPaxosReplicaNumberTask::build_task_from_sql_result( const sqlclient::ObMySQLResult &res) { int ret = OB_SUCCESS; - uint64_t tenant_id = OB_INVALID_TENANT_ID; - int64_t ls_id = ObLSID::INVALID_LS_ID; - common::ObString task_id; - int64_t priority = 2; - common::ObString dest_ip; - int64_t dest_port = OB_INVALID_INDEX; - int64_t transmit_data_size = 0; - int64_t src_paxos_replica_number = OB_INVALID_COUNT; - int64_t dest_paxos_replica_number = OB_INVALID_COUNT; - int64_t schedule_time_us = 0; - int64_t generate_time_us = 0; - common::ObString comment; - bool is_manual = false; - //STEP1_0: read certain members from sql result - EXTRACT_INT_FIELD_MYSQL(res, "tenant_id", tenant_id, uint64_t); - { - ObTimeZoneInfoWrap tz_info_wrap; - ObTZMapWrap tz_map_wrap; - OZ(OTTZ_MGR.get_tenant_tz(tenant_id, tz_map_wrap)); - tz_info_wrap.set_tz_info_map(tz_map_wrap.get_tz_map()); - (void)GET_COL_IGNORE_NULL(res.get_timestamp, "generate_time", tz_info_wrap.get_time_zone_info(), generate_time_us); - (void)GET_COL_IGNORE_NULL(res.get_timestamp, "schedule_time", tz_info_wrap.get_time_zone_info(), schedule_time_us); - } - (void)GET_COL_IGNORE_NULL(res.get_int, "ls_id", ls_id); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "task_id", task_id); - (void)GET_COL_IGNORE_NULL(res.get_int, "priority", priority); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "task_exec_svr_ip", dest_ip); - (void)GET_COL_IGNORE_NULL(res.get_int, "task_exec_svr_port", dest_port); - (void)GET_COL_IGNORE_NULL(res.get_int, "source_paxos_replica_number", src_paxos_replica_number); - (void)GET_COL_IGNORE_NULL(res.get_int, "target_paxos_replica_number", dest_paxos_replica_number); - (void)GET_COL_IGNORE_NULL(res.get_varchar, "comment", comment); - EXTRACT_BOOL_FIELD_MYSQL_SKIP_RET(res, "is_manual", is_manual); - //STEP2_0: make necessary members to build a task - ObDRTaskKey task_key; - common::ObAddr dest_server; - rootserver::ObDRTaskPriority priority_to_set; + READ_TASK_FROM_SQL_RES_FOR_DISASTER_RECOVERY + common::ObAddr execute_server; common::ObMemberList member_list; - share::ObTaskId task_id_to_set; - ObSqlString comment_to_set; - ObSqlString task_id_sqlstring_format; - if (OB_FAIL(ret)) { - } else if (OB_FAIL(comment_to_set.assign(comment))) { - LOG_WARN("fai to assign a ObString to ObSqlString", KR(ret), K(comment)); - } else if (OB_FAIL(task_id_sqlstring_format.assign(task_id))) { - LOG_WARN("fail to assign task id to ObSqlString format", KR(ret), K(task_id)); - } else if (OB_FAIL(task_id_to_set.set(task_id_sqlstring_format.ptr()))) { - LOG_WARN("fail to init a task_id", KR(ret), K(task_id_sqlstring_format)); - } else if (OB_FAIL(task_key.init( - tenant_id, - ls_id, - 0/* set to 0 */, - 0/* set to 0 */, - ObDRTaskKeyType::FORMAL_DR_KEY))) { - LOG_WARN("fail to init a ObDRTaskKey", KR(ret), K(tenant_id), K(ls_id)); - } else if (false == dest_server.set_ip_addr(dest_ip, static_cast(dest_port))) { + } else if (false == execute_server.set_ip_addr(execute_ip, static_cast(execute_port))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid server address", K(dest_ip), K(dest_port)); - } else if (OB_FAIL(member_list.add_member(ObMember(dest_server, 0)))) { + LOG_WARN("invalid server address", K(execute_ip), K(execute_port)); + } else if (OB_FAIL(member_list.add_member(ObMember(execute_server, 0)))) { // this field is not accurate and will not be used by tasks that are reloaded into memory. // it just ensures that the task build is valid. - LOG_WARN("fail to add server to member list", KR(ret), K(dest_server)); - } else { - //transform priority(int) -> priority_to_set(ObDRTaskPriority) - if (priority == 0) { - priority_to_set = ObDRTaskPriority::HIGH_PRI; - } else if (priority == 1) { - priority_to_set = ObDRTaskPriority::LOW_PRI; - } else { - priority_to_set = ObDRTaskPriority::MAX_PRI; - } - } - //STEP3_0: to build a task - if (OB_FAIL(ret)) { + LOG_WARN("fail to add server to member list", KR(ret), K(execute_server)); } else if (OB_FAIL(build( task_key, //(in used) - tenant_id, //(in used) - ObLSID(ls_id), //(in used) + task_key.get_tenant_id(), //(in used) + task_key.get_ls_id(), //(in used) task_id_to_set, //(in used) schedule_time_us, generate_time_us, GCONF.cluster_id, //(not used)cluster_id - transmit_data_size, //(not used) + 0/*transmit_data_size*/, //(not used) is_manual ? obrpc::ObAdminClearDRTaskArg::TaskType::MANUAL : obrpc::ObAdminClearDRTaskArg::TaskType::AUTO,//(not used)invoked_source priority_to_set, //(not used) comment_to_set.ptr(), //comment - dest_server, //(in used)leader + execute_server, //(in used)leader src_paxos_replica_number, //(in used) dest_paxos_replica_number, //(in used) member_list))) { diff --git a/src/rootserver/ob_disaster_recovery_task.h b/src/rootserver/ob_disaster_recovery_task.h index ca6be2628..ddb74a49c 100644 --- a/src/rootserver/ob_disaster_recovery_task.h +++ b/src/rootserver/ob_disaster_recovery_task.h @@ -127,9 +127,9 @@ enum ObDRTaskRetComment class ObDRTaskQueue; const char *ob_disaster_recovery_task_type_strs(const rootserver::ObDRTaskType type); +int parse_disaster_recovery_task_type_from_string(const ObString &task_type_str, rootserver::ObDRTaskType& task_type); const char *ob_disaster_recovery_task_priority_strs(const rootserver::ObDRTaskPriority task_priority); const char* ob_disaster_recovery_task_ret_comment_strs(const rootserver::ObDRTaskRetComment ret_comment); -const char *ob_replica_type_strs(const ObReplicaType type); bool is_manual_dr_task_data_version_match(uint64_t tenant_data_version); int build_execute_result( const int ret_code, @@ -179,50 +179,6 @@ private: }; class ObDRTaskMgr; -enum class ObDRTaskKeyType : int64_t -{ - INVALID = -1, - FORMAL_DR_KEY = 0, -}; - -class ObDRTaskKey -{ -public: - ObDRTaskKey() : key_1_(-1), - key_2_(-1), - key_3_(-1), - key_4_(-1), - key_type_(ObDRTaskKeyType::INVALID), - hash_value_(0) {} - virtual ~ObDRTaskKey() {} -public: - bool is_valid() const; - bool operator==(const ObDRTaskKey &that) const; - ObDRTaskKey &operator=(const ObDRTaskKey &that); - uint64_t hash() const; - int hash(uint64_t &hash_val) const { hash_val = hash(); return OB_SUCCESS; } - int init(const uint64_t key_1, - const uint64_t key_2, - const uint64_t key_3, - const uint64_t key_4, - const ObDRTaskKeyType key_type); - int init(const ObDRTaskKey &that); - ObDRTaskKeyType get_key_type() const { return key_type_; } - TO_STRING_KV(K_(key_1), - K_(key_2), - K_(key_3), - K_(key_4), - K_(key_type)); -private: - uint64_t inner_hash() const; -private: - uint64_t key_1_; - uint64_t key_2_; - uint64_t key_3_; - uint64_t key_4_; - ObDRTaskKeyType key_type_; - uint64_t hash_value_; -}; enum class ObDRTaskType : int64_t { @@ -236,6 +192,40 @@ enum class ObDRTaskType : int64_t MAX_TYPE, }; +class ObDRTaskKey +{ +public: + ObDRTaskKey() : tenant_id_(OB_INVALID_TENANT_ID), + ls_id_(), + task_execute_zone_(), + task_type_(ObDRTaskType::MAX_TYPE) {} + virtual ~ObDRTaskKey() {} +public: + void reset(); + bool is_valid() const; + bool operator==(const ObDRTaskKey &that) const; + int init(const uint64_t tenant_id, + const share::ObLSID &ls_id, + const common::ObZone &task_execute_zone, + const ObDRTaskType &task_type); + int assign(const ObDRTaskKey &that); + ObDRTaskKey& operator=(const ObDRTaskKey&) = delete; + TO_STRING_KV(K_(tenant_id), + K_(ls_id), + K_(task_execute_zone), + K_(task_type)); + + uint64_t get_tenant_id() const { return tenant_id_; } + const share::ObLSID &get_ls_id() const { return ls_id_; } + const common::ObZone &get_zone() const { return task_execute_zone_; } + const ObDRTaskType &get_task_type() const { return task_type_; } +private: + uint64_t tenant_id_; + share::ObLSID ls_id_; + common::ObZone task_execute_zone_; + ObDRTaskType task_type_; +}; + enum class ObDRTaskPriority : int64_t { HIGH_PRI = 0, @@ -255,7 +245,6 @@ public: ls_id_(), cluster_id_(-1), transmit_data_size_(0), - sibling_in_schedule_(false), invoked_source_(obrpc::ObAdminClearDRTaskArg::TaskType::AUTO), generate_time_(common::ObTimeUtility::current_time()), priority_(ObDRTaskPriority::MAX_PRI), @@ -332,7 +321,6 @@ public: K_(ls_id), K_(cluster_id), K_(transmit_data_size), - K_(sibling_in_schedule), K_(invoked_source), K_(generate_time), K_(priority), @@ -346,12 +334,6 @@ public: const ObDRTaskKey &get_task_key() const { return task_key_; } int set_task_key( const ObDRTaskKey &task_key); - int set_task_key( - const uint64_t key_1, - const uint64_t key_2, - const uint64_t key_3, - const uint64_t key_4, - const ObDRTaskKeyType key_type); // operations of tenant_id uint64_t get_tenant_id() const { return tenant_id_; } void set_tenant_id(const uint64_t tenant_id) { tenant_id_ = tenant_id; } @@ -364,9 +346,6 @@ public: // operations of transmit_data_size int64_t get_transmit_data_size() const { return transmit_data_size_; } void set_transmit_data_size(const int64_t size) { transmit_data_size_ = size; } - // operations of sibling_in_schedule - bool is_sibling_in_schedule() const { return sibling_in_schedule_; } - void set_sibling_in_schedule(bool is_schedule) { sibling_in_schedule_ = is_schedule; } // operations of invoked_source_ obrpc::ObAdminClearDRTaskArg::TaskType get_invoked_source() const { return invoked_source_; } void set_invoked_source(obrpc::ObAdminClearDRTaskArg::TaskType t) { invoked_source_ = t; } @@ -412,7 +391,6 @@ protected: * transmitted, so the tranmit_data_size_ is set to zero. */ int64_t transmit_data_size_; - bool sibling_in_schedule_; obrpc::ObAdminClearDRTaskArg::TaskType invoked_source_; int64_t generate_time_; ObDRTaskPriority priority_; @@ -430,7 +408,8 @@ public: src_member_(), data_src_member_(), force_data_src_member_(), - paxos_replica_number_(0) {} + paxos_replica_number_(0), + prioritize_same_zone_src_(false) {} virtual ~ObMigrateLSReplicaTask() {} public: int build( @@ -449,8 +428,7 @@ public: const common::ObReplicaMember &src_member, const common::ObReplicaMember &data_src_member, const common::ObReplicaMember &force_data_src_member, - const int64_t paxos_replica_number - ); + const int64_t paxos_replica_number); // only use some necessary information build a ObMigrateLSReplicaTask // Specifically, this method is only used when manually executing operation and maintenance commands @@ -480,7 +458,8 @@ public: K(src_member_), K(data_src_member_), K(force_data_src_member_), - K(paxos_replica_number_)); + K(paxos_replica_number_), + K(prioritize_same_zone_src_)); virtual int get_execute_transmit_size( int64_t &execute_transmit_size) const override; @@ -514,6 +493,8 @@ public: void *input_ptr, ObDRTask *&output_task) const override; public: + bool get_prioritize_same_zone_src() const { return prioritize_same_zone_src_; }; + void set_prioritize_same_zone_src(bool p) { prioritize_same_zone_src_ = p; }; // operations of dst_replica_ int set_dst_replica( const ObDstReplica &that); @@ -548,6 +529,7 @@ private: common::ObReplicaMember data_src_member_; common::ObReplicaMember force_data_src_member_; int64_t paxos_replica_number_; + bool prioritize_same_zone_src_; }; class ObAddLSReplicaTask : public ObDRTask diff --git a/src/rootserver/ob_disaster_recovery_task_mgr.cpp b/src/rootserver/ob_disaster_recovery_task_mgr.cpp index 5d3543f8a..ef3c6e834 100644 --- a/src/rootserver/ob_disaster_recovery_task_mgr.cpp +++ b/src/rootserver/ob_disaster_recovery_task_mgr.cpp @@ -48,12 +48,59 @@ using namespace share; namespace rootserver { + +static const char* ls_replica_parallel_migration_mode[] = { + "AUTO", + "ON", + "OFF" +}; + +const char* ObParallelMigrationMode::get_mode_str() const { + STATIC_ASSERT(ARRAYSIZEOF(ls_replica_parallel_migration_mode) == (int64_t)MAX, + "ls_replica_parallel_migration_mode string array size mismatch enum ParallelMigrationMode count"); + const char *str = NULL; + if (mode_ >= AUTO && mode_ < MAX) { + str = ls_replica_parallel_migration_mode[static_cast(mode_)]; + } else { + LOG_WARN_RET(OB_ERR_UNEXPECTED, "invalid ParallelMigrationMode", K_(mode)); + } + return str; +} + +int64_t ObParallelMigrationMode::to_string(char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + J_OBJ_START(); + J_KV(K_(mode), "mode", get_mode_str()); + J_OBJ_END(); + return pos; +} + +int ObParallelMigrationMode::parse_from_string(const ObString &mode) +{ + int ret = OB_SUCCESS; + bool found = false; + STATIC_ASSERT(ARRAYSIZEOF(ls_replica_parallel_migration_mode) == (int64_t)MAX, + "ls_replica_parallel_migration_mode string array size mismatch enum ParallelMigrationMode count"); + for (int64_t i = 0; i < ARRAYSIZEOF(ls_replica_parallel_migration_mode) && !found; i++) { + if (0 == mode.case_compare(ls_replica_parallel_migration_mode[i])) { + mode_ = static_cast(i); + found = true; + break; + } + } + if (!found) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("fail to parse type from string", KR(ret), K(mode), K_(mode)); + } + return ret; +} + ObDRTaskQueue::ObDRTaskQueue() : inited_(false), config_(nullptr), task_alloc_(), wait_list_(), schedule_list_(), - task_map_(), rpc_proxy_(nullptr), priority_(ObDRTaskPriority::MAX_PRI) { @@ -69,20 +116,18 @@ void ObDRTaskQueue::reuse() { while (!wait_list_.is_empty()) { ObDRTask *t = wait_list_.remove_first(); - remove_task_from_map_and_free_it_(t); + free_task_(t); } while (!schedule_list_.is_empty()) { ObDRTask *t = schedule_list_.remove_first(); - remove_task_from_map_and_free_it_(t); + free_task_(t); } - task_map_.clear(); } void ObDRTaskQueue::reset() { wait_list_.reset(); schedule_list_.reset(); - task_map_.clear(); } void ObDRTaskQueue::free_task_(ObDRTask *&task) @@ -94,17 +139,8 @@ void ObDRTaskQueue::free_task_(ObDRTask *&task) } } -void ObDRTaskQueue::remove_task_from_map_and_free_it_(ObDRTask *&task) -{ - if (OB_NOT_NULL(task)) { - task_map_.erase_refactored(task->get_task_key()); - free_task_(task); - } -} - int ObDRTaskQueue::init( common::ObServerConfig &config, - const int64_t bucket_num, obrpc::ObSrvRpcProxy *rpc_proxy, ObDRTaskPriority priority) { @@ -112,13 +148,10 @@ int ObDRTaskQueue::init( if (OB_UNLIKELY(inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", KR(ret)); - } else if (OB_UNLIKELY(bucket_num <= 0) - || OB_ISNULL(rpc_proxy) - || (ObDRTaskPriority::LOW_PRI != priority && ObDRTaskPriority::HIGH_PRI != priority)) { + } else if (OB_ISNULL(rpc_proxy) + || (ObDRTaskPriority::LOW_PRI != priority && ObDRTaskPriority::HIGH_PRI != priority)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(bucket_num), KP(rpc_proxy), K(priority)); - } else if (OB_FAIL(task_map_.create(bucket_num, "DRTaskMap"))) { - LOG_WARN("fail to create task map", KR(ret), K(bucket_num)); + LOG_WARN("invalid argument", KR(ret), KP(rpc_proxy), K(priority)); } else if (OB_FAIL(task_alloc_.init( ObMallocAllocator::get_instance(), OB_MALLOC_MIDDLE_BLOCK_SIZE, ObMemAttr(common::OB_SERVER_TENANT_ID, "DRTaskAlloc")))) { @@ -132,100 +165,11 @@ int ObDRTaskQueue::init( return ret; } -int ObDRTaskQueue::check_task_in_scheduling( - const ObDRTaskKey &task_key, - bool &task_in_scheduling) const -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else if (OB_UNLIKELY(!task_key.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(task_key)); - } else { - ObDRTask *task = nullptr; - int tmp_ret = task_map_.get_refactored(task_key, task); - if (OB_SUCCESS == tmp_ret) { - if (OB_ISNULL(task)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("a null task ptr getted from task_map", KR(ret), K(task_key)); - } else { - task_in_scheduling = task->in_schedule(); - } - } else if (OB_HASH_NOT_EXIST == tmp_ret) { - // task not exist means task not executing - task_in_scheduling = false; - } else { - ret = tmp_ret; - LOG_WARN("fail to get from map", KR(ret), K(task_key)); - } - } - return ret; -} - -int ObDRTaskQueue::check_task_exist( - const ObDRTaskKey &task_key, - bool &task_exist) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else if (OB_UNLIKELY(!task_key.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(task_key)); - } else { - ObDRTask *task = nullptr; - int tmp_ret = task_map_.get_refactored(task_key, task); - if (OB_SUCCESS == tmp_ret) { - task_exist = true; - } else if (OB_HASH_NOT_EXIST == tmp_ret) { - task_exist = false; - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get from task_map", KR(ret), K(task_key)); - } - } - return ret; -} - -int ObDRTaskQueue::push_task_in_wait_list( - ObDRTaskMgr &task_mgr, - const ObDRTaskQueue &sibling_queue, - const ObDRTask &task, - bool &has_task_in_schedule) -{ - int ret = OB_SUCCESS; - has_task_in_schedule = false; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else { - const ObDRTaskKey &task_key = task.get_task_key(); - ObDRTask *task_ptr = nullptr; - int tmp_ret = task_map_.get_refactored(task_key, task_ptr); - if (OB_HASH_NOT_EXIST == tmp_ret) { - if (OB_FAIL(do_push_task_in_wait_list_(task_mgr, sibling_queue, task, has_task_in_schedule))) { - LOG_WARN("fail to push back", KR(ret)); - } - } else if (OB_SUCCESS == tmp_ret) { - ret = OB_ENTRY_EXIST; - LOG_INFO("disaster recovery task exist", KR(ret), K(task_key), K(task), KP(this)); - } else { - ret = tmp_ret; - LOG_WARN("fail to check task exist", KR(ret), K(task_key)); - } - } - return ret; -} - int ObDRTaskQueue::push_task_in_schedule_list( const ObDRTask &task) { // STEP 1: push task into schedule list - // STEP 2: push task into task_map - // STEP 3: set task in schedule + // STEP 2: set task in schedule int ret = OB_SUCCESS; void *raw_ptr = nullptr; ObDRTask *new_task = nullptr; @@ -241,8 +185,6 @@ int ObDRTaskQueue::push_task_in_schedule_list( } else if (OB_ISNULL(new_task)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("new_task is nullptr", KR(ret)); - } else if (OB_FAIL(task_map_.set_refactored(new_task->get_task_key(), new_task))) { - LOG_WARN("fail to set map", KR(ret), "task_key", new_task->get_task_key()); } else { // set schedule_time for this task new_task->set_schedule(); @@ -256,7 +198,7 @@ int ObDRTaskQueue::push_task_in_schedule_list( if (OB_FAIL(ret)) { if (OB_NOT_NULL(new_task)) { - remove_task_from_map_and_free_it_(new_task); + free_task_(new_task); } else if (OB_NOT_NULL(raw_ptr)) { task_alloc_.free(raw_ptr); raw_ptr = nullptr; @@ -265,11 +207,9 @@ int ObDRTaskQueue::push_task_in_schedule_list( return ret; } -int ObDRTaskQueue::do_push_task_in_wait_list_( +int ObDRTaskQueue::do_push_task_in_wait_list( ObDRTaskMgr &task_mgr, - const ObDRTaskQueue &sibling_queue, - const ObDRTask &task, - bool &has_task_in_schedule) + const ObDRTask &task) { int ret = OB_SUCCESS; void *raw_ptr = nullptr; @@ -277,11 +217,7 @@ int ObDRTaskQueue::do_push_task_in_wait_list_( if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); - } else if (OB_ISNULL(config_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("config_ ptr is null", KR(ret), KP(config_)); - } else if (OB_ISNULL(raw_ptr = task_alloc_.alloc( - task.get_clone_size()))) { + } else if (OB_ISNULL(raw_ptr = task_alloc_.alloc(task.get_clone_size()))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to alloc task", KR(ret), "size", task.get_clone_size()); } else if (OB_FAIL(task.clone(raw_ptr, new_task))) { @@ -293,36 +229,12 @@ int ObDRTaskQueue::do_push_task_in_wait_list_( ret = OB_ERR_UNEXPECTED; LOG_WARN("fail to add new task to wait list", KR(ret), "task", *new_task); } else { - has_task_in_schedule = false; - bool sibling_in_schedule = false; - if (OB_FAIL(sibling_queue.check_task_in_scheduling( - new_task->get_task_key(), sibling_in_schedule))) { - LOG_WARN("fail to check has in schedule task", KR(ret), - "task_key", new_task->get_task_key()); - } else if (OB_FAIL(task_map_.set_refactored( - new_task->get_task_key(), new_task))) { - LOG_WARN("fail to set map", KR(ret), "task_key", new_task->get_task_key()); - } else if (task_mgr.get_reach_concurrency_limit() && !sibling_in_schedule) { - task_mgr.clear_reach_concurrency_limit(); - } - if (OB_SUCC(ret)) { - has_task_in_schedule = sibling_in_schedule; - if (OB_FAIL(set_sibling_in_schedule(new_task->get_task_key(), sibling_in_schedule))) { - LOG_WARN("fail to set sibling in schedule", KR(ret), - "task_key", new_task->get_task_key(), K(sibling_in_schedule)); - } - } - - if (OB_FAIL(ret)) { - wait_list_.remove(new_task); - } else { - LOG_INFO("success to push a task in waiting list", K(task), K(has_task_in_schedule)); - } + LOG_INFO("success to push a task in waiting list", K(task)); } if (OB_FAIL(ret)) { if (OB_NOT_NULL(new_task)) { - remove_task_from_map_and_free_it_(new_task); + free_task_(new_task); } else if (OB_NOT_NULL(raw_ptr)) { task_alloc_.free(raw_ptr); raw_ptr = nullptr; @@ -344,13 +256,8 @@ int ObDRTaskQueue::pop_task( LOG_WARN("config_ ptr is null", KR(ret), KP(config_)); } else { DLIST_FOREACH(t, wait_list_) { - if (t->is_sibling_in_schedule()) { - // task can not pop - LOG_INFO("can not pop this task because a sibling task already in schedule", KPC(t)); - } else { - task = t; - break; - } + task = t; // any task can be pop, no other task in double queue is conflict with it. + break; } if (OB_NOT_NULL(task)) { // when task not empty, we move it from wait to schedule list, @@ -365,39 +272,56 @@ int ObDRTaskQueue::pop_task( } // if fail to add to schedule list, clean it directly if (OB_FAIL(ret)) { - remove_task_from_map_and_free_it_(task); + free_task_(task); } } } return ret; } -int ObDRTaskQueue::get_task( +int ObDRTaskQueue::get_task_by_task_id( const share::ObTaskId &task_id, - const ObDRTaskKey &task_key, ObDRTask *&task) { int ret = OB_SUCCESS; + task = nullptr; if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); + } else if (OB_UNLIKELY(!task_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(task_id)); + } else if (OB_FAIL(get_task_by_task_id_in_list_(task_id, wait_list_, task))) { + LOG_WARN("check task exist in wait list failed", KR(ret), K(task_id), K(wait_list_)); + } else if (OB_NOT_NULL(task)) { + } else if (OB_FAIL(get_task_by_task_id_in_list_(task_id, schedule_list_, task))) { + LOG_WARN("check task exist in schedule list failed", KR(ret), K(task_id), K(schedule_list_)); + } + return ret; +} + +int ObDRTaskQueue::get_task_by_task_id_in_list_( + const share::ObTaskId &task_id, + TaskList &list, + ObDRTask *&task) +{ + int ret = OB_SUCCESS; + task = nullptr; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_UNLIKELY(!task_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(task_id)); } else { - ObDRTask *my_task = nullptr; - int tmp_ret = task_map_.get_refactored(task_key, my_task); - if (OB_HASH_NOT_EXIST == tmp_ret) { - task = nullptr; - } else if (OB_SUCCESS == tmp_ret) { - if (OB_ISNULL(my_task)) { + DLIST_FOREACH(task_in_list, list) { + if (OB_ISNULL(task_in_list)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("my_task ptr is null", KR(ret), KP(my_task)); - } else if (my_task->get_task_id() == task_id) { - task = my_task; - } else { - task = nullptr; + LOG_WARN("task_in_list is null ptr", KR(ret), K(task_id), K(list)); + } else if (task_in_list->get_task_id() == task_id) { + task = task_in_list; + break; } - } else { - ret = tmp_ret; - LOG_WARN("fail to get task from map", KR(ret), K(task_key), K(task_id)); } } return ret; @@ -528,7 +452,7 @@ int ObDRTaskQueue::try_clean_and_cancel_task( FLOG_INFO("need cancel migrate task in wait list", K(*t)); // remove from wait_list_, need use DLIST_FOREACH_REMOVESAFE wait_list_.remove(t); - remove_task_from_map_and_free_it_(t); + free_task_(t); } } } @@ -552,45 +476,19 @@ int ObDRTaskQueue::finish_schedule( // remove from schedule_list_ schedule_list_.remove(task); FLOG_INFO("[DRTASK_NOTICE] success to finish schedule task", KR(ret), KPC(task)); - remove_task_from_map_and_free_it_(task); + free_task_(task); } return ret; } -int ObDRTaskQueue::set_sibling_in_schedule( - const ObDRTask &task, - const bool in_schedule) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else { - const ObDRTaskKey &task_key = task.get_task_key(); - ObDRTask *my_task = nullptr; - int tmp_ret = task_map_.get_refactored(task_key, my_task); - if (OB_HASH_NOT_EXIST == tmp_ret) { - // bypass - } else if (OB_SUCCESS == tmp_ret) { - if (OB_ISNULL(my_task)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("my_task ptr is null", KR(ret), K(task)); - } else { - my_task->set_sibling_in_schedule(in_schedule); - } - } else { - ret = tmp_ret; - LOG_WARN("fail to get task from map", KR(ret), K(task_key)); - } - } - return ret; -} - -int ObDRTaskQueue::set_sibling_in_schedule( +int ObDRTaskQueue::check_whether_task_conflict_in_list_( const ObDRTaskKey &task_key, - const bool in_schedule) + const TaskList &list, + const bool enable_parallel_migration, + bool &is_conflict) const { int ret = OB_SUCCESS; + is_conflict = false; if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); @@ -598,25 +496,63 @@ int ObDRTaskQueue::set_sibling_in_schedule( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(task_key)); } else { - ObDRTask *my_task = nullptr; - int tmp_ret = task_map_.get_refactored(task_key, my_task); - if (OB_HASH_NOT_EXIST == tmp_ret) { - // bypass - } else if (OB_SUCCESS == tmp_ret) { - if (OB_ISNULL(my_task)) { + DLIST_FOREACH(task_in_list, list) { + if (OB_ISNULL(task_in_list)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("my_task ptr is null", KR(ret), K(task_key), KP(my_task)); - } else { - my_task->set_sibling_in_schedule(in_schedule); - } - } else { - ret = tmp_ret; - LOG_WARN("fail to get task from map", KR(ret), K(task_key)); + LOG_WARN("task_in_list is null ptr", KR(ret), K(task_key), K(list)); + } else if (task_in_list->get_task_key().get_tenant_id() == task_key.get_tenant_id() + && task_in_list->get_task_key().get_ls_id() == task_key.get_ls_id()) { + // tenant_id + ls_id is same + if (!enable_parallel_migration) { + LOG_INFO("enable_parallel_migration is false, tenant_id + ls_id is the same, prohibit parallel", + KR(ret), K(enable_parallel_migration), K(task_key)); + // if tenant_id + ls_id is the same, then task conflict + is_conflict = true; + } else if (ObDRTaskType::LS_MIGRATE_REPLICA != task_key.get_task_type() + || ObDRTaskType::LS_MIGRATE_REPLICA != task_in_list->get_disaster_recovery_task_type()) { + // if one of the two task is not a migration task, the task is conflict. + LOG_INFO("there is a task is not migration task, prohibit parallel", KR(ret), + K(task_key), K(task_in_list->get_disaster_recovery_task_type())); + is_conflict = true; + } else if (task_in_list->get_task_key().get_zone() == task_key.get_zone()) { + LOG_INFO("two migrate task conflict, task execution zone is the same, prohibit parallel", + KR(ret), K(task_key), K(task_in_list->get_task_key())); + is_conflict = true; + } + if (is_conflict) { + break; + } + } // end with tenant_id + ls_id is same } } return ret; } +int ObDRTaskQueue::check_whether_task_conflict( + const ObDRTaskKey &task_key, + const bool enable_parallel_migration, + bool &is_conflict) +{ + int ret = OB_SUCCESS; + is_conflict = false; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_UNLIKELY(!task_key.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(task_key)); + } else if (OB_FAIL(check_whether_task_conflict_in_list_(task_key, wait_list_, enable_parallel_migration, is_conflict))) { + LOG_WARN("check task conflict in wait list failed", KR(ret), K(task_key), K(wait_list_), K(enable_parallel_migration)); + } else if (is_conflict) { + LOG_INFO("task conflict in wait list", KR(ret), K(task_key), K(enable_parallel_migration), K(priority_)); + } else if (OB_FAIL(check_whether_task_conflict_in_list_(task_key, schedule_list_, enable_parallel_migration, is_conflict))) { + LOG_WARN("check task conflict in schedule list failed", KR(ret), K(task_key), K(schedule_list_), K(enable_parallel_migration)); + } else if (is_conflict) { + LOG_INFO("task conflict in schedule list", KR(ret), K(task_key), K(enable_parallel_migration), K(priority_)); + } + return ret; +} + int ObDRTaskQueue::dump_statistic() const { int ret = OB_SUCCESS; @@ -668,10 +604,10 @@ int ObDRTaskMgr::init( sql_proxy_ = sql_proxy; schema_service_ = schema_service; if (OB_FAIL(high_task_queue_.init( - config, TASK_QUEUE_LIMIT, rpc_proxy_, ObDRTaskPriority::HIGH_PRI))) { + config, rpc_proxy_, ObDRTaskPriority::HIGH_PRI))) { LOG_WARN("fail to init high priority task queue", KR(ret)); } else if (OB_FAIL(low_task_queue_.init( - config, TASK_QUEUE_LIMIT, rpc_proxy_, ObDRTaskPriority::LOW_PRI))) { + config, rpc_proxy_, ObDRTaskPriority::LOW_PRI))) { LOG_WARN("fail to init low priority task queue", KR(ret)); } else if (OB_FAIL(disaster_recovery_task_table_updater_.init(sql_proxy, this))) { LOG_WARN("fail to init a ObDRTaskTableUpdater", KR(ret)); @@ -800,55 +736,7 @@ void ObDRTaskMgr::run3() FLOG_INFO("disaster task mgr exits"); } -int ObDRTaskMgr::check_task_in_executing( - const ObDRTaskKey &task_key, - const ObDRTaskPriority priority, - bool &task_in_executing) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(check_inner_stat_())) { - LOG_WARN("fail to check inner stat", KR(ret), K_(inited), K_(stopped), K_(loaded)); - } else if (ObDRTaskPriority::HIGH_PRI != priority - && ObDRTaskPriority::LOW_PRI != priority) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(priority)); - } else { - ObThreadCondGuard guard(cond_); - ObDRTaskQueue &queue = ObDRTaskPriority::LOW_PRI == priority - ? low_task_queue_ - : high_task_queue_; - if (OB_FAIL(queue.check_task_in_scheduling(task_key, task_in_executing))) { - LOG_WARN("fail to check task exist", KR(ret), K(task_key)); - } - } - return ret; -} - -int ObDRTaskMgr::check_task_exist( - const ObDRTaskKey &task_key, - const ObDRTaskPriority priority, - bool &task_exist) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(check_inner_stat_())) { - LOG_WARN("fail to check inner stat", KR(ret), K_(inited), K_(stopped), K_(loaded)); - } else if (ObDRTaskPriority::HIGH_PRI != priority - && ObDRTaskPriority::LOW_PRI != priority) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(priority)); - } else { - ObThreadCondGuard guard(cond_); - ObDRTaskQueue &queue = ObDRTaskPriority::LOW_PRI == priority - ? low_task_queue_ - : high_task_queue_; - if (OB_FAIL(queue.check_task_exist(task_key, task_exist))) { - LOG_WARN("fail to check task exist", KR(ret), K(task_key)); - } - } - return ret; -} - -int ObDRTaskMgr::add_task_in_queue_and_execute(const ObDRTask &task) +int ObDRTaskMgr::add_task_in_queue_and_execute(ObDRTask &task) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -859,31 +747,19 @@ int ObDRTaskMgr::add_task_in_queue_and_execute(const ObDRTask &task) LOG_WARN("invalid dr task", KR(ret), K(task)); } else { ObThreadCondGuard guard(cond_); - bool task_exist = false; - bool sibling_in_schedule = false; + bool is_conflict = false; ObDRTaskQueue &queue = task.is_high_priority_task() ? high_task_queue_ : low_task_queue_; - ObDRTaskQueue &sibling_queue = task.is_high_priority_task() ? low_task_queue_ : high_task_queue_; if (OB_UNLIKELY(queue.task_cnt() >= TASK_QUEUE_LIMIT)) { ret = OB_SIZE_OVERFLOW; LOG_WARN("disaster recovery task queue is full", KR(ret), "task_cnt", queue.task_cnt()); - } else if (OB_FAIL(queue.check_task_exist(task.get_task_key(), task_exist))) { - LOG_WARN("fail to check task in scheduling", KR(ret), K(task)); - } else if (task_exist) { + } else if (OB_FAIL(check_whether_task_conflict_(task, is_conflict))) { + LOG_WARN("fail to check whether task conflict", KR(ret), K(task)); + } else if (is_conflict) { + // task conflict return error code, report an error to user ret = OB_ENTRY_EXIST; - LOG_WARN("ls disaster recovery task has existed in queue", KR(ret), K(task), K(task_exist)); - } else if (OB_FAIL(sibling_queue.check_task_in_scheduling(task.get_task_key(), sibling_in_schedule))) { - LOG_WARN("fail to check task in scheduling", KR(ret), K(task)); - } else if (sibling_in_schedule) { - ret = OB_ENTRY_EXIST; - LOG_WARN("ls disaster recovery task has existed in sibling_queue", - KR(ret), K(task), K(sibling_in_schedule)); + LOG_WARN("ls disaster recovery task has existed in queue", KR(ret), K(task), K(is_conflict)); } else if (OB_FAIL(queue.push_task_in_schedule_list(task))) { LOG_WARN("fail to add task to schedule list", KR(ret), K(task)); - } else if (OB_FAIL(set_sibling_in_schedule(task, true/*in_schedule*/))) { - //after successfully adding the scheduling queue, - //need mark in both queues that the task has been scheduled in the queue. - //if there is a task in the waiting queue of another queue, need update its mark - LOG_WARN("set sibling in schedule failed", KR(ret), K(task)); } else { if (OB_SUCCESS != (tmp_ret = task.log_execute_start())) { LOG_WARN("fail to log task start", KR(tmp_ret), K(task)); @@ -898,45 +774,30 @@ int ObDRTaskMgr::add_task_in_queue_and_execute(const ObDRTask &task) } int ObDRTaskMgr::add_task( - const ObDRTask &task) + ObDRTask &task) { int ret = OB_SUCCESS; if (OB_FAIL(check_inner_stat_())) { - ret = OB_NOT_INIT; LOG_WARN("fail to check inner stat", KR(ret), K_(inited), K_(loaded), K_(stopped)); } else if (OB_UNLIKELY(!task.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid dr task", KR(ret), K(task)); } else { ObThreadCondGuard guard(cond_); - ObDRTaskQueue &queue = task.is_high_priority_task() - ? high_task_queue_ - : low_task_queue_; - ObDRTaskQueue &sibling_queue = task.is_high_priority_task() - ? low_task_queue_ - : high_task_queue_; - bool has_task_in_schedule = false; + ObDRTaskQueue &queue = task.is_high_priority_task() ? high_task_queue_ : low_task_queue_; + bool is_conflict = false; if (OB_UNLIKELY(queue.task_cnt() >= TASK_QUEUE_LIMIT)) { ret = OB_SIZE_OVERFLOW; LOG_WARN("disaster recovery task queue is full", KR(ret), "task_cnt", queue.task_cnt()); - } else if (OB_FAIL(queue.push_task_in_wait_list(*this, sibling_queue, task, has_task_in_schedule))) { - if (OB_ENTRY_EXIST != ret) { - LOG_WARN("fail to push task", KR(ret), K(task)); - } else { - ret = OB_SUCCESS; - LOG_INFO("task already exist in queue", K(task)); - } + } else if (OB_FAIL(check_whether_task_conflict_(task, is_conflict))) { + LOG_WARN("fail to check whether task conflict", KR(ret), K(task)); + } else if (is_conflict) { + ret = OB_ENTRY_EXIST; + LOG_WARN("ls disaster recovery task has existed in queue", KR(ret), K(task), K(is_conflict)); + } else if (OB_FAIL(queue.do_push_task_in_wait_list(*this, task))) { + LOG_WARN("fail to push task", KR(ret), K(task)); } else { - int64_t wait_cnt = 0; - int64_t schedule_cnt = 0; - if (OB_FAIL(inner_get_task_cnt_(wait_cnt, schedule_cnt))) { - LOG_WARN("fail to get task cnt", KR(ret)); - } else if (!has_task_in_schedule - && 0 == get_reach_concurrency_limit()) { - cond_.broadcast(); - LOG_INFO("success to broad cast cond_", K(wait_cnt), K(schedule_cnt)); - } - clear_reach_concurrency_limit(); + cond_.broadcast(); LOG_INFO("[DRTASK_NOTICE] add task to disaster recovery task mgr finish", KR(ret), K(task)); } } @@ -947,22 +808,13 @@ int ObDRTaskMgr::deal_with_task_reply( const ObDRTaskReplyResult &reply) { int ret = OB_SUCCESS; - ObDRTaskKey task_key; if (OB_FAIL(check_inner_stat_())) { - ret = OB_NOT_INIT; LOG_WARN("fail to check inner stat", KR(ret), K_(inited), K_(loaded), K_(stopped)); - } else if (OB_FAIL(task_key.init( - reply.tenant_id_, - reply.ls_id_.id(), - 0, /* set to 0 */ - 0, /* set to 0 */ - ObDRTaskKeyType::FORMAL_DR_KEY))) { - LOG_WARN("fail to init task key", KR(ret), K(reply)); } else { int tmp_ret = OB_SUCCESS; ObDRTask *task = nullptr; ObThreadCondGuard guard(cond_); - if (OB_SUCCESS != (tmp_ret = get_task_by_id_(reply.task_id_, task_key, task))) { + if (OB_SUCCESS != (tmp_ret = get_task_by_id_(reply.task_id_, task))) { if (OB_ENTRY_NOT_EXIST == tmp_ret) { // task not exist, try record this reply result ROOTSERVICE_EVENT_ADD("disaster_recovery", "finish_disaster_recovery_task", @@ -972,7 +824,7 @@ int ObDRTaskMgr::deal_with_task_reply( "execute_result", reply.result_, "ret_comment", ob_disaster_recovery_task_ret_comment_strs(ObDRTaskRetComment::RECEIVE_FROM_STORAGE_RPC)); } else { - LOG_WARN("fail to get task from task manager", KR(tmp_ret), K(reply), K(task_key)); + LOG_WARN("fail to get task from task manager", KR(tmp_ret), K(reply)); } } else if (OB_SUCCESS != (tmp_ret = task->log_execute_result(reply.result_, ObDRTaskRetComment::RECEIVE_FROM_STORAGE_RPC))){ LOG_WARN("fail to log execute result", KR(tmp_ret), K(reply)); @@ -980,7 +832,7 @@ int ObDRTaskMgr::deal_with_task_reply( if (OB_FAIL(async_add_cleaning_task_to_updater( reply.task_id_, - task_key, + task->get_task_key(), reply.result_, false,/*need_record_event*/ ObDRTaskRetComment::RECEIVE_FROM_STORAGE_RPC, @@ -1003,12 +855,12 @@ int ObDRTaskMgr::async_add_cleaning_task_to_updater( ObDRTask *task = nullptr; if (OB_FAIL(check_inner_stat_())) { LOG_WARN("fail to check inner stat", KR(ret), K_(inited), K_(stopped), K_(loaded)); - } else if (OB_FAIL(get_task_by_id_(task_id, task_key, task))) { + } else if (OB_FAIL(get_task_by_id_(task_id, task))) { if (OB_ENTRY_NOT_EXIST == ret) { - LOG_WARN("fail to get task, task may be cleaned earlier", KR(ret), K(task_id), K(task_key)); + LOG_WARN("fail to get task, task may be cleaned earlier", KR(ret), K(task_id)); ret = OB_SUCCESS; } else { - LOG_WARN("fail to get task from task manager", KR(ret), K(task_id), K(task_key)); + LOG_WARN("fail to get task from task manager", KR(ret), K(task_id)); } } if (OB_SUCC(ret) @@ -1031,7 +883,6 @@ int ObDRTaskMgr::async_add_cleaning_task_to_updater( int ObDRTaskMgr::do_cleaning( const share::ObTaskId &task_id, - const ObDRTaskKey &task_key, const int ret_code, const bool need_clear_server_data_in_limit, const bool need_record_event, @@ -1046,7 +897,7 @@ int ObDRTaskMgr::do_cleaning( ObDRTask *task = nullptr; common::ObAddr dst_server; for (int64_t i = 0; OB_SUCC(ret) && i < ARRAYSIZEOF(queues_); ++i) { - if (OB_FAIL(queues_[i].get_task(task_id, task_key, task))) { + if (OB_FAIL(queues_[i].get_task_by_task_id(task_id, task))) { LOG_WARN("fail to get schedule task from queue", KR(ret), "priority", queues_[i].get_priority_str()); } else if (OB_NOT_NULL(task)) { task_queue = &queues_[i]; @@ -1057,21 +908,18 @@ int ObDRTaskMgr::do_cleaning( if (OB_SUCC(ret)) { if (OB_ISNULL(task)) { LOG_INFO("in schedule taks not found, maybe not sync because of network traffic", - K(task_id), K(task_key), K(ret_code)); + K(task_id), K(ret_code)); } else { if (need_record_event) { (void)log_task_result(*task, ret_code, ret_comment); } dst_server = task->get_dst_server(); - if (OB_FAIL(set_sibling_in_schedule(*task, false/* not in schedule*/))) { - LOG_WARN("fail to set sibling in schedule", KR(ret), KPC(task)); - } else if (OB_ISNULL(task_queue)) { + if (OB_ISNULL(task_queue)) { LOG_INFO("task_queue is null"); // by pass } else if (OB_FAIL(task_queue->finish_schedule(task))) { LOG_WARN("fail to finish scheduling task", KR(ret), KPC(task)); } } - clear_reach_concurrency_limit(); } } return ret; @@ -1085,7 +933,6 @@ int ObDRTaskMgr::get_all_task_count( { int ret = OB_SUCCESS; if (OB_FAIL(check_inner_stat_())) { - ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret), K_(inited), K_(loaded), K_(stopped)); } else { ObThreadCondGuard guard(cond_); @@ -1111,14 +958,13 @@ int ObDRTaskMgr::log_task_result( int ObDRTaskMgr::get_task_by_id_( const share::ObTaskId &task_id, - const ObDRTaskKey &task_key, ObDRTask *&task) { int ret = OB_SUCCESS; ObDRTask *task_to_get = nullptr; void *raw_ptr = nullptr; for (int64_t i = 0; OB_SUCC(ret) && i < ARRAYSIZEOF(queues_); ++i) { - if (OB_FAIL(queues_[i].get_task(task_id, task_key, task_to_get))) { + if (OB_FAIL(queues_[i].get_task_by_task_id(task_id, task_to_get))) { LOG_WARN("fail to get schedule task from queue", KR(ret), "priority", queues_[i].get_priority_str()); } else if (OB_NOT_NULL(task_to_get)) { break; @@ -1127,7 +973,7 @@ int ObDRTaskMgr::get_task_by_id_( if (OB_SUCC(ret) && OB_ISNULL(task_to_get)) { task = nullptr; ret = OB_ENTRY_NOT_EXIST; - LOG_WARN("task not exist, maybe cleaned earier", KR(ret), K(task_id), K(task_key)); + LOG_WARN("task not exist, maybe cleaned earier", KR(ret), K(task_id)); } else { task = task_to_get; } @@ -1164,7 +1010,6 @@ int ObDRTaskMgr::load_task_to_schedule_list_() for (int64_t i = 0; i < static_cast(ObDRTaskPriority::MAX_PRI); ++i) { queues_[i].reuse(); } - clear_reach_concurrency_limit(); for (int64_t i = 0; OB_SUCC(ret) && i < tenant_id_array.count(); ++i) { // load this tenant's task info into schedule_list // TODO@jingyu.cr: need to isolate different tenant @@ -1449,8 +1294,7 @@ int ObDRTaskMgr::try_pop_task( LOG_WARN("fail to check inner stat", KR(ret), K_(inited), K_(stopped), K_(loaded)); } else if (OB_FAIL(inner_get_task_cnt_(wait_cnt, in_schedule_cnt))) { LOG_WARN("fail to get task cnt", KR(ret)); - } else if (wait_cnt > 0 - && 0 == concurrency_limited_ts_) { + } else if (wait_cnt > 0) { if (OB_FAIL(pop_task(my_task))) { LOG_WARN("fail to pop task", KR(ret)); } else if (OB_ISNULL(my_task)) { @@ -1478,13 +1322,9 @@ int ObDRTaskMgr::try_pop_task( } else { int64_t now = ObTimeUtility::current_time(); cond_.wait(get_schedule_interval()); - if (get_reach_concurrency_limit() + CONCURRENCY_LIMIT_INTERVAL < now) { - clear_reach_concurrency_limit(); - LOG_TRACE("success to clear concurrency limit"); - } } if (OB_SUCC(ret) && OB_NOT_NULL(task)) { - LOG_INFO("[DRTASK_NOTICE] success to pop a task", KPC(task), K_(concurrency_limited_ts), + LOG_INFO("[DRTASK_NOTICE] success to pop a task", KPC(task), K(in_schedule_cnt)); } return ret; @@ -1509,18 +1349,6 @@ int ObDRTaskMgr::pop_task( } } } - if (OB_SUCC(ret)) { - if (OB_ISNULL(task)) { - if (wait_cnt > 0) { - set_reach_concurrency_limit(); - } - } else { - const bool in_schedule = true; - if (OB_FAIL(set_sibling_in_schedule(*task, in_schedule))) { - LOG_WARN("set sibling in schedule failed", KR(ret), KPC(task)); - } - } - } } return ret; } @@ -1595,22 +1423,112 @@ int ObDRTaskMgr::execute_task( return ret; } -int ObDRTaskMgr::set_sibling_in_schedule( - const ObDRTask &task, - const bool in_schedule) +int ObDRTaskMgr::check_whether_task_conflict_( + ObDRTask &task, + bool &is_conflict) +{ + int ret = OB_SUCCESS; + is_conflict = false; + bool enable_parallel_migration = false; + if (OB_FAIL(check_inner_stat_())) { + LOG_WARN("fail to check inner stat", KR(ret), K_(inited), K_(stopped), K_(loaded)); + } else if (OB_UNLIKELY(!task.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(task)); + } else if (OB_FAIL(check_tenant_enable_parallel_migration_(task.get_tenant_id(), enable_parallel_migration))) { + LOG_WARN("check tenant enable parallel migration failed", KR(ret), K(task), K(enable_parallel_migration)); + } else if (OB_FAIL(high_task_queue_.check_whether_task_conflict(task.get_task_key(), enable_parallel_migration, is_conflict))) { + LOG_WARN("fail to check task conflict", KR(ret), K(task), K(enable_parallel_migration)); + } else if (is_conflict) { + LOG_INFO("task conflict in high_task_queue", K(task.get_task_key()), K(enable_parallel_migration)); + } else if (OB_FAIL(low_task_queue_.check_whether_task_conflict(task.get_task_key(), enable_parallel_migration, is_conflict))) { + LOG_WARN("fail to check task conflict", KR(ret), K(task), K(enable_parallel_migration)); + } else if (is_conflict) { + LOG_INFO("task conflict in low_task_queue", K(task.get_task_key()), K(enable_parallel_migration)); + } else if (enable_parallel_migration && OB_FAIL(set_migrate_task_prioritize_src_(enable_parallel_migration, task))) { + // migrate task need set prioritize_same_zone_src_ + LOG_WARN("failed to set enable_parallel_migration", KR(ret), K(enable_parallel_migration), K(task)); + } + return ret; +} + +int ObDRTaskMgr::check_tenant_enable_parallel_migration_( + const uint64_t &tenant_id, + bool &enable_parallel_migration) +{ + int ret = OB_SUCCESS; + const char *str = "auto"; + ObParallelMigrationMode mode; + enable_parallel_migration = false; + uint64_t tenant_data_version = 0; + if (OB_FAIL(check_inner_stat_())) { + LOG_WARN("fail to check inner stat", KR(ret), K_(inited), K_(loaded), K_(stopped)); + } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id)); + } else { + omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id)); + share::ObTenantRole tenant_role; + if (OB_FAIL(GET_MIN_DATA_VERSION(gen_meta_tenant_id(tenant_id), tenant_data_version))) { + LOG_WARN("fail to get min data version", KR(ret), K(tenant_id)); + } else if (!((tenant_data_version >= DATA_VERSION_4_3_5_0) + || (tenant_data_version >= MOCK_DATA_VERSION_4_2_5_0 && tenant_data_version < DATA_VERSION_4_3_0_0))) { + enable_parallel_migration = false; + } else if (OB_UNLIKELY(!tenant_config.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant config is invalid", KR(ret), K(tenant_id)); + } else if (FALSE_IT(str = tenant_config->replica_parallel_migration_mode.str())) { + } else if (OB_FAIL(mode.parse_from_string(str))) { + LOG_WARN("mode parse failed", KR(ret), K(str)); + } else if (!mode.is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("parallel migration mode is invalid", KR(ret), K(mode)); + } else if (mode.is_on_mode()) { + enable_parallel_migration = true; + } else if (mode.is_off_mode()) { + enable_parallel_migration = false; + } else if (mode.is_auto_mode()) { + if (!is_user_tenant(tenant_id)) { + // sys and meta tenant is primary tenant + enable_parallel_migration = false; + } else if (OB_FAIL(ObAllTenantInfoProxy::get_tenant_role(GCTX.sql_proxy_, tenant_id, tenant_role))) { + LOG_WARN("fail to get tenant_role", KR(ret), K(tenant_id)); + } else if (!tenant_role.is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant_role is invalid", KR(ret), K(tenant_role)); + } else if (tenant_role.is_primary()) { + enable_parallel_migration = false; + } else { + enable_parallel_migration = true; + // in auto mode, other tenant(clone restore standby) enable_parallel_migration is true + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("parallel migration mode is invalid", KR(ret), K(mode)); + } + LOG_INFO("check tenant enable_parallel_migration over", KR(ret), + K(tenant_id), K(enable_parallel_migration), K(tenant_role), K(mode)); + } + return ret; +} + +int ObDRTaskMgr::set_migrate_task_prioritize_src_( + const bool enable_parallel_migration, + ObDRTask &task) { int ret = OB_SUCCESS; if (OB_FAIL(check_inner_stat_())) { - LOG_WARN("fail to check inner stat", KR(ret), K_(inited), K_(stopped), K_(loaded)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < ARRAYSIZEOF(queues_); ++i) { - if (OB_FAIL(queues_[i].set_sibling_in_schedule(task, in_schedule))) { - if (i == 0) { - LOG_WARN("fail to set sibling in schedule in high priority queue", KR(ret), K(task)); - } else { - LOG_WARN("fail to set sibling in schedule in low priority queue", KR(ret), K(task)); - } - } + LOG_WARN("fail to check inner stat", KR(ret), K_(inited), K_(loaded), K_(stopped)); + } else if (OB_UNLIKELY(!task.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid dr task", KR(ret), K(task)); + } else if (ObDRTaskType::LS_MIGRATE_REPLICA == task.get_task_key().get_task_type()) { + ObMigrateLSReplicaTask *migrate_task = static_cast(&task); + if (OB_ISNULL(migrate_task)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("migrate_task is nullptr", KR(ret), K(task)); + } else { + migrate_task->set_prioritize_same_zone_src(enable_parallel_migration); } } return ret; diff --git a/src/rootserver/ob_disaster_recovery_task_mgr.h b/src/rootserver/ob_disaster_recovery_task_mgr.h index 56c6a7c0c..f3c0dff27 100644 --- a/src/rootserver/ob_disaster_recovery_task_mgr.h +++ b/src/rootserver/ob_disaster_recovery_task_mgr.h @@ -35,13 +35,42 @@ namespace rootserver class ObDRTaskExecutor; class ObDRTaskMgr; +class ObParallelMigrationMode +{ + OB_UNIS_VERSION(1); +public: + enum ParallelMigrationMode + { + AUTO = 0, + ON, + OFF, + MAX + }; +public: + ObParallelMigrationMode() : mode_(MAX) {} + ObParallelMigrationMode(ParallelMigrationMode mode) : mode_(mode) {} + ObParallelMigrationMode &operator=(const ParallelMigrationMode mode) { mode_ = mode; return *this; } + ObParallelMigrationMode &operator=(const ObParallelMigrationMode &other) { mode_ = other.mode_; return *this; } + bool operator==(const ObParallelMigrationMode &other) const { return other.mode_ == mode_; } + bool operator!=(const ObParallelMigrationMode &other) const { return other.mode_ != mode_; } + void reset() { mode_ = MAX; } + void assign(const ObParallelMigrationMode &other) { mode_ = other.mode_; } + bool is_auto_mode() const { return AUTO == mode_; } + bool is_on_mode() const { return ON == mode_; } + bool is_off_mode() const { return OFF == mode_; } + bool is_valid() const { return MAX != mode_; } + const ParallelMigrationMode &get_mode() const { return mode_; } + int parse_from_string(const ObString &mode); + int64_t to_string(char *buf, const int64_t buf_len) const; + const char* get_mode_str() const; +private: + ParallelMigrationMode mode_; +}; + class ObDRTaskQueue { public: typedef common::ObDList TaskList; - typedef common::hash::ObHashMap TaskMap; public: ObDRTaskQueue(); virtual ~ObDRTaskQueue(); @@ -50,39 +79,35 @@ public: void reset(); // init a ObDRTaskQueue // @param [in] config, server config - // @param [in] bucket_num, the size of task_map // @param [in] rpc_proxy, to send rpc int init( common::ObServerConfig &config, - const int64_t bucket_num, obrpc::ObSrvRpcProxy *rpc_proxy, ObDRTaskPriority priority); public: - // check whether a task in task_map is executing - // @param [in] task_key, task's tenant_id and ls_id - // @param [out] task_in_scheduling, whether this task is in scheduling - int check_task_in_scheduling( + // check whether task is conflict with task in queue + // @param [in] task_key, task's tenant_id and ls_id and zone + // @param [in] enable_parallel_migration, if enable parallel migration + // @param [out] is_conflict, whether this task is conflict with any task in queue + int check_whether_task_conflict( const ObDRTaskKey &task_key, - bool &task_in_scheduling) const; + const bool enable_parallel_migration, + bool &is_conflict); - // check whether a task exist in task_map - // @param [in] task_key, task's tenant_id and ls_id - // @param [out] task_exist, whether task exist in task_map - int check_task_exist( - const ObDRTaskKey &task_key, - bool &task_exist); - - // push a task into this queue's wait_list + // do push a task in wait list // @param [in] task_mgr, to deal with concurrency_limit - // @param [in] sibling_queue, to check whether another task is scheduling in sibling_queue // @param [in] task, the task to push in - // @param [out] has_task_in_schedule, whether another task is scheduling in sibling_queue - int push_task_in_wait_list( + int do_push_task_in_wait_list( ObDRTaskMgr &task_mgr, - const ObDRTaskQueue &sibling_queue, - const ObDRTask &task, - bool &has_task_in_schedule); + const ObDRTask &task); + + // get a certain task from queue by task_id + // @param [in] task_id, the only id to identify a task + // @param [out] task, the task getted + int get_task_by_task_id( + const share::ObTaskId &task_id, + ObDRTask *&task); // push a task into this queue's schedule_list // @param [in] task, the task to push in @@ -94,15 +119,6 @@ public: int pop_task( ObDRTask *&task); - // get a certain task from task_map - // @param [in] task_id, the only id to identify a task - // @param [in] task_key, tenant_id and ls_id to locate task quickly in task_map - // @param [out] the task getted - int get_task( - const share::ObTaskId &task_id, - const ObDRTaskKey &task_key, - ObDRTask *&task); - // to deal with those not running tasks in schedule_list // @param [in] task_mgr, to execute over a task int try_clean_and_cancel_task( @@ -113,23 +129,9 @@ public: int finish_schedule( ObDRTask *task); - // set task's sibling_in_schedule - // @param [in] task, to locate which task to modify - // @oaram [in] in_schedule, whether another task is scheduling in sibling queue - int set_sibling_in_schedule( - const ObDRTask &task, - const bool in_schedule); - - // set task's sibling_in_schedule - // @param [in] task_key, to locate which task to modify quickly - // @oaram [in] in_schedule, whether another task is scheduling in sibling queue - int set_sibling_in_schedule( - const ObDRTaskKey &task_key, - const bool in_schedule); - int64_t wait_task_cnt() const { return wait_list_.get_size(); } int64_t in_schedule_task_cnt() const { return schedule_list_.get_size(); } - int64_t task_cnt() const { return task_map_.size(); } + int64_t task_cnt() const { return wait_list_.get_size() + schedule_list_.get_size(); } int dump_statistic() const; const char* get_priority_str() const { @@ -153,17 +155,25 @@ public: TaskList &get_schedule_list() { return schedule_list_; } private: - // do push a task in wait list - // @param [in] task_mgr, to deal with concurrency limit - // @param [in] sibling_queue, another queue - // @param [in] task, the task to push in - // @param [out] has_task_in_schedule, whether task is schedule in sibling queue - int do_push_task_in_wait_list_( - ObDRTaskMgr &task_mgr, - const ObDRTaskQueue &sibling_queue, - const ObDRTask &task, - bool &has_task_in_schedule); + // check whether a task is conflict with task in list + // @param [in] task_key, task's tenant_id and ls_id and zone + // @param [in] list, target list to check + // @param [in] enable_parallel_migration, if enable parallel migration + // @param [out] is_conflict, whether this task is conflict with any task in list + int check_whether_task_conflict_in_list_( + const ObDRTaskKey &task_key, + const TaskList &list, + const bool enable_parallel_migration, + bool &is_conflict) const; + // get a certain task from list by task_id + // @param [in] task_id, the only id to identify a task + // @param [in] list, target list to find + // @param [out] task, the task getted + int get_task_by_task_id_in_list_( + const share::ObTaskId &task_id, + TaskList &list, + ObDRTask *&task); // check whether to clean this task // @param [in] task, the task to check // @param [out] need_cleaning, whether to clean this task @@ -176,16 +186,12 @@ private: // @param [in] task, task to free void free_task_(ObDRTask *&task); - // remove task from task_map and free it - // @param [in] task, task to clean - void remove_task_from_map_and_free_it_(ObDRTask *&task); private: bool inited_; common::ObServerConfig *config_; common::ObFIFOAllocator task_alloc_; TaskList wait_list_; TaskList schedule_list_; - TaskMap task_map_; obrpc::ObSrvRpcProxy *rpc_proxy_; ObDRTaskPriority priority_; private: @@ -207,7 +213,6 @@ public: stopped_(true), loaded_(false), config_(nullptr), - concurrency_limited_ts_(0), cond_(), queues_(), high_task_queue_(queues_[0]), @@ -255,32 +260,14 @@ public: int send_rpc_to_cancel_migrate_task( const ObDRTask &task); - // check whether a task is in schedule - // @param [in] task_key, the task to check - // @param [in] priority, which queue to check - // @param [out] task_in_executing, whether task is scheduling - virtual int check_task_in_executing( - const ObDRTaskKey &task_key, - const ObDRTaskPriority priority, - bool &task_in_executing); - - // check whether a task exist - // @param [in] task_key, the task to check - // @param [in] priority, which queue to check - // @param [out] task_exist, whether task is exist - virtual int check_task_exist( - const ObDRTaskKey &task_key, - const ObDRTaskPriority priority, - bool &task_exist); - // add task in schedule list and execute task // @param [in] task, target task virtual int add_task_in_queue_and_execute( - const ObDRTask &task); + ObDRTask &task); // add a task into queue // @param [in] task, the task to push in virtual int add_task( - const ObDRTask &task); + ObDRTask &task); // to do something after receive task reply // param [in] reply, the execute result of this task @@ -302,27 +289,16 @@ public: // finish schedule this task and clean it // param [in] task_id, to identify a task - // param [in] task_key, to locate a task quickly // param [in] ret_code, execute result of this task // param [in] need_clear_server_data_in_limit, whether clear data_in_limit + // param [in] ret_comment, ret comment int do_cleaning( const share::ObTaskId &task_id, - const ObDRTaskKey &task_key, const int ret_code, const bool need_clear_server_data_in_limit, const bool need_record_event, const ObDRTaskRetComment &ret_comment); - // operations of reach_concurrency_limit - void set_reach_concurrency_limit() { - concurrency_limited_ts_ = common::ObTimeUtility::current_time(); - } - void clear_reach_concurrency_limit() { - concurrency_limited_ts_ = 0; - } - int64_t get_reach_concurrency_limit() const { - return concurrency_limited_ts_; - } int64_t get_schedule_interval() const { return 1000L; // 1s } @@ -355,18 +331,34 @@ private: const uint64_t tenant_id, const common::ObAddr &server_addr, bool &has_unit); + // check whether a task conflict with any task in double queue + // @param [in] task, the task to check + // @param [out] is_conflict, whether task is conflict + int check_whether_task_conflict_( + ObDRTask &task, + bool &is_conflict); + // check if tenant enable parallel migration + // @param [in] tenant_id, tenant_id to check + // @param [out] enable_parallel_migration, if enable parallel migration + int check_tenant_enable_parallel_migration_( + const uint64_t &tenant_id, + bool &enable_parallel_migration); + // set migrate task prioritize_same_zone_src field + // @param [in] enable_parallel_migration, if enable parallel migration + // @param [out] task, target task to set + int set_migrate_task_prioritize_src_( + const bool enable_parallel_migration, + ObDRTask &task); ObDRTaskQueue &get_high_priority_queue_() { return high_task_queue_; } ObDRTaskQueue &get_low_priority_queue_() { return low_task_queue_; } int check_inner_stat_() const; // get a task by task id // @param [in] task_id, to identify a certain task - // @param [in] task_key, to quickly locate a task // @param [out] task, the task to get // ATTENTION: need to lock task memory before use this function int get_task_by_id_( const share::ObTaskId &task_id, - const ObDRTaskKey &task_key, ObDRTask *&task); // free a task @@ -428,13 +420,6 @@ private: int execute_manual_task_( const ObDRTask &task); - // set sibling in schedule - // @param [in] task, which task to deal with - // @param [in] in_schedule, whether in schedule - int set_sibling_in_schedule( - const ObDRTask &task, - const bool in_schedule); - private: bool inited_; bool stopped_; @@ -443,7 +428,6 @@ private: /* has waiting task but cannot be scheduled, * since mgr reaches server_data_copy_[in/out]_concurrency */ - volatile int64_t concurrency_limited_ts_; mutable common::ObThreadCond cond_; ObDRTaskQueue queues_[static_cast(ObDRTaskPriority::MAX_PRI)]; ObDRTaskQueue &high_task_queue_; // queues_[0] diff --git a/src/rootserver/ob_disaster_recovery_task_table_updater.cpp b/src/rootserver/ob_disaster_recovery_task_table_updater.cpp index 575c581b0..bfff2a41e 100644 --- a/src/rootserver/ob_disaster_recovery_task_table_updater.cpp +++ b/src/rootserver/ob_disaster_recovery_task_table_updater.cpp @@ -46,12 +46,13 @@ int ObDRTaskTableUpdateTask::init( ret = OB_INVALID_ARGUMENT; LOG_WARN("task init failed", KR(ret), K(tenant_id), K(ls_id), K(task_type), K(task_id), K(add_timestamp)); + } else if (OB_FAIL(task_key_.assign(task_key))) { + LOG_WARN("failed to assign task_key", KR(ret), K(task_key)); } else { tenant_id_ = tenant_id; ls_id_ = ls_id; task_type_ = task_type; task_id_ = task_id; - task_key_ = task_key; ret_code_ = ret_code; need_clear_server_data_in_limit_ = need_clear_server_data_in_limit; need_record_event_ = need_record_event; @@ -65,16 +66,19 @@ int ObDRTaskTableUpdateTask::assign(const ObDRTaskTableUpdateTask &other) { int ret = OB_SUCCESS; if (this != &other) { - tenant_id_ = other.tenant_id_; - ls_id_ = other.ls_id_; - task_type_ = other.task_type_; - task_id_ = other.task_id_; - task_key_ = other.task_key_; - ret_code_ = other.ret_code_; - need_clear_server_data_in_limit_ = other.need_clear_server_data_in_limit_; - need_record_event_ = other.need_record_event_; - ret_comment_ = other.ret_comment_; - add_timestamp_ = other.add_timestamp_; + if (OB_FAIL(task_key_.assign(other.task_key_))) { + LOG_WARN("failed to assign task_key", KR(ret), K(other)); + } else { + tenant_id_ = other.tenant_id_; + ls_id_ = other.ls_id_; + task_type_ = other.task_type_; + task_id_ = other.task_id_; + ret_code_ = other.ret_code_; + need_clear_server_data_in_limit_ = other.need_clear_server_data_in_limit_; + need_record_event_ = other.need_record_event_; + ret_comment_ = other.ret_comment_; + add_timestamp_ = other.add_timestamp_; + } } return ret; } @@ -314,7 +318,6 @@ int ObDRTaskTableUpdater::process_task_( // must be committed successfully, if memory cleaning fails, rely on detection to clean up the memory. if (FAILEDx(task_mgr_->do_cleaning( task.get_task_id(), - task.get_task_key(), task.get_ret_code(), task.get_need_clear_server_data_in_limit(), task.get_need_record_event(), diff --git a/src/rootserver/ob_disaster_recovery_task_table_updater.h b/src/rootserver/ob_disaster_recovery_task_table_updater.h index c4d0aa471..ef754cb31 100644 --- a/src/rootserver/ob_disaster_recovery_task_table_updater.h +++ b/src/rootserver/ob_disaster_recovery_task_table_updater.h @@ -35,27 +35,6 @@ public: need_record_event_(true), ret_comment_(ObDRTaskRetComment::MAX), add_timestamp_(OB_INVALID_TIMESTAMP) {} - explicit ObDRTaskTableUpdateTask( - const uint64_t tenant_id, - const share::ObLSID &ls_id, - const ObDRTaskType &task_type, - const share::ObTaskId &task_id, - const ObDRTaskKey &task_key, - const int ret_code, - const bool need_clear_server_data_in_limit, - const bool need_record_event, - const ObDRTaskRetComment &ret_comment, - const int64_t add_timestamp) - : tenant_id_(tenant_id), - ls_id_(ls_id), - task_type_(task_type), - task_id_(task_id), - task_key_(), - ret_code_(ret_code), - need_clear_server_data_in_limit_(need_clear_server_data_in_limit), - need_record_event_(need_record_event), - ret_comment_(ret_comment), - add_timestamp_(add_timestamp) { task_key_ = task_key; } virtual ~ObDRTaskTableUpdateTask() {} int init( const uint64_t tenant_id, diff --git a/src/rootserver/ob_disaster_recovery_worker.cpp b/src/rootserver/ob_disaster_recovery_worker.cpp index d589fd218..9d83968e9 100755 --- a/src/rootserver/ob_disaster_recovery_worker.cpp +++ b/src/rootserver/ob_disaster_recovery_worker.cpp @@ -1496,8 +1496,8 @@ int ObDRWorker::LocalityAlignment::try_get_normal_locality_alignment_task( LOG_WARN("this task ptr is null", KR(ret)); } else { switch (this_task->get_task_type()) { - case RemovePaxos: - case RemoveNonPaxos: + case ObDRTaskType::LS_REMOVE_PAXOS_REPLICA: + case ObDRTaskType::LS_REMOVE_NON_PAXOS_REPLICA: if (OB_FAIL(try_review_remove_replica_task( unit_provider, this_task, @@ -1508,7 +1508,7 @@ int ObDRWorker::LocalityAlignment::try_get_normal_locality_alignment_task( LOG_INFO("success to try review remove replica task", KR(ret), KPC(this_task), K(found)); } break; - case AddReplica: + case ObDRTaskType::LS_ADD_REPLICA: if (OB_FAIL(try_review_add_replica_task( unit_provider, this_task, @@ -1519,7 +1519,7 @@ int ObDRWorker::LocalityAlignment::try_get_normal_locality_alignment_task( LOG_INFO("success to try review add replica task", KR(ret), KPC(this_task), K(found)); } break; - case TypeTransform: + case ObDRTaskType::LS_TYPE_TRANSFORM: if (OB_FAIL(try_review_type_transform_task( unit_provider, this_task, @@ -1530,7 +1530,7 @@ int ObDRWorker::LocalityAlignment::try_get_normal_locality_alignment_task( LOG_INFO("success to try review type transform task", KR(ret), KPC(this_task), K(found)); } break; - case ModifyPaxosReplicaNumber: + case ObDRTaskType::LS_MODIFY_PAXOS_REPLICA_NUMBER: if (OB_FAIL(try_review_modify_paxos_replica_number_task( unit_provider, this_task, @@ -2470,7 +2470,38 @@ int ObDRWorker::do_cancel_ls_replica_task( return ret; } -int ObDRWorker::add_task_in_queue_and_execute_(const ObDRTask &task) +int ObDRWorker::add_task_to_task_mgr_( + ObDRTask &task, + int64_t &acc_dr_task) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("DRWorker not init", KR(ret)); + } else if (OB_UNLIKELY(!task.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(task)); + } else if (OB_ISNULL(disaster_recovery_task_mgr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("disaster_recovery_task_mgr_ null", KR(ret), KP(disaster_recovery_task_mgr_)); + } else if (OB_FAIL(disaster_recovery_task_mgr_->add_task(task))) { + if (OB_ENTRY_EXIST == ret) { + ret = OB_SUCCESS; + acc_dr_task++; + // if this task is conflict, all subsequent tasks in this round are conflicting. + // so set acc_dr_task++, no need continue to check + LOG_INFO("task has conflict in task mgr", KR(ret), K(task)); + } else { + LOG_WARN("fail to add task", KR(ret), K(task)); + } + } else { + acc_dr_task++; + LOG_INFO("success to add a task to task manager", KR(ret), K(task)); + } + return ret; +} + +int ObDRWorker::add_task_in_queue_and_execute_(ObDRTask &task) { int ret = OB_SUCCESS; FLOG_INFO("add task in schedule list and execute", K(task)); @@ -3313,26 +3344,27 @@ int ObDRWorker::check_other_inactive_server_count_( int ObDRWorker::generate_task_key( const DRLSInfo &dr_ls_info, + const common::ObAddr &task_exe_server, + const ObDRTaskType &task_type, ObDRTaskKey &task_key) const { int ret = OB_SUCCESS; + uint64_t tenant_id = OB_INVALID_TENANT_ID; + ObLSID ls_id; + common::ObZone zone; + task_key.reset(); if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); - } else { - uint64_t tenant_id = OB_INVALID_ID; - ObLSID ls_id; - if (OB_FAIL(dr_ls_info.get_ls_id( - tenant_id, ls_id))) { - LOG_WARN("fail to get log stream id", KR(ret)); - } else if (OB_FAIL(task_key.init( - tenant_id, - ls_id.id(), - 0/* set to 0 */, - 0/* set to 0 */, - ObDRTaskKeyType::FORMAL_DR_KEY))) { - LOG_WARN("fail to init task key", KR(ret), K(tenant_id), K(ls_id)); - } + } else if (OB_UNLIKELY(ObDRTaskType::MAX_TYPE == task_type || !task_exe_server.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(task_type), K(task_exe_server)); + } else if (OB_FAIL(dr_ls_info.get_ls_id(tenant_id, ls_id))) { + LOG_WARN("fail to get log stream id", KR(ret), K(dr_ls_info)); + } else if (OB_FAIL(SVR_TRACER.get_server_zone(task_exe_server, zone))) { + LOG_WARN("get server zone failed", KR(ret), K(task_exe_server)); + } else if (OB_FAIL(task_key.init(tenant_id, ls_id, zone, task_type))) { + LOG_WARN("fail to init task key", KR(ret), K(tenant_id), K(ls_id), K(zone), K(task_type)); } return ret; } @@ -3443,81 +3475,37 @@ int ObDRWorker::check_has_leader_while_remove_replica( return ret; } -int ObDRWorker::check_task_already_exist( - const ObDRTaskKey &task_key, - const DRLSInfo &dr_ls_info, - const int64_t &priority, - bool &task_exist) -{ - int ret = OB_SUCCESS; - task_exist = true; - bool sibling_task_executing = false; - ObDRTaskPriority task_priority = priority == 0 ? ObDRTaskPriority::HIGH_PRI : ObDRTaskPriority::LOW_PRI; - ObDRTaskPriority sibling_priority = priority == 0 - ? ObDRTaskPriority::LOW_PRI - : ObDRTaskPriority::HIGH_PRI; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else if (OB_ISNULL(disaster_recovery_task_mgr_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("disaster recovery task mgr ptr is null", KR(ret)); - } else if (OB_FAIL(disaster_recovery_task_mgr_->check_task_exist( - task_key, task_priority, task_exist))) { - LOG_WARN("fail to check task exist", KR(ret), K(task_key)); - } else if (task_exist) { - FLOG_INFO("high prio task exist for this ls", K(task_key)); - } else if (OB_FAIL(disaster_recovery_task_mgr_->check_task_in_executing( - task_key, sibling_priority, sibling_task_executing))) { - LOG_WARN("fail to check task in executing", KR(ret), K(task_key)); - } else if (sibling_task_executing) { - task_exist = true; - FLOG_INFO("has sibling task in executing for this ls", K(task_key)); - } - return ret; -} - int ObDRWorker::check_can_generate_task( const int64_t acc_dr_task, const bool need_check_has_leader_while_remove_replica, - const bool is_high_priority_task, const ObAddr &server_addr, DRLSInfo &dr_ls_info, - ObDRTaskKey &task_key, + const ObDRTaskType &task_type, bool &can_generate) { int ret = OB_SUCCESS; - bool task_exist = false; bool has_leader_while_remove_replica = false; - int64_t task_pri = is_high_priority_task ? 0 : 1; - can_generate = false; + can_generate = true; if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); - } else if (acc_dr_task != 0) { + } else if (ObDRTaskType::LS_MIGRATE_REPLICA != task_type && acc_dr_task != 0) { + // if a disaster recovery task is generated in this round, or a task is currently executing in the mgr queue + // and a conflict is detected, acc_dr_task++ will be used to prohibit subsequent task generation. + // in order to implement parallel migration, when the task of the first replica of LS is generated or a conflict + // in the migration of the first replica is detected, it is necessary to continue to generate the migration task of the next replica. can_generate = false; - LOG_INFO("can not generate task because another task is generated", K(dr_ls_info), K(is_high_priority_task)); - } else if (OB_FAIL(generate_task_key(dr_ls_info, task_key))) { - LOG_WARN("fail to generate task key", KR(ret), K(dr_ls_info)); - } else if (OB_FAIL(check_task_already_exist(task_key, dr_ls_info, task_pri, task_exist))) { - LOG_WARN("task exist in task manager", KR(ret), K(task_key), K(dr_ls_info), K(is_high_priority_task)); - } else if (task_exist) { - can_generate = false; - LOG_INFO("can not generate task because already exist", K(dr_ls_info), K(is_high_priority_task)); + LOG_INFO("can not generate task because another task is generated", K(dr_ls_info), K(task_type), K(acc_dr_task)); } else if (need_check_has_leader_while_remove_replica) { if (OB_FAIL(check_has_leader_while_remove_replica( server_addr, dr_ls_info, has_leader_while_remove_replica))) { LOG_WARN("fail to check has leader while member change", KR(ret), K(dr_ls_info), K(server_addr)); - } else if (has_leader_while_remove_replica) { - can_generate = true; - } else { + } else if (!has_leader_while_remove_replica) { can_generate = false; - LOG_INFO("can not generate task because has no leader while remove replica", K(dr_ls_info), K(is_high_priority_task)); + LOG_INFO("can not generate task because has no leader", K(dr_ls_info), K(task_type), K(acc_dr_task), K(has_leader_while_remove_replica)); } - } else { - can_generate = true; } return ret; } @@ -3590,10 +3578,8 @@ int ObDRWorker::generate_remove_permanent_offline_replicas_and_push_into_task_ma replica_type))) { LOG_WARN("fail to build remove member task", KR(ret), K(task_key), K(tenant_id), K(ls_id), K(leader_addr), K(remove_member), K(old_paxos_replica_number), K(new_paxos_replica_number), K(replica_type)); - } else if (OB_FAIL(disaster_recovery_task_mgr_->add_task(remove_replica_task))) { + } else if (OB_FAIL(add_task_to_task_mgr_(remove_replica_task, acc_dr_task))) { LOG_WARN("fail to add task", KR(ret), K(remove_replica_task)); - } else { - acc_dr_task++; } return ret; } @@ -3696,7 +3682,6 @@ int ObDRWorker::do_single_replica_permanent_offline_( common::ObAddr leader_addr; const common::ObAddr source_server; // not useful const bool need_check_has_leader_while_remove_replica = ObReplicaTypeCheck::is_paxos_replica_V2(replica_type); - const bool is_high_priority_task = true; const int64_t memstore_percent = 100; ObDRTaskKey task_key; bool can_generate = false; @@ -3740,17 +3725,18 @@ int ObDRWorker::do_single_replica_permanent_offline_( } else { LOG_INFO("success to add display info", KR(ret), K(display_info)); } + } else if (OB_FAIL(generate_task_key(dr_ls_info, leader_addr, task_type, task_key))) { + LOG_WARN("fail to generate task key", KR(ret), K(dr_ls_info), K(leader_addr), K(task_type)); } else if (OB_FAIL(check_can_generate_task( acc_dr_task, need_check_has_leader_while_remove_replica, - is_high_priority_task, member_to_remove.get_server(), dr_ls_info, - task_key, + task_type, can_generate))) { LOG_WARN("fail to check can generate remove permanent offline task", KR(ret), K(acc_dr_task), - K(need_check_has_leader_while_remove_replica), K(is_high_priority_task), K(member_to_remove), - K(dr_ls_info), K(task_key), K(can_generate)); + K(need_check_has_leader_while_remove_replica), K(member_to_remove), + K(dr_ls_info), K(can_generate)); } else if (can_generate) { if (OB_FAIL(generate_remove_permanent_offline_replicas_and_push_into_task_manager( task_key, @@ -3888,10 +3874,8 @@ int ObDRWorker::generate_replicate_to_unit_and_push_into_task_manager( ObReplicaMember()/*empty force_data_source*/, old_paxos_replica_number))) { LOG_WARN("fail to build migrate task", KR(ret)); - } else if (OB_FAIL(disaster_recovery_task_mgr_->add_task(migrate_task))) { + } else if (OB_FAIL(add_task_to_task_mgr_(migrate_task, acc_dr_task))) { LOG_WARN("fail to add task", KR(ret), K(migrate_task)); - } else { - ++acc_dr_task; } return ret; } @@ -3977,7 +3961,6 @@ int ObDRWorker::generate_migrate_ls_task( ObDstReplica dst_replica; int64_t old_paxos_replica_number = 0; const bool need_check_has_leader_while_remove_replica = false; - const bool is_high_priority_task = true; bool can_generate = false; ObReplicaMember src_member( ls_replica.get_server(), ls_replica.get_member_time_us(), @@ -4007,10 +3990,16 @@ int ObDRWorker::generate_migrate_ls_task( } else { LOG_INFO("success to add display info", KR(ret), K(display_info)); } + } else if (OB_FAIL(generate_task_key(dr_ls_info, dst_replica.get_server(), ObDRTaskType::LS_MIGRATE_REPLICA, task_key))) { + LOG_WARN("fail to generate task key", KR(ret), K(dr_ls_info), K(dst_replica)); } else if (OB_FAIL(check_can_generate_task( - acc_dr_task, need_check_has_leader_while_remove_replica, - is_high_priority_task, server_stat_info.get_server(), dr_ls_info, - task_key, can_generate))) { + /*when shrink unit num, both dup log stream and normal log stream are here*/ + acc_dr_task, + need_check_has_leader_while_remove_replica, + server_stat_info.get_server(), + dr_ls_info, + ObDRTaskType::LS_MIGRATE_REPLICA, + can_generate))) { LOG_WARN("fail to check can generate replicate to unit task", KR(ret)); } else if (can_generate) { if (OB_FAIL(generate_replicate_to_unit_and_push_into_task_manager( @@ -4026,7 +4015,6 @@ int ObDRWorker::generate_migrate_ls_task( int ObDRWorker::try_generate_remove_replica_locality_alignment_task( DRLSInfo &dr_ls_info, - const ObDRTaskKey &task_key, const LATask *task, int64_t &acc_dr_task) { @@ -4035,12 +4023,13 @@ int ObDRWorker::try_generate_remove_replica_locality_alignment_task( bool sibling_task_executing = false; uint64_t tenant_id = OB_INVALID_ID; share::ObLSID ls_id; - if (OB_UNLIKELY(!task_key.is_valid()) || OB_ISNULL(task)) { + if (OB_ISNULL(task)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(task_key), KP(task)); + LOG_WARN("invalid argument", KR(ret), KP(task)); } else if (OB_FAIL(dr_ls_info.get_ls_id(tenant_id, ls_id))) { LOG_WARN("fail to get ls id", KR(ret)); } else { + ObDRTaskKey task_key; const RemoveReplicaLATask *my_task = reinterpret_cast(task); ObReplicaMember remove_member(my_task->remove_server_, my_task->member_time_us_, @@ -4070,6 +4059,8 @@ int ObDRWorker::try_generate_remove_replica_locality_alignment_task( LOG_INFO("may has no leader while member change", K(dr_ls_info)); } else if (OB_FAIL(dr_ls_info.get_leader(leader_addr))) { LOG_WARN("fail to get leader", KR(ret)); + } else if (OB_FAIL(generate_task_key(dr_ls_info, leader_addr, task->get_task_type(), task_key))) { + LOG_WARN("fail to generate task key", KR(ret)); } else if (OB_FAIL(remove_paxos_task.build( task_key, tenant_id, @@ -4089,11 +4080,8 @@ int ObDRWorker::try_generate_remove_replica_locality_alignment_task( my_task->replica_type_))) { LOG_WARN("fail to build task", KR(ret), K(task_key), K(tenant_id), K(ls_id), K(task_id), K(leader_addr), K(remove_member), KPC(my_task)); - } else if (OB_FAIL(disaster_recovery_task_mgr_->add_task(remove_paxos_task))) { - LOG_WARN("fail to add task", KR(ret)); - } else { - LOG_INFO("success to add a ObRemoveLSReplicaTask to task manager", KR(ret), K(remove_paxos_task)); - acc_dr_task++; + } else if (OB_FAIL(add_task_to_task_mgr_(remove_paxos_task, acc_dr_task))) { + LOG_WARN("fail to add task", KR(ret), K(remove_paxos_task)); } } return ret; @@ -4101,7 +4089,6 @@ int ObDRWorker::try_generate_remove_replica_locality_alignment_task( int ObDRWorker::try_generate_add_replica_locality_alignment_task( DRLSInfo &dr_ls_info, - const ObDRTaskKey &task_key, const LATask *task, int64_t &acc_dr_task) { @@ -4110,12 +4097,13 @@ int ObDRWorker::try_generate_add_replica_locality_alignment_task( bool sibling_task_executing = false; uint64_t tenant_id = OB_INVALID_ID; share::ObLSID ls_id; - if (OB_UNLIKELY(!task_key.is_valid() || nullptr == task)) { + if (OB_ISNULL(task)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(task_key), KP(task)); + LOG_WARN("invalid argument", KR(ret), KP(task)); } else if (OB_FAIL(dr_ls_info.get_ls_id(tenant_id, ls_id))) { LOG_WARN("fail to get ls id", KR(ret)); } else { + ObDRTaskKey task_key; const AddReplicaLATask *my_task = reinterpret_cast(task); int64_t data_size = 0; ObReplicaMember data_source; @@ -4145,6 +4133,8 @@ int ObDRWorker::try_generate_add_replica_locality_alignment_task( LOG_WARN("fail to assign dst replica", KR(ret)); } else if (OB_FAIL(dr_ls_info.get_default_data_source(data_source, data_size))) { LOG_WARN("fail to get data_size", KR(ret), K(dr_ls_info)); + } else if (OB_FAIL(generate_task_key(dr_ls_info, my_task->dst_server_, task->get_task_type(), task_key))) { + LOG_WARN("fail to generate task key", KR(ret)); } else if (OB_FAIL(add_replica_task.build( task_key, tenant_id, @@ -4163,11 +4153,8 @@ int ObDRWorker::try_generate_add_replica_locality_alignment_task( my_task->orig_paxos_replica_number_, my_task->paxos_replica_number_))) { LOG_WARN("fail to build add replica task", KR(ret)); - } else if (OB_FAIL(disaster_recovery_task_mgr_->add_task(add_replica_task))) { - LOG_WARN("fail to add task", KR(ret)); - } else { - LOG_INFO("success to add a ObAddLSReplicaTask to task manager", KR(ret), K(add_replica_task)); - acc_dr_task++; + } else if (OB_FAIL(add_task_to_task_mgr_(add_replica_task, acc_dr_task))) { + LOG_WARN("fail to add task", KR(ret), K(add_replica_task)); } } return ret; @@ -4175,7 +4162,6 @@ int ObDRWorker::try_generate_add_replica_locality_alignment_task( int ObDRWorker::try_generate_type_transform_locality_alignment_task( DRLSInfo &dr_ls_info, - const ObDRTaskKey &task_key, const LATask *task, int64_t &acc_dr_task) { @@ -4184,12 +4170,13 @@ int ObDRWorker::try_generate_type_transform_locality_alignment_task( bool sibling_task_executing = false; uint64_t tenant_id = OB_INVALID_ID; share::ObLSID ls_id; - if (OB_UNLIKELY(!task_key.is_valid() || nullptr == task)) { + if (OB_ISNULL(task)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(task_key), KP(task)); + LOG_WARN("invalid argument", KR(ret), KP(task)); } else if (OB_FAIL(dr_ls_info.get_ls_id(tenant_id, ls_id))) { LOG_WARN("fail to get ls id", KR(ret)); } else { + ObDRTaskKey task_key; const TypeTransformLATask *my_task = reinterpret_cast(task); bool has_leader = false; int64_t data_size = 0; @@ -4222,6 +4209,8 @@ int ObDRWorker::try_generate_type_transform_locality_alignment_task( LOG_WARN("fail to assign dst replica", KR(ret)); } else if (OB_FAIL(dr_ls_info.get_default_data_source(data_source, data_size))) { LOG_WARN("fail to get data_size", KR(ret), K(dr_ls_info)); + } else if (OB_FAIL(generate_task_key(dr_ls_info, my_task->dst_server_, task->get_task_type(), task_key))) { + LOG_WARN("fail to generate task key", KR(ret)); } else if (OB_FAIL(type_transform_task.build( task_key, tenant_id, @@ -4240,11 +4229,8 @@ int ObDRWorker::try_generate_type_transform_locality_alignment_task( my_task->orig_paxos_replica_number_, my_task->paxos_replica_number_))) { LOG_WARN("fail to build type transform task", KR(ret)); - } else if (OB_FAIL(disaster_recovery_task_mgr_->add_task(type_transform_task))) { - LOG_WARN("fail to add task", KR(ret)); - } else { - LOG_INFO("success to add a ObLSTypeTransformTask to task manager", KR(ret), K(type_transform_task)); - acc_dr_task++; + } else if (OB_FAIL(add_task_to_task_mgr_(type_transform_task, acc_dr_task))) { + LOG_WARN("fail to add task", KR(ret), K(type_transform_task)); } } return ret; @@ -4252,7 +4238,6 @@ int ObDRWorker::try_generate_type_transform_locality_alignment_task( int ObDRWorker::try_generate_modify_paxos_replica_number_locality_alignment_task( DRLSInfo &dr_ls_info, - const ObDRTaskKey &task_key, const LATask *task, int64_t &acc_dr_task_cnt) { @@ -4262,12 +4247,13 @@ int ObDRWorker::try_generate_modify_paxos_replica_number_locality_alignment_task uint64_t tenant_id = OB_INVALID_ID; share::ObLSID ls_id; GlobalLearnerList learner_list; - if (OB_UNLIKELY(!task_key.is_valid() || nullptr == task)) { + if (OB_ISNULL(task)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(task_key), KP(task)); + LOG_WARN("invalid argument", KR(ret), KP(task)); } else if (OB_FAIL(dr_ls_info.get_ls_id(tenant_id, ls_id))) { LOG_WARN("fail to get ls id", KR(ret)); } else { + ObDRTaskKey task_key; const ModifyPaxosReplicaNumberLATask *my_task = reinterpret_cast(task); common::ObAddr leader_addr; common::ObMemberList member_list; @@ -4280,6 +4266,8 @@ int ObDRWorker::try_generate_modify_paxos_replica_number_locality_alignment_task member_list, learner_list))) { LOG_WARN("fail to get leader", KR(ret)); + } else if (OB_FAIL(generate_task_key(dr_ls_info, leader_addr, task->get_task_type(), task_key))) { + LOG_WARN("fail to generate task key", KR(ret)); } else if (OB_FAIL(modify_paxos_replica_number_task.build( task_key, tenant_id, @@ -4297,11 +4285,8 @@ int ObDRWorker::try_generate_modify_paxos_replica_number_locality_alignment_task my_task->paxos_replica_number_, member_list))) { LOG_WARN("fail to build a modify paxos replica number task", KR(ret)); - } else if (OB_FAIL(disaster_recovery_task_mgr_->add_task(modify_paxos_replica_number_task))) { - LOG_WARN("fail to add task", KR(ret)); - } else { - LOG_INFO("success to add a ObLSModifyPaxosReplicaNumberTask to task manager", KR(ret), K(modify_paxos_replica_number_task), K(member_list)); - acc_dr_task_cnt++; + } else if (OB_FAIL(add_task_to_task_mgr_(modify_paxos_replica_number_task, acc_dr_task_cnt))) { + LOG_WARN("fail to add task", KR(ret), K(modify_paxos_replica_number_task)); } } return ret; @@ -4313,56 +4298,49 @@ int ObDRWorker::try_generate_locality_alignment_task( int64_t &acc_dr_task_cnt) { int ret = OB_SUCCESS; - ObDRTaskKey task_key; if (OB_ISNULL(task)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), KP(task)); - } else if (OB_FAIL(generate_task_key(dr_ls_info, task_key))) { - LOG_WARN("fail to generate task key", KR(ret)); } else { switch (task->get_task_type()) { - case RemovePaxos: - case RemoveNonPaxos: { + case ObDRTaskType::LS_REMOVE_PAXOS_REPLICA: + case ObDRTaskType::LS_REMOVE_NON_PAXOS_REPLICA: { if (OB_FAIL(try_generate_remove_replica_locality_alignment_task( dr_ls_info, - task_key, task, acc_dr_task_cnt))) { LOG_WARN("fail to try generate remove replica task", - KR(ret), K(task_key), KPC(task)); + KR(ret), KPC(task)); } break; } - case AddReplica: { + case ObDRTaskType::LS_ADD_REPLICA: { if (OB_FAIL(try_generate_add_replica_locality_alignment_task( dr_ls_info, - task_key, task, acc_dr_task_cnt))) { LOG_WARN("fail to try generate add replica paxos task", - KR(ret), K(task_key), KPC(task)); + KR(ret), KPC(task)); } break; } - case TypeTransform: { + case ObDRTaskType::LS_TYPE_TRANSFORM: { if (OB_FAIL(try_generate_type_transform_locality_alignment_task( dr_ls_info, - task_key, task, acc_dr_task_cnt))) { LOG_WARN("fail to try generate type transform paxos task", - KR(ret), K(task_key), KPC(task)); + KR(ret), KPC(task)); } break; } - case ModifyPaxosReplicaNumber: { + case ObDRTaskType::LS_MODIFY_PAXOS_REPLICA_NUMBER: { if (OB_FAIL(try_generate_modify_paxos_replica_number_locality_alignment_task( dr_ls_info, - task_key, task, acc_dr_task_cnt))) { LOG_WARN("fail to try generate modify paxos replica number task", - KR(ret), K(task_key), KPC(task)); + KR(ret), KPC(task)); } break; } @@ -4409,10 +4387,10 @@ int ObDRWorker::record_task_plan_for_locality_alignment( } else { ObLSReplicaTaskDisplayInfo display_info; switch (task->get_task_type()) { - case RemovePaxos: - case RemoveNonPaxos: { + case ObDRTaskType::LS_REMOVE_PAXOS_REPLICA: + case ObDRTaskType::LS_REMOVE_NON_PAXOS_REPLICA: { const RemoveReplicaLATask *my_task = reinterpret_cast(task); - task_type = RemovePaxos == task->get_task_type() ? ObDRTaskType::LS_REMOVE_PAXOS_REPLICA : ObDRTaskType::LS_REMOVE_NON_PAXOS_REPLICA; + task_type = task->get_task_type(); source_replica_type = REPLICA_TYPE_INVALID; target_replica_type = my_task->replica_type_; task_priority = task_type == ObDRTaskType::LS_REMOVE_PAXOS_REPLICA ? ObDRTaskPriority::HIGH_PRI : ObDRTaskPriority::LOW_PRI; @@ -4427,7 +4405,7 @@ int ObDRWorker::record_task_plan_for_locality_alignment( } break; } - case AddReplica: { + case ObDRTaskType::LS_ADD_REPLICA: { const AddReplicaLATask *my_task = reinterpret_cast(task); if (OB_FAIL(dr_ls_info.get_default_data_source(data_source, data_size))) { LOG_WARN("fail to get data_size", KR(ret), K(dr_ls_info)); @@ -4449,7 +4427,7 @@ int ObDRWorker::record_task_plan_for_locality_alignment( } break; } - case TypeTransform: { + case ObDRTaskType::LS_TYPE_TRANSFORM: { const TypeTransformLATask *my_task = reinterpret_cast(task); if (OB_FAIL(dr_ls_info.get_default_data_source(data_source, data_size))) { LOG_WARN("fail to get data_size", KR(ret), K(dr_ls_info)); @@ -4467,7 +4445,7 @@ int ObDRWorker::record_task_plan_for_locality_alignment( } break; } - case ModifyPaxosReplicaNumber: { + case ObDRTaskType::LS_MODIFY_PAXOS_REPLICA_NUMBER: { const ModifyPaxosReplicaNumberLATask *my_task = reinterpret_cast(task); task_type = ObDRTaskType::LS_MODIFY_PAXOS_REPLICA_NUMBER; source_replica_type = REPLICA_TYPE_FULL; @@ -4542,9 +4520,7 @@ int ObDRWorker::try_locality_alignment( while (OB_SUCC(ret) && OB_SUCC(locality_alignment.get_next_locality_alignment_task(task))) { bool can_generate = false; const bool need_check_has_leader_while_remove_replica = false; - const bool is_high_priority_task = task->get_task_type() != LATaskType::RemoveNonPaxos; ObAddr server_addr; //useless - ObDRTaskKey task_key; if (OB_ISNULL(task)) { // bypass, there is no task to generate } else if (only_for_display) { @@ -4556,10 +4532,9 @@ int ObDRWorker::try_locality_alignment( } else if (OB_FAIL(check_can_generate_task( acc_dr_task, need_check_has_leader_while_remove_replica, - is_high_priority_task, server_addr, dr_ls_info, - task_key, + task->get_task_type(), can_generate))) { LOG_WARN("fail to check can generate locality alignment task", KR(ret)); } else if (!can_generate) { @@ -4787,17 +4762,18 @@ int ObDRWorker::try_remove_non_paxos_replica_for_deleting_unit_( } else { LOG_INFO("success to add display info", KR(ret), K(display_info)); } + } else if (OB_FAIL(generate_task_key(dr_ls_info, leader_addr, task_type, task_key))) { + LOG_WARN("fail to generate task key", KR(ret), K(dr_ls_info), K(leader_addr), K(task_type)); } else if (OB_FAIL(check_can_generate_task( acc_dr_task, need_check_has_leader_while_remove_replica, - is_high_priority_task, ls_replica.get_server(), dr_ls_info, - task_key, + task_type, can_generate))) { LOG_WARN("fail to check can generate remove permanent offline task", KR(ret), K(acc_dr_task), - K(need_check_has_leader_while_remove_replica), K(is_high_priority_task), K(ls_replica), - K(dr_ls_info), K(task_key), K(can_generate)); + K(need_check_has_leader_while_remove_replica), K(ls_replica), + K(dr_ls_info), K(can_generate)); } else if (can_generate) { ObRemoveLSReplicaTask remove_replica_task; if (OB_FAIL(remove_replica_task.build( @@ -4819,10 +4795,8 @@ int ObDRWorker::try_remove_non_paxos_replica_for_deleting_unit_( ls_replica.get_replica_type()))) { LOG_WARN("fail to build remove member task", KR(ret), K(task_key), K(tenant_id), K(ls_id), K(leader_addr), K(remove_learner), K(old_paxos_replica_number), K(new_paxos_replica_number)); - } else if (OB_FAIL(disaster_recovery_task_mgr_->add_task(remove_replica_task))) { + } else if (OB_FAIL(add_task_to_task_mgr_(remove_replica_task, acc_dr_task))) { LOG_WARN("fail to add task", KR(ret), K(remove_replica_task)); - } else { - acc_dr_task++; } } } @@ -4970,13 +4944,14 @@ int ObDRWorker::try_type_transform_for_deleting_unit_( } else { LOG_INFO("success to add display info", KR(ret), K(display_info)); } + } else if (OB_FAIL(generate_task_key(dr_ls_info, target_replica.get_server(), ObDRTaskType::LS_TYPE_TRANSFORM, task_key))) { + LOG_WARN("fail to generate task key", KR(ret), K(dr_ls_info), K(target_replica)); } else if (OB_FAIL(check_can_generate_task( acc_dr_task, false/*need_check_has_leader_while_remove_replica*/, - true/*is_high_priority_task*/, target_replica.get_server(), dr_ls_info, - task_key, + ObDRTaskType::LS_TYPE_TRANSFORM, can_generate))) { LOG_WARN("fail to check whether can generate task", KR(ret), K(acc_dr_task), K(target_replica), K(dr_ls_info)); @@ -5163,11 +5138,8 @@ int ObDRWorker::generate_type_transform_task_( LOG_WARN("fail to build type transform task", KR(ret), K(task_key), K(tenant_id), K(ls_id), K(data_size), K(dst_replica), K(src_member), K(data_source), K(old_paxos_replica_number), K(new_paxos_replica_number)); - } else if (OB_FAIL(disaster_recovery_task_mgr_->add_task(type_transform_task))) { + } else if (OB_FAIL(add_task_to_task_mgr_(type_transform_task, acc_dr_task))) { LOG_WARN("fail to add task", KR(ret), K(type_transform_task)); - } else { - LOG_INFO("success to add a ObLSTypeTransformTask to task manager", KR(ret), K(type_transform_task)); - acc_dr_task++; } return ret; } @@ -5301,10 +5273,8 @@ int ObDRWorker::generate_cancel_unit_migration_task( new_paxos_replica_number, replica_type))) { LOG_WARN("fail to build remove member task", KR(ret), K(task_key), K(task_id)); - } else if (OB_FAIL(disaster_recovery_task_mgr_->add_task(remove_member_task))) { + } else if (OB_FAIL(add_task_to_task_mgr_(remove_member_task, acc_dr_task))) { LOG_WARN("fail to add task", KR(ret), K(remove_member_task)); - } else { - ++acc_dr_task; } return ret; } @@ -5329,8 +5299,6 @@ int ObDRWorker::try_cancel_unit_migration( LOG_WARN("config_ ptr is null", KR(ret), KP(config_)); } else if (!config_->is_rereplication_enabled()) { // bypass - } else if (OB_FAIL(generate_task_key(dr_ls_info, task_key))) { - LOG_WARN("fail to generate task key", KR(ret)); } else if (!dr_ls_info.has_leader()) { LOG_WARN("has no leader, maybe not report yet", KR(ret), K(dr_ls_info)); @@ -5418,13 +5386,14 @@ int ObDRWorker::try_cancel_unit_migration( } else { LOG_INFO("success to add display info", KR(ret), K(display_info)); } + } else if (OB_FAIL(generate_task_key(dr_ls_info, leader_addr, task_type, task_key))) { + LOG_WARN("fail to generate task key", KR(ret), K(dr_ls_info), K(leader_addr), K(task_type)); } else if (OB_FAIL(check_can_generate_task( acc_dr_task, need_check_has_leader_while_remove_replica, - is_paxos_replica_related, ls_replica->get_server(), dr_ls_info, - task_key, + task_type, can_generate))) { LOG_WARN("fail to check can generate cancel unit migration task", KR(ret)); } else if (can_generate) { @@ -5585,10 +5554,8 @@ int ObDRWorker::generate_migrate_to_unit_task( ObReplicaMember()/*empty force_data_source*/, old_paxos_replica_number))) { LOG_WARN("fail to build migrate task", KR(ret)); - } else if (OB_FAIL(disaster_recovery_task_mgr_->add_task(migrate_task))) { + } else if (OB_FAIL(add_task_to_task_mgr_(migrate_task, acc_dr_task))) { LOG_WARN("fail to add task", KR(ret), K(migrate_task)); - } else { - ++acc_dr_task; } return ret; } @@ -5599,8 +5566,6 @@ int ObDRWorker::try_migrate_to_unit( int64_t &acc_dr_task) { int ret = OB_SUCCESS; - - ObDRTaskKey task_key; bool task_exist = false; bool sibling_task_executing = false; int64_t replica_cnt = 0; @@ -5612,8 +5577,6 @@ int ObDRWorker::try_migrate_to_unit( LOG_WARN("config_ ptr is null", KR(ret), KP(config_)); } else if (!config_->is_rereplication_enabled()) { // bypass - } else if (OB_FAIL(generate_task_key(dr_ls_info, task_key))) { - LOG_WARN("fail to generate task key", KR(ret)); } else if (!dr_ls_info.has_leader()) { LOG_WARN("has no leader, maybe not report yet", KR(ret), K(dr_ls_info)); @@ -5649,8 +5612,8 @@ int ObDRWorker::try_migrate_to_unit( ObDstReplica dst_replica; int64_t old_paxos_replica_number = 0; bool can_generate = false; + ObDRTaskKey task_key; const bool need_check_has_leader_while_remove_replica = false; - const bool is_high_priority_task = false; ObReplicaMember src_member(ls_replica->get_server(), ls_replica->get_member_time_us(), ls_replica->get_replica_type(), @@ -5711,14 +5674,15 @@ int ObDRWorker::try_migrate_to_unit( } else { LOG_INFO("success to add display info", KR(ret), K(display_info)); } + } else if (OB_FAIL(generate_task_key(dr_ls_info, dst_member.get_server(), ObDRTaskType::LS_MIGRATE_REPLICA, task_key))) { + LOG_WARN("fail to generate task key", KR(ret), K(dr_ls_info), K(dst_member)); } else if (OB_FAIL(check_can_generate_task( - acc_dr_task, - need_check_has_leader_while_remove_replica, - is_high_priority_task, - ls_replica->get_server(), - dr_ls_info, - task_key, - can_generate))) { + acc_dr_task, + need_check_has_leader_while_remove_replica, + ls_replica->get_server(), + dr_ls_info, + ObDRTaskType::LS_MIGRATE_REPLICA, + can_generate))) { LOG_WARN("fail to check can generate migrate to unir task", KR(ret)); } else if (can_generate) { if (OB_FAIL(generate_migrate_to_unit_task( @@ -5736,8 +5700,8 @@ int ObDRWorker::try_migrate_to_unit( LOG_WARN("fail to generate migrate to unit task", KR(ret)); } } - } - } + } // end need_generate + } // end for } // no need to print task key, since the previous log contains that LOG_INFO("finish try migrate to unit", KR(ret), K(acc_dr_task)); diff --git a/src/rootserver/ob_disaster_recovery_worker.h b/src/rootserver/ob_disaster_recovery_worker.h index 07c4c6726..1a2ca5e4f 100755 --- a/src/rootserver/ob_disaster_recovery_worker.h +++ b/src/rootserver/ob_disaster_recovery_worker.h @@ -135,10 +135,15 @@ public: common::ObSArray &task_plan); private: - + // add task to queue of task mgr + // @param [out] task, the task to execute + // @param [out] acc_dr_task, acc_dr_task + int add_task_to_task_mgr_( + ObDRTask &task, + int64_t &acc_dr_task); // add task in queue in mgr and execute task // @param [in] task, the task to execute - int add_task_in_queue_and_execute_(const ObDRTask &task); + int add_task_in_queue_and_execute_(ObDRTask &task); // check ls exist and init dr_ls_info // @param [in] arg, task info // @param [out] dr_ls_info, target dr_ls_info to init @@ -294,14 +299,6 @@ private: const MemberChangeType member_change_type, int64_t &new_paxos_replica_number, bool &found); - enum LATaskType - { - RemovePaxos = 0, - RemoveNonPaxos, - AddReplica, - TypeTransform, - ModifyPaxosReplicaNumber, - }; enum class LATaskPrio : int64_t { @@ -324,7 +321,7 @@ private: LATask() {} virtual ~LATask() {} public: - virtual LATaskType get_task_type() const = 0; + virtual ObDRTaskType get_task_type() const = 0; virtual LATaskPrio get_task_priority() const = 0; virtual int64_t to_string(char *buf, const int64_t buf_len) const = 0; }; @@ -355,9 +352,9 @@ private: paxos_replica_number_(0) {} virtual ~RemoveReplicaLATask() {} public: - virtual LATaskType get_task_type() const override { return ObReplicaTypeCheck::is_paxos_replica_V2(replica_type_) - ? RemovePaxos - : RemoveNonPaxos; } + virtual ObDRTaskType get_task_type() const override { return ObReplicaTypeCheck::is_paxos_replica_V2(replica_type_) + ? ObDRTaskType::LS_REMOVE_PAXOS_REPLICA + : ObDRTaskType::LS_REMOVE_NON_PAXOS_REPLICA; } virtual LATaskPrio get_task_priority() const override { LATaskPrio priority = ObReplicaTypeCheck::is_paxos_replica_V2(replica_type_) ? LATaskPrio::LA_P_REMOVE_PAXOS @@ -396,7 +393,7 @@ private: paxos_replica_number_(0) {} virtual ~AddReplicaLATask() {} public: - virtual LATaskType get_task_type() const override{ return AddReplica; } + virtual ObDRTaskType get_task_type() const override { return ObDRTaskType::LS_ADD_REPLICA; } virtual LATaskPrio get_task_priority() const override { LATaskPrio priority = LATaskPrio::LA_P_MAX; if (common::REPLICA_TYPE_FULL == replica_type_) { @@ -452,7 +449,7 @@ private: paxos_replica_number_(0) {} virtual ~TypeTransformLATask() {} public: - virtual LATaskType get_task_type() const override { return TypeTransform; } + virtual ObDRTaskType get_task_type() const override { return ObDRTaskType::LS_TYPE_TRANSFORM; } virtual LATaskPrio get_task_priority() const override { LATaskPrio priority = LATaskPrio::LA_P_MAX; if (common::REPLICA_TYPE_FULL == dst_replica_type_ @@ -503,7 +500,7 @@ private: paxos_replica_number_(0) {} virtual ~ModifyPaxosReplicaNumberLATask() {} public: - virtual LATaskType get_task_type() const override { return ModifyPaxosReplicaNumber; } + virtual ObDRTaskType get_task_type() const override { return ObDRTaskType::LS_MODIFY_PAXOS_REPLICA_NUMBER; } virtual LATaskPrio get_task_priority() const override { LATaskPrio priority = LATaskPrio::LA_P_MODIFY_PAXOS_REPLICA_NUMBER; return priority; @@ -794,12 +791,6 @@ private: private: void reset_task_plans_() { display_tasks_.reset(); } - int check_task_already_exist( - const ObDRTaskKey &task_key, - const DRLSInfo &dr_ls_info, - const int64_t &priority, - bool &task_exist); - int check_whether_the_tenant_role_can_exec_dr_(const uint64_t tenant_id); int try_remove_permanent_offline_replicas( @@ -822,10 +813,9 @@ private: int check_can_generate_task( const int64_t acc_dr_task, const bool need_check_has_leader_while_remove_replica, - const bool is_high_priority_task, const ObAddr &server_addr, DRLSInfo &dr_ls_info, - ObDRTaskKey &task_key, + const ObDRTaskType &task_type, bool &can_generate); int construct_extra_infos_to_build_remove_replica_task( @@ -991,6 +981,8 @@ private: int generate_task_key( const DRLSInfo &dr_ls_info, + const common::ObAddr &task_exe_server, + const ObDRTaskType &task_type, ObDRTaskKey &task_key) const; int add_display_info(const ObLSReplicaTaskDisplayInfo &display_info); @@ -1006,25 +998,21 @@ private: int try_generate_remove_replica_locality_alignment_task( DRLSInfo &dr_ls_info, - const ObDRTaskKey &task_key, const LATask *task, int64_t &acc_dr_task); int try_generate_add_replica_locality_alignment_task( DRLSInfo &dr_ls_info, - const ObDRTaskKey &task_key, const LATask *task, int64_t &acc_dr_task); int try_generate_type_transform_locality_alignment_task( DRLSInfo &dr_ls_info, - const ObDRTaskKey &task_key, const LATask *task, int64_t &acc_dr_task); int try_generate_modify_paxos_replica_number_locality_alignment_task( DRLSInfo &dr_ls_info, - const ObDRTaskKey &task_key, const LATask *task, int64_t &acc_dr_task); diff --git a/src/share/config/ob_config_helper.cpp b/src/share/config/ob_config_helper.cpp index ac14adfd4..2e2bc0af6 100644 --- a/src/share/config/ob_config_helper.cpp +++ b/src/share/config/ob_config_helper.cpp @@ -1476,6 +1476,14 @@ bool ObConfigRegexpEngineChecker::check(const ObConfigItem &t) const return valid; } +bool ObConfigReplicaParallelMigrationChecker::check(const ObConfigItem &t) const +{ + ObString v_str(t.str()); + return 0 == v_str.case_compare("auto") + || 0 == v_str.case_compare("on") + || 0 == v_str.case_compare("off"); +} + bool ObConfigS3URLEncodeTypeChecker::check(const ObConfigItem &t) const { // When compliantRfc3986Encoding is set to true: diff --git a/src/share/config/ob_config_helper.h b/src/share/config/ob_config_helper.h index 2ce4612cd..c1232da7b 100644 --- a/src/share/config/ob_config_helper.h +++ b/src/share/config/ob_config_helper.h @@ -963,6 +963,16 @@ private: DISALLOW_COPY_AND_ASSIGN(ObConfigS3URLEncodeTypeChecker); }; +class ObConfigReplicaParallelMigrationChecker : public ObConfigChecker +{ +public: + ObConfigReplicaParallelMigrationChecker() {} + virtual ~ObConfigReplicaParallelMigrationChecker() {} + bool check(const ObConfigItem &t) const; +private: + DISALLOW_COPY_AND_ASSIGN(ObConfigReplicaParallelMigrationChecker); +}; + class ObConfigDegradationPolicyChecker : public ObConfigChecker { public: diff --git a/src/share/ob_rpc_struct.cpp b/src/share/ob_rpc_struct.cpp index af5ef5ce8..10887700f 100644 --- a/src/share/ob_rpc_struct.cpp +++ b/src/share/ob_rpc_struct.cpp @@ -4465,7 +4465,8 @@ int ObLSMigrateReplicaArg::init( const common::ObReplicaMember &discarded_data_source, const int64_t paxos_replica_number, const bool skip_change_member_list, - const common::ObReplicaMember &force_data_source) + const common::ObReplicaMember &force_data_source, + const bool prioritize_same_zone_src) { int ret = OB_SUCCESS; task_id_ = task_id; @@ -4477,6 +4478,7 @@ int ObLSMigrateReplicaArg::init( paxos_replica_number_ = paxos_replica_number; skip_change_member_list_ = skip_change_member_list; force_data_source_ = force_data_source; + prioritize_same_zone_src_ = prioritize_same_zone_src; return ret; } diff --git a/src/share/ob_rpc_struct.h b/src/share/ob_rpc_struct.h index 2cb4fbb7f..6871a24b2 100644 --- a/src/share/ob_rpc_struct.h +++ b/src/share/ob_rpc_struct.h @@ -4873,7 +4873,8 @@ public: const common::ObReplicaMember &discarded_data_source, const int64_t paxos_replica_number, const bool skip_change_member_list, - const common::ObReplicaMember &force_data_source); + const common::ObReplicaMember &force_data_source, + const bool prioritize_same_zone_src); TO_STRING_KV(K_(task_id), K_(tenant_id), diff --git a/src/share/parameter/ob_parameter_seed.ipp b/src/share/parameter/ob_parameter_seed.ipp index af0861590..54264a044 100644 --- a/src/share/parameter/ob_parameter_seed.ipp +++ b/src/share/parameter/ob_parameter_seed.ipp @@ -842,6 +842,15 @@ DEF_BOOL(enable_sys_unit_standalone, OB_CLUSTER_PARAMETER, "False", "Value: True:turned on False: turned off", ObParameterAttr(Section::LOAD_BALANCE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +DEF_STR_WITH_CHECKER(replica_parallel_migration_mode, OB_TENANT_PARAMETER, "auto", + common::ObConfigReplicaParallelMigrationChecker, + "specify the strategy for parallel migration of LS replicas. " + "'auto' means to allow parallel migration of LS replica of standby tenant " + "and prohibit the parallel migration of LS replica of primary tenant. " + "'on' means to allow parallel migration of LS replica of primary tenant and standby tenant. " + "'off' means to prohibit parallel migration of LS replica of primary tenant and standby tenant", + ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); + //// daily merge config // set to disable if don't want major freeze launch auto DEF_MOMENT(major_freeze_duty_time, OB_TENANT_PARAMETER, "02:00", @@ -2287,6 +2296,11 @@ ERRSIM_DEF_INT(errsim_rebuild_ls_id, OB_CLUSTER_PARAMETER, "0", "[0,)", ERRSIM_DEF_STR(errsim_rebuild_addr, OB_CLUSTER_PARAMETER, "", "rebuild addr (ip:port)", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); + +ERRSIM_DEF_BOOL(enable_parallel_migration, OB_CLUSTER_PARAMETER, "False", + "turn on parallel migration, observer preferentially choose same zone as src", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); + DEF_BOOL(_enable_adaptive_auto_dop, OB_CLUSTER_PARAMETER, "False", "Enable or disable adaptive auto dop feature.", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); diff --git a/src/storage/high_availability/ob_ls_migration.cpp b/src/storage/high_availability/ob_ls_migration.cpp index e18543472..9596f7cd8 100644 --- a/src/storage/high_availability/ob_ls_migration.cpp +++ b/src/storage/high_availability/ob_ls_migration.cpp @@ -1223,6 +1223,7 @@ int ObStartMigrationTask::choose_src_() omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id)); ObLS* ls = nullptr; ObLSHandle ls_handle; + bool use_c_replica_policy = false; if (OB_FAIL(get_local_ls_checkpoint_scn_(local_clog_checkpoint_scn))) { LOG_WARN("failed to get local ls checkpoint ts", K(ret)); } else if (!tenant_config.is_valid()) { @@ -1240,12 +1241,11 @@ int ObStartMigrationTask::choose_src_() ctx_->arg_.dst_, param.info_, param.is_first_c_replica_))) { LOG_WARN("failed to get member list.", K(ret), K(tenant_id), "ls_id", ctx_->arg_.ls_id_, "dst", ctx_->arg_.dst_); } else if (OB_FAIL(ObStorageHAChooseSrcHelper::get_policy_type(ctx_->arg_, tenant_id, - enable_choose_source_policy, str, param.info_.learner_list_, policy))) { + enable_choose_source_policy, str, param.info_.learner_list_, param.policy_, param.use_c_replica_policy_))) { LOG_WARN("failed to get policy type", K(ret), K(ctx_->arg_), K(tenant_id), - K(enable_choose_source_policy), K(str)); - } else if (OB_FAIL(choose_src_helper.init(param, policy, - storage_rpc_, &member_helper))) { - LOG_WARN("failed to init src provider.", K(ret), K(param), K(policy), KP(storage_rpc_)); + K(enable_choose_source_policy), K(str), K(param)); + } else if (OB_FAIL(choose_src_helper.init(param, storage_rpc_, &member_helper))) { + LOG_WARN("failed to init src provider.", K(ret), K(param), KP(storage_rpc_)); } else if (OB_FAIL(choose_src_helper.get_available_src(ctx_->arg_, src_info))) { LOG_WARN("failed to choose ob src", K(ret), K(tenant_id), K(ls_id), K(local_clog_checkpoint_scn), K(ctx_->arg_)); } else if (OB_FAIL(fetch_ls_info_(tenant_id, ls_id, src_info.src_addr_, ls_info))) { diff --git a/src/storage/high_availability/ob_rebuild_service.cpp b/src/storage/high_availability/ob_rebuild_service.cpp index 1d36490dd..88578fdb6 100644 --- a/src/storage/high_availability/ob_rebuild_service.cpp +++ b/src/storage/high_availability/ob_rebuild_service.cpp @@ -1147,6 +1147,7 @@ int ObLSRebuildMgr::generate_rebuild_task_() arg.priority_ = ObMigrationOpPriority::PRIO_MID; arg.src_ = src_replica_member; arg.type_ = ObMigrationOpType::REBUILD_LS_OP; + arg.prioritize_same_zone_src_ = false; if (OB_FAIL(ls->get_ls_migration_handler()->add_ls_migration_task(rebuild_ctx_.task_id_, arg))) { LOG_WARN("failed to add ls migration task", K(ret), K(arg), KPC(ls)); diff --git a/src/storage/high_availability/ob_storage_ha_src_provider.cpp b/src/storage/high_availability/ob_storage_ha_src_provider.cpp old mode 100644 new mode 100755 index 984edc96c..d6ca1e6f8 --- a/src/storage/high_availability/ob_storage_ha_src_provider.cpp +++ b/src/storage/high_availability/ob_storage_ha_src_provider.cpp @@ -335,6 +335,8 @@ ObStorageHASrcProvider::ObStorageHASrcProvider() : is_inited_(false), member_list_info_(), is_first_c_replica_(false), + chosen_policy_type_(ChooseSourcePolicy::MAX_POLICY), + use_c_replica_policy_(false), tenant_id_(OB_INVALID_ID), ls_id_(), type_(ObMigrationOpType::MAX_LS_OP), @@ -351,8 +353,8 @@ ObStorageHASrcProvider::~ObStorageHASrcProvider() storage_rpc_ = nullptr; } -int ObStorageHASrcProvider::init(const ObMigrationChooseSrcHelperInitParam ¶m, - const ChooseSourcePolicy policy_type, +int ObStorageHASrcProvider::init( + const ObMigrationChooseSrcHelperInitParam ¶m, storage::ObStorageRpc *storage_rpc, ObStorageHAGetMemberHelper *member_helper) { @@ -361,12 +363,9 @@ int ObStorageHASrcProvider::init(const ObMigrationChooseSrcHelperInitParam ¶ ret = OB_INIT_TWICE; LOG_WARN("storage ha src provider init twice", K(ret)); } else if (!param.is_valid() || OB_ISNULL(storage_rpc) - || policy_type < ChooseSourcePolicy::IDC - || policy_type >= ChooseSourcePolicy::MAX_POLICY || OB_ISNULL(member_helper)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("get invalid argument", K(ret), K(param), - KP(storage_rpc), K(policy_type), KP(member_helper)); + LOG_WARN("get invalid argument", K(ret), K(param), KP(storage_rpc), KP(member_helper)); } else { storage_rpc_ = storage_rpc; member_helper_ = member_helper; @@ -380,13 +379,45 @@ int ObStorageHASrcProvider::init(const ObMigrationChooseSrcHelperInitParam ¶ ls_id_ = param.ls_id_; type_ = param.arg_.type_; local_clog_checkpoint_scn_ = param.local_clog_checkpoint_scn_; - policy_type_ = policy_type; + policy_type_ = param.policy_; + chosen_policy_type_ = ChooseSourcePolicy::MAX_POLICY; + use_c_replica_policy_ = param.use_c_replica_policy_; is_first_c_replica_ = param.is_first_c_replica_; } } return ret; } +int ObStorageHASrcProvider::choose_ob_src(const ObMigrationOpArg &arg, common::ObAddr &chosen_src_addr) +{ + int ret = OB_SUCCESS; + chosen_src_addr.reset(); + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("provider is not init.", K(ret)); + } else if (!arg.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument!", K(ret), K(arg)); + } else { + bool must_choose_c_replica = use_c_replica_policy_; + if (OB_FAIL(inner_choose_ob_src(member_list_info_.leader_addr_, member_list_info_.learner_list_, + member_list_info_.member_list_, arg, must_choose_c_replica, chosen_src_addr))) { + if (OB_DATA_SOURCE_NOT_EXIST == ret && use_c_replica_policy_ && is_first_c_replica_) { + ret = OB_SUCCESS; + must_choose_c_replica = false; + LOG_INFO("try find F/R replica for first C replica", K(ret), K(member_list_info_), K(arg), K(must_choose_c_replica)); + if (OB_FAIL(inner_choose_ob_src(member_list_info_.leader_addr_, member_list_info_.learner_list_, + member_list_info_.member_list_, arg, must_choose_c_replica, chosen_src_addr))) { + LOG_WARN("failed to find F/R replica for first C replica", K(ret), K(member_list_info_), K(arg), K(must_choose_c_replica)); + } + } else { + LOG_WARN("failed to choose src", K(ret), K(member_list_info_), K(arg), K(must_choose_c_replica)); + } + } + } + return ret; +} + int ObStorageHASrcProvider::fetch_ls_meta_info_(const uint64_t tenant_id, const share::ObLSID &ls_id, const common::ObAddr &member_addr, obrpc::ObFetchLSMetaInfoResp &ls_meta_info) { @@ -443,6 +474,7 @@ int ObStorageHASrcProvider::check_replica_type_for_c_replica_( } else if (src.is_columnstore()) { is_replica_type_valid = true; } + LOG_INFO("check replica type for c replica", K(ret), K(addr), K(dst), K(learner_list), K(is_replica_type_valid), K(src.is_columnstore())); return ret; } @@ -496,6 +528,7 @@ int ObStorageHASrcProvider::check_replica_type_( const common::ObAddr &addr, const common::ObReplicaMember &dst, const common::GlobalLearnerList &learner_list, + const bool &must_choose_c_replica, bool &is_replica_type_valid) { int ret = OB_SUCCESS; @@ -503,32 +536,14 @@ int ObStorageHASrcProvider::check_replica_type_( if (!addr.is_valid() || !dst.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument!", K(ret), K(addr), K(dst)); - } else { - switch (policy_type_) { - case ChooseSourcePolicy::C_TYPE_REPLICA: { - // TODO(zhixing.yh) When the learner concurrency can be controlled, this function will be overwritten - is_replica_type_valid = true; - // if (OB_FAIL(check_replica_type_for_c_replica_(addr, dst, - // learner_list, is_replica_type_valid))) { - // LOG_WARN("failed to check replica type for c replica", K(ret), K(addr), K(dst), K(learner_list)); - // } - break; - }; - case ChooseSourcePolicy::IDC: - case ChooseSourcePolicy::REGION: - case ChooseSourcePolicy::CHECKPOINT: - case ChooseSourcePolicy::RECOMMEND: { - if (OB_FAIL(check_replica_type_for_normal_replica_(addr, dst, learner_list, is_replica_type_valid))) { - LOG_WARN("failed to check replica type", K(ret), K(addr), K(dst), K(learner_list)); - } - break; - } - default: { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected policy type", K(ret), K(policy_type_)); - } + } else if (must_choose_c_replica) { + if (OB_FAIL(check_replica_type_for_c_replica_(addr, dst, learner_list, is_replica_type_valid))) { + LOG_WARN("failed to check replica type for c replica", K(ret), K(addr), K(dst), K(learner_list)); } + } else if (OB_FAIL(check_replica_type_for_normal_replica_(addr, dst, learner_list, is_replica_type_valid))) { + LOG_WARN("failed to check replica type", K(ret), K(addr), K(dst), K(learner_list)); } + return ret; } @@ -625,7 +640,8 @@ int ObStorageHASrcProvider::get_palf_parent_addr_(const uint64_t tenant_id, cons int ObStorageHASrcProvider::check_replica_validity( const common::ObAddr &addr, const common::ObReplicaMember &dst, - const common::GlobalLearnerList &learner_list, obrpc::ObFetchLSMetaInfoResp &ls_info) + const common::GlobalLearnerList &learner_list, const bool& must_choose_c_replica, + obrpc::ObFetchLSMetaInfoResp &ls_info) { int ret = OB_SUCCESS; ls_info.reset(); @@ -636,7 +652,7 @@ int ObStorageHASrcProvider::check_replica_validity( } else if (!addr.is_valid() || !dst.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument!", K(ret), K(addr), K(dst)); - } else if (OB_FAIL(check_replica_type_(addr, dst, learner_list, is_replica_type_valid))) { + } else if (OB_FAIL(check_replica_type_(addr, dst, learner_list, must_choose_c_replica, is_replica_type_valid))) { LOG_WARN("failed to check replica type", K(ret), K(tenant_id_), K(ls_id_), K(addr), K(dst), K(learner_list)); } else if (!is_replica_type_valid) { @@ -662,17 +678,18 @@ int ObStorageHASrcProvider::check_replica_validity( } const char *ObStorageHASrcProvider::ObChooseSourcePolicyStr[static_cast(ChooseSourcePolicy::MAX_POLICY)] = { + "zone", "idc", "region", + "different_region" "checkpoint", "recommend", - "c_type_replica" }; const char *ObStorageHASrcProvider::get_policy_str(const ChooseSourcePolicy policy_type) { const char *str = ""; - if (policy_type >= ChooseSourcePolicy::MAX_POLICY || policy_type < ChooseSourcePolicy::IDC) { + if (policy_type >= ChooseSourcePolicy::MAX_POLICY || policy_type < ChooseSourcePolicy::ZONE) { str = "invalid_type"; } else { str = ObChooseSourcePolicyStr[static_cast(policy_type)]; @@ -680,6 +697,26 @@ const char *ObStorageHASrcProvider::get_policy_str(const ChooseSourcePolicy poli return str; } +int ObStorageHASrcProvider::get_policy_detailed_info_str(char *buf, const int64_t buf_len) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(buf) || buf_len <= 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid buf argument", K(ret), KP(buf), K(buf_len)); + } else { + ChooseSourcePolicyDetailedInfo detailed_info; + detailed_info.policy_type_ = policy_type_; + detailed_info.chosen_policy_type_ = chosen_policy_type_; + detailed_info.use_c_replica_policy_ = use_c_replica_policy_; + detailed_info.is_first_c_replica_ = is_first_c_replica_; + + if (OB_FAIL(databuff_printf(buf, buf_len, "%s", to_cstring(detailed_info)))) { + LOG_WARN("failed to print policy type", K(ret), K(detailed_info)); + } + } + return ret; +} + int ObStorageHASrcProvider::check_tenant_primary(bool &is_primary) { int ret = OB_SUCCESS; @@ -705,8 +742,8 @@ ObMigrationSrcByLocationProvider::~ObMigrationSrcByLocationProvider() { } -int ObMigrationSrcByLocationProvider::init(const ObMigrationChooseSrcHelperInitParam ¶m, - const ChooseSourcePolicy policy_type, +int ObMigrationSrcByLocationProvider::init( + const ObMigrationChooseSrcHelperInitParam ¶m, storage::ObStorageRpc *storage_rpc, ObStorageHAGetMemberHelper *member_helper) { @@ -714,14 +751,13 @@ int ObMigrationSrcByLocationProvider::init(const ObMigrationChooseSrcHelperInitP if (is_inited_) { ret = OB_INIT_TWICE; LOG_WARN("ObMigrationSrcByLocationProvider init twice", K(ret)); - } else if (!param.is_valid() || OB_ISNULL(storage_rpc) - || policy_type < ChooseSourcePolicy::IDC - || policy_type >= ChooseSourcePolicy::MAX_POLICY + } else if (!param.is_valid() + || OB_ISNULL(storage_rpc) || OB_ISNULL(member_helper)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("get invalid argument", K(ret), K(param), K(policy_type), KP(storage_rpc), KP(member_helper)); - } else if (OB_FAIL(ObStorageHASrcProvider::init(param, policy_type, storage_rpc, member_helper))) { - LOG_WARN("failed to init src provider", K(ret), K(param), KP(storage_rpc), K(policy_type), KP(member_helper)); + LOG_WARN("get invalid argument", K(ret), K(param), KP(storage_rpc), KP(member_helper)); + } else if (OB_FAIL(ObStorageHASrcProvider::init(param, storage_rpc, member_helper))) { + LOG_WARN("failed to init src provider", K(ret), K(param), KP(storage_rpc), KP(member_helper)); } else { locality_manager_ = GCTX.locality_manager_; is_inited_ = true; @@ -729,65 +765,80 @@ int ObMigrationSrcByLocationProvider::init(const ObMigrationChooseSrcHelperInitP return ret; } -int ObMigrationSrcByLocationProvider::choose_ob_src( - const ObMigrationOpArg &arg, common::ObAddr &chosen_src_addr) +int ObMigrationSrcByLocationProvider::inner_choose_ob_src( + const common::ObAddr &leader_addr, + const common::GlobalLearnerList &learner_list, + const common::ObIArray &addr_list, + const ObMigrationOpArg &arg, + const bool &must_choose_c_replica, + common::ObAddr &chosen_src_addr) { int ret = OB_SUCCESS; + common::ObArray sorted_addr_list; + int64_t zone_end_index = 0; + int64_t idc_end_index = 0; + int64_t region_end_index = 0; chosen_src_addr.reset(); if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("ObMigrationSrcByLocationProvider is not init.", K(ret)); - } else if (!arg.is_valid()) { + } else if (addr_list.empty() || !leader_addr.is_valid() || !arg.dst_.is_valid()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument!", K(ret), K(arg)); - } else if (OB_FAIL(inner_choose_ob_src( - member_list_info_.leader_addr_, member_list_info_.learner_list_, - member_list_info_.member_list_, arg, chosen_src_addr))) { - LOG_WARN("failed to inner choose ob src", K(ret), "tenant_id", get_tenant_id(), "ls_id", get_ls_id(), - K(member_list_info_), K(arg)); - } - return ret; -} - -int ObMigrationSrcByLocationProvider::inner_choose_ob_src( - const common::ObAddr &leader_addr, const common::GlobalLearnerList &learner_list, - const common::ObIArray &addr_list, const ObMigrationOpArg &arg, - common::ObAddr &choosen_src_addr) -{ - int ret = OB_SUCCESS; - common::ObArray sorted_addr_list; - int64_t idc_end_index = 0; - int64_t region_end_index = 0; - choosen_src_addr.reset(); - if (!is_inited_) { - ret = OB_NOT_INIT; - LOG_WARN("ObMigrationSrcByLocationProvider is not init.", K(ret)); - } else if (!leader_addr.is_valid() || addr_list.empty() || !arg.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument!", K(ret), K(addr_list), K(arg)); - } else if (OB_FAIL(divide_addr_list(addr_list, arg.dst_, sorted_addr_list, idc_end_index, region_end_index))) { + LOG_WARN("invalid argument!", K(ret), K(addr_list)); + } else if (OB_FAIL(divide_addr_list(addr_list, arg.dst_, sorted_addr_list, zone_end_index, idc_end_index, region_end_index))) { LOG_WARN("failed to divide addr", K(ret), K(addr_list), K(arg.dst_)); } else { - bool is_idc_policy = ChooseSourcePolicy::IDC == get_policy_type() ? true : false; - int64_t region_start_index = is_idc_policy ? idc_end_index + 1 : 0; - if (is_idc_policy && OB_FAIL(find_src(sorted_addr_list, 0/*idc_start_index*/, idc_end_index, - learner_list, leader_addr, arg.dst_, choosen_src_addr))) { // find in same idc - LOG_WARN("failed to find source in same idc scope", K(ret), K(sorted_addr_list), - K(idc_end_index), K(learner_list), K(leader_addr), K(arg.dst_)); - } else if (!choosen_src_addr.is_valid() && OB_FAIL(find_src(sorted_addr_list, region_start_index, region_end_index, - learner_list, leader_addr, arg.dst_, choosen_src_addr))) { // find in same region - LOG_WARN("failed to find source in same region scope", K(ret), K(sorted_addr_list), - K(region_start_index), K(region_end_index), K(learner_list), K(leader_addr), K(arg.dst_)); - } else if (!choosen_src_addr.is_valid() && OB_FAIL(find_src(sorted_addr_list, region_end_index + 1, addr_list.count() - 1, - learner_list, leader_addr, arg.dst_, choosen_src_addr))) { // find in different region - LOG_WARN("failed to find source in different region scope", K(ret), K(sorted_addr_list), - K(region_end_index), K(addr_list.count()), K(learner_list), K(leader_addr), K(arg.dst_)); - } else if (!choosen_src_addr.is_valid()) { + int64_t end_indices[LOCATION_POLICY_COUNT] = { + zone_end_index, + idc_end_index, + region_end_index, + addr_list.count() - 1 + }; + ChooseSourcePolicy chosen_policys[LOCATION_POLICY_COUNT] = { + ChooseSourcePolicy::ZONE, + ChooseSourcePolicy::IDC, + ChooseSourcePolicy::REGION, + ChooseSourcePolicy::DIFFERENT_REGION + }; + int64_t start_idx = 0; + int64_t end_pos = 0; + + switch (get_policy_type()) { + case ChooseSourcePolicy::ZONE: { end_pos = 0; break; } + case ChooseSourcePolicy::IDC: { end_pos = 1; break; } + case ChooseSourcePolicy::REGION: { end_pos = 2; break; } + default: { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("unexpected policy type", K(ret), K(get_policy_type())); + break; + } + } + + // use while loop to find src in + // SAME_ZONE -> SAME_IDC -> SAME_REGION -> DIFFERENT_REGION + while (OB_SUCC(ret) && end_pos < LOCATION_POLICY_COUNT) { + int64_t end_idx = end_indices[end_pos]; + if (OB_FAIL(find_src_in_sorted_addr_list_(sorted_addr_list, start_idx, end_idx, learner_list, leader_addr, arg.dst_, must_choose_c_replica, chosen_src_addr))) { + LOG_WARN("failed to find source in target scope", K(ret), K(sorted_addr_list), + K(start_idx), K(end_idx), K(end_pos), K(learner_list), K(leader_addr), K(arg.dst_)); + } else if (chosen_src_addr.is_valid()) { + // find source successfully, break + chosen_policy_type_ = chosen_policys[end_pos]; + LOG_INFO("success to find source", K(ret), K(chosen_src_addr), K(get_policy_type()), K(chosen_policy_type_)); + break; + } else { + start_idx = end_idx + 1; + end_pos++; + } + } + + if (OB_SUCC(ret) && end_pos == LOCATION_POLICY_COUNT) { ret = OB_DATA_SOURCE_NOT_EXIST; LOG_WARN("all region no available data source exist", K(ret), "tenant_id", get_tenant_id(), - "ls_id", get_ls_id(), K(leader_addr), K(learner_list), K(addr_list), K(arg)); + "ls_id", get_ls_id(), K(leader_addr), K(learner_list), K(addr_list), K(must_choose_c_replica)); } } + return ret; } @@ -795,6 +846,7 @@ int ObMigrationSrcByLocationProvider::divide_addr_list( const common::ObIArray &addr_list, const common::ObReplicaMember &dst, common::ObIArray &sorted_addr_list, + int64_t &zone_end_index, int64_t &idc_end_index, int64_t ®ion_end_index) { @@ -802,13 +854,18 @@ int ObMigrationSrcByLocationProvider::divide_addr_list( sorted_addr_list.reset(); common::ObRegion dst_region; common::ObIDC dst_idc; + common::ObZone dst_zone; common::ObRegion src_region; common::ObIDC src_idc; + common::ObZone src_zone; + int64_t same_zone_count = 0; int64_t same_idc_count = 0; int64_t same_region_count = 0; + common::ObArray same_zone_addr; common::ObArray same_idc_addr; common::ObArray same_region_addr; common::ObArray different_region_addr; + zone_end_index = 0; idc_end_index = 0; region_end_index = 0; if (!is_inited_) { @@ -820,22 +877,29 @@ int ObMigrationSrcByLocationProvider::divide_addr_list( } else if (OB_ISNULL(locality_manager_)) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("locality manager is null", K(ret), KP(locality_manager_)); - } else if (OB_FAIL(get_server_region_and_idc_(dst.get_server(), dst_region, dst_idc))) { - LOG_WARN("failed to get dst idc", K(ret), "addr", dst.get_server()); + } else if (OB_FAIL(get_server_geography_info_(dst.get_server(), dst_region, dst_idc, dst_zone))) { + LOG_WARN("failed to get dst geography info", K(ret), "addr", dst.get_server()); } else { LOG_INFO("succeed to get dst region and idc", K(dst_idc), K(dst_region)); for (int64_t i = 0; OB_SUCC(ret) && i < addr_list.count(); ++i) { - if (OB_FAIL(get_server_region_and_idc_(addr_list.at(i), src_region, src_idc))) { - LOG_WARN("failed to get src region and idc", K(ret), "addr", addr_list.at(i), K(dst)); + if (OB_FAIL(get_server_geography_info_(addr_list.at(i), src_region, src_idc, src_zone))) { + LOG_WARN("failed to get src geography info", K(ret), "addr", addr_list.at(i), K(dst)); + } + // TODO(zhixing.yh) need to confirm the relationship of zone and idc + else if (src_region == dst_region && src_idc == dst_idc && src_zone == dst_zone) { + same_zone_count++; + if (OB_FAIL(same_zone_addr.push_back(addr_list.at(i)))) { + LOG_WARN("failed to add zone addr", K(ret), "addr", addr_list.at(i), K(dst)); + } } else if (src_region == dst_region && src_idc == dst_idc) { // get distinct region server same_idc_count++; if (OB_FAIL(same_idc_addr.push_back(addr_list.at(i)))) { - LOG_WARN("failed to add distinct region addr", K(ret), "addr", addr_list.at(i), K(dst)); + LOG_WARN("failed to add idc addr", K(ret), "addr", addr_list.at(i), K(dst)); } } else if (src_region == dst_region) { same_region_count++; if (OB_FAIL(same_region_addr.push_back(addr_list.at(i)))) { - LOG_WARN("failed to add distinct region addr", K(ret), "addr", addr_list.at(i), K(dst)); + LOG_WARN("failed to add region addr", K(ret), "addr", addr_list.at(i), K(dst)); } } else if (src_region != dst_region) { if (OB_FAIL(different_region_addr.push_back(addr_list.at(i)))) { @@ -848,6 +912,8 @@ int ObMigrationSrcByLocationProvider::divide_addr_list( } if (OB_FAIL(ret)) { // do nothing + } else if (OB_FAIL(common::append(sorted_addr_list, same_zone_addr))) { + LOG_WARN("failed to append same_zone_addr to sorted_addr_list", K(ret), K(sorted_addr_list), K(same_zone_addr)); } else if (OB_FAIL(common::append(sorted_addr_list, same_idc_addr))) { LOG_WARN("failed to append same_idc_addr to sorted_addr_list", K(ret), K(sorted_addr_list), K(same_idc_addr)); } else if (OB_FAIL(common::append(sorted_addr_list, same_region_addr))) { @@ -855,23 +921,25 @@ int ObMigrationSrcByLocationProvider::divide_addr_list( } else if (OB_FAIL(common::append(sorted_addr_list, different_region_addr))) { LOG_WARN("failed to append different_region_addr to sorted_addr_list", K(ret), K(sorted_addr_list), K(different_region_addr)); } else { - idc_end_index = same_idc_count - 1; - region_end_index = same_idc_count + same_region_count - 1; - LOG_INFO("succeed to divide addr list", K(ret), K(sorted_addr_list), K(same_idc_addr), - K(same_region_addr), K(different_region_addr), K(idc_end_index), K(region_end_index)); + zone_end_index = same_zone_count - 1; + idc_end_index = same_zone_count + same_idc_count - 1; + region_end_index = same_zone_count + same_idc_count + same_region_count - 1; + LOG_INFO("succeed to divide addr list", K(ret), K(sorted_addr_list), K(same_zone_addr), K(same_idc_addr), + K(same_region_addr), K(different_region_addr), K(zone_end_index), K(idc_end_index), K(region_end_index)); } } return ret; } -int ObMigrationSrcByLocationProvider::find_src( +int ObMigrationSrcByLocationProvider::find_src_in_sorted_addr_list_( const common::ObIArray &addr_list, const int64_t start_index, const int64_t end_index, const common::GlobalLearnerList &learner_list, const common::ObAddr &leader_addr, const common::ObReplicaMember &dst, - common::ObAddr &choosen_src_addr) + const bool &must_choose_c_replica, + common::ObAddr &chosen_src_addr) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -880,7 +948,7 @@ int ObMigrationSrcByLocationProvider::find_src( obrpc::ObFetchLSMetaInfoResp ls_info; common::ObArray candidate_addr_list; int64_t leader_index = -1; - choosen_src_addr.reset(); + chosen_src_addr.reset(); bool is_primary = false; LOG_INFO("start find source", K(start_index), K(end_index)); if (!is_inited_) { @@ -894,7 +962,7 @@ int ObMigrationSrcByLocationProvider::find_src( LOG_WARN("failed to check tenant primary", K(ret), "tenant_id", get_tenant_id()); } else { for (int64_t i = start_index; OB_SUCC(ret) && i <= end_index && i < addr_list.count(); ++i) { - if (OB_TMP_FAIL(check_replica_validity(addr_list.at(i), dst, learner_list, ls_info))) { + if (OB_TMP_FAIL(check_replica_validity(addr_list.at(i), dst, learner_list, must_choose_c_replica, ls_info))) { if (OB_DATA_SOURCE_NOT_EXIST == tmp_ret) { // overwrite ret ret = tmp_ret; @@ -921,25 +989,26 @@ int ObMigrationSrcByLocationProvider::find_src( "ls_id", get_ls_id(), K(addr_list), K(dst), K(learner_list)); } else if (!candidate_addr_list.empty()) { int64_t num = candidate_addr_list.count(); - choosen_src_addr = addr_list.at(candidate_addr_list.at(rand() % num)); + chosen_src_addr = addr_list.at(candidate_addr_list.at(rand() % num)); LOG_INFO("found available data follower source in this area", "tenant_id", get_tenant_id(), - "ls_id", get_ls_id(), K(addr_list), K(dst), K(learner_list), K(choosen_src_addr)); + "ls_id", get_ls_id(), K(addr_list), K(dst), K(learner_list), K(chosen_src_addr)); } else { - choosen_src_addr = addr_list.at(leader_index); + chosen_src_addr = addr_list.at(leader_index); LOG_INFO("found available data leader source in this area", "tenant_id", get_tenant_id(), - "ls_id", get_ls_id(), K(addr_list), K(dst), K(learner_list), K(choosen_src_addr), K(leader_index)); + "ls_id", get_ls_id(), K(addr_list), K(dst), K(learner_list), K(chosen_src_addr), K(leader_index)); } } } return ret; } -int ObMigrationSrcByLocationProvider::get_server_region_and_idc_( - const common::ObAddr &addr, common::ObRegion ®ion, common::ObIDC &idc) +int ObMigrationSrcByLocationProvider::get_server_geography_info_( + const common::ObAddr &addr, common::ObRegion ®ion, common::ObIDC &idc, common::ObZone &zone) { int ret = OB_SUCCESS; region.reset(); idc.reset(); + zone.reset(); if (OB_ISNULL(locality_manager_)) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("locality manager is null", K(ret), KP(locality_manager_)); @@ -947,8 +1016,15 @@ int ObMigrationSrcByLocationProvider::get_server_region_and_idc_( LOG_WARN("failed to get src region", K(ret), K(addr)); } else if (OB_FAIL(locality_manager_->get_server_idc(addr, idc))) { LOG_WARN("failed to get src idc", K(ret), K(addr)); + } else if (OB_FAIL(locality_manager_->get_server_zone(addr, zone))) { + if (OB_ENTRY_NOT_EXIST == ret) { + ret = OB_SUCCESS; + LOG_INFO("not set zone, default same.", K(zone)); + } else { + LOG_WARN("failed to get src idc", K(ret), K(addr)); + } } else { - LOG_INFO("succeed to get region and idc", K(addr), K(region), K(idc)); + LOG_INFO("succeed to get region and idc", K(addr), K(region), K(idc), K(zone)); } return ret; } @@ -975,8 +1051,8 @@ ObMigrationSrcByCheckpointProvider::~ObMigrationSrcByCheckpointProvider() { } -int ObMigrationSrcByCheckpointProvider::init(const ObMigrationChooseSrcHelperInitParam ¶m, - const ChooseSourcePolicy policy_type, +int ObMigrationSrcByCheckpointProvider::init( + const ObMigrationChooseSrcHelperInitParam ¶m, storage::ObStorageRpc *storage_rpc, ObStorageHAGetMemberHelper *member_helper) { @@ -984,48 +1060,30 @@ int ObMigrationSrcByCheckpointProvider::init(const ObMigrationChooseSrcHelperIni if (is_inited_) { ret = OB_INIT_TWICE; LOG_WARN("ObMigrationSrcByCheckpointProvider init twice", K(ret)); - } else if (!param.is_valid() || OB_ISNULL(storage_rpc) - || policy_type < ChooseSourcePolicy::IDC - || policy_type >= ChooseSourcePolicy::MAX_POLICY + } else if (!param.is_valid() + || OB_ISNULL(storage_rpc) || OB_ISNULL(member_helper)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("get invalid argument", K(ret), K(param), K(policy_type), KP(storage_rpc), KP(member_helper)); - } else if (OB_FAIL(ObStorageHASrcProvider::init(param, policy_type, storage_rpc, member_helper))) { - LOG_WARN("failed to init src provider", K(ret), K(param), KP(storage_rpc), K(policy_type), KP(member_helper)); + LOG_WARN("get invalid argument", K(ret), K(param), KP(storage_rpc), KP(member_helper)); + } else if (OB_FAIL(ObStorageHASrcProvider::init(param, storage_rpc, member_helper))) { + LOG_WARN("failed to init src provider", K(ret), K(param), KP(storage_rpc), KP(member_helper)); } else { is_inited_ = true; } return ret; } -int ObMigrationSrcByCheckpointProvider::choose_ob_src( - const ObMigrationOpArg &arg, common::ObAddr &chosen_src_addr) -{ - int ret = OB_SUCCESS; - chosen_src_addr.reset(); - if (!is_inited_) { - ret = OB_NOT_INIT; - LOG_WARN("ObMigrationSrcByCheckpointProvider is not init.", K(ret)); - } else if (!arg.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument!", K(ret), K(arg)); - } else if (OB_FAIL(inner_choose_ob_src_( - member_list_info_.leader_addr_, member_list_info_.learner_list_, - member_list_info_.member_list_, arg, chosen_src_addr))) { - LOG_WARN("failed to inner choose ob src", K(ret), "tenant_id", get_tenant_id(), "ls_id", get_ls_id(), - K(member_list_info_), K(arg)); - } - return ret; -} - -int ObMigrationSrcByCheckpointProvider::inner_choose_ob_src_( - const common::ObAddr &leader_addr, const common::GlobalLearnerList &learner_list, - const common::ObIArray &addr_list, const ObMigrationOpArg &arg, - common::ObAddr &choosen_src_addr) +int ObMigrationSrcByCheckpointProvider::inner_choose_ob_src( + const common::ObAddr &leader_addr, + const common::GlobalLearnerList &learner_list, + const common::ObIArray &addr_list, + const ObMigrationOpArg &arg, + const bool &must_choose_c_replica, + common::ObAddr &chosen_src_addr) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - choosen_src_addr.reset(); + chosen_src_addr.reset(); share::SCN max_clog_checkpoint_scn = share::SCN::min_scn(); int64_t choose_member_idx = -1; obrpc::ObFetchLSMetaInfoResp ls_info; @@ -1037,7 +1095,7 @@ int ObMigrationSrcByCheckpointProvider::inner_choose_ob_src_( LOG_WARN("invalid argument!", K(ret), K(leader_addr), K(addr_list), K(arg)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < addr_list.count(); ++i) { - if (OB_TMP_FAIL(check_replica_validity(addr_list.at(i), arg.dst_, learner_list, ls_info))) { + if (OB_TMP_FAIL(check_replica_validity(addr_list.at(i), arg.dst_, learner_list, must_choose_c_replica, ls_info))) { // OB_DATA_SOURCE_NOT_EXIST make migration exit. It is used to return when check restore fail. // Only check restore fail use OB_DATA_SOURCE_NOT_EXIST currently if (OB_DATA_SOURCE_NOT_EXIST == tmp_ret) { @@ -1063,7 +1121,8 @@ int ObMigrationSrcByCheckpointProvider::inner_choose_ob_src_( LOG_WARN("no available data source exist", K(ret), "tenant_id", get_tenant_id(), "ls_id", get_ls_id(), K(learner_list), K(addr_list)); } else { - choosen_src_addr = addr_list.at(choose_member_idx); + chosen_src_addr = addr_list.at(choose_member_idx); + chosen_policy_type_ = ChooseSourcePolicy::CHECKPOINT; } } } @@ -1081,8 +1140,8 @@ ObRSRecommendSrcProvider::~ObRSRecommendSrcProvider() { } -int ObRSRecommendSrcProvider::init(const ObMigrationChooseSrcHelperInitParam ¶m, - const ChooseSourcePolicy policy_type, +int ObRSRecommendSrcProvider::init( + const ObMigrationChooseSrcHelperInitParam ¶m, storage::ObStorageRpc *storage_rpc, ObStorageHAGetMemberHelper *member_helper) { @@ -1090,57 +1149,27 @@ int ObRSRecommendSrcProvider::init(const ObMigrationChooseSrcHelperInitParam &pa if (is_inited_) { ret = OB_INIT_TWICE; LOG_WARN("ObRSRecommendSrcProvider init twice", K(ret)); - } else if (!param.is_valid() || OB_ISNULL(storage_rpc) - || policy_type < ChooseSourcePolicy::IDC - || policy_type >= ChooseSourcePolicy::MAX_POLICY + } else if (!param.is_valid() + || OB_ISNULL(storage_rpc) || OB_ISNULL(member_helper)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("get invalid argument", K(ret), K(param), K(policy_type), - KP(storage_rpc), KP(member_helper)); - } else if (OB_FAIL(ObStorageHASrcProvider::init(param, policy_type, - storage_rpc, member_helper))) { - LOG_WARN("failed to init src provider", K(ret), K(param), KP(storage_rpc), K(policy_type), KP(member_helper)); + LOG_WARN("get invalid argument", K(ret), K(param), KP(storage_rpc), KP(member_helper)); + } else if (OB_FAIL(ObStorageHASrcProvider::init(param, storage_rpc, member_helper))) { + LOG_WARN("failed to init src provider", K(ret), K(param), KP(storage_rpc), KP(member_helper)); } else { is_inited_ = true; } return ret; } -int ObRSRecommendSrcProvider::check_replica_validity_( - const int64_t cluster_id, - const common::ObIArray &addr_list, - const common::ObAddr &addr, const common::ObReplicaMember &dst, - const common::GlobalLearnerList &learner_list) -{ - int ret = OB_SUCCESS; - obrpc::ObFetchLSMetaInfoResp ls_info; - int64_t gconf_cluster_id = GCONF.cluster_id; - if (addr_list.empty() || !addr.is_valid() || !dst.is_valid() || cluster_id != gconf_cluster_id) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument!", K(ret), K(addr_list), K(addr), K(dst), - K(cluster_id), K(gconf_cluster_id)); - } else if (OB_FAIL(check_replica_validity(addr, dst, learner_list, ls_info))) { - LOG_WARN("failed to check replica validity", K(ret), K(addr), K(dst), K(learner_list)); - } else { - bool is_exist = false; - for (int64_t i = 0; OB_SUCC(ret) && i < addr_list.count(); i++) { - if (addr_list.at(i) == addr) { - is_exist = true; - break; - } - } - if (!is_exist) { - ret = OB_DATA_SOURCE_NOT_EXIST; - LOG_WARN("addr not in addr_list", K(ret), K(addr), K(addr_list)); - } - } - return ret; -} - int ObRSRecommendSrcProvider::choose_ob_src( const ObMigrationOpArg &arg, common::ObAddr &chosen_src_addr) { int ret = OB_SUCCESS; + // for rs selected source, only need to check once: + // 1. If the source is F/R/first C, the selected source can not only be C (F: F, R: F/R, C: F/R/C), must_choose_c_replica = false + // 2. If the source is C but not the first one in the learner list, the selected source must be C, must_choose_c_replica = true + bool must_choose_c_replica = use_c_replica_policy_ && !is_first_c_replica_; chosen_src_addr.reset(); if (!is_inited_) { ret = OB_NOT_INIT; @@ -1148,159 +1177,70 @@ int ObRSRecommendSrcProvider::choose_ob_src( } else if (!arg.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument!", K(ret), K(arg)); - } else if (OB_FAIL(check_replica_validity_(arg.cluster_id_, member_list_info_.member_list_, - arg.data_src_.get_server(), arg.dst_, member_list_info_.learner_list_))) { - LOG_WARN("failed to check replica validity", K(ret), K(member_list_info_), K(arg)); - } else { - chosen_src_addr = arg.data_src_.get_server(); - } - return ret; -} -/** - * ------------------------------ObCTypeReplicaSrcProvider--------------------- - */ -ObCTypeReplicaSrcProvider::ObCTypeReplicaSrcProvider() - : ObStorageHASrcProvider() -{ -} - -ObCTypeReplicaSrcProvider::~ObCTypeReplicaSrcProvider() -{ -} - -int ObCTypeReplicaSrcProvider::init(const ObMigrationChooseSrcHelperInitParam ¶m, - const ChooseSourcePolicy policy_type, - storage::ObStorageRpc *storage_rpc, - ObStorageHAGetMemberHelper *member_helper) -{ - int ret = OB_SUCCESS; - if (is_inited_) { - ret = OB_INIT_TWICE; - LOG_WARN("ObCTypeReplicaSrcProvider init twice", K(ret)); - } else if (!param.is_valid() || OB_ISNULL(storage_rpc) - || policy_type < ChooseSourcePolicy::IDC - || policy_type >= ChooseSourcePolicy::MAX_POLICY - || OB_ISNULL(member_helper)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("get invalid argument", K(ret), K(param), K(policy_type), - KP(storage_rpc), KP(member_helper)); - } else if (OB_FAIL(ObStorageHASrcProvider::init(param, policy_type, - storage_rpc, member_helper))) { - LOG_WARN("failed to init src provider", K(ret), K(param), KP(storage_rpc), K(policy_type), KP(member_helper)); - } else { - is_inited_ = true; + } else if (OB_FAIL(inner_choose_ob_src(member_list_info_.leader_addr_, member_list_info_.learner_list_, + member_list_info_.member_list_, arg, must_choose_c_replica, chosen_src_addr))) { + LOG_WARN("failed to choose ob src", K(ret), K(member_list_info_), K(arg)); } return ret; } -int ObCTypeReplicaSrcProvider::get_c_replica_( - const common::ObAddr &addr, - const common::GlobalLearnerList &learner_list, - ObIArray &c_replica_list) const +int ObRSRecommendSrcProvider::inner_choose_ob_src( + const common::ObAddr &leader_addr, const common::GlobalLearnerList &learner_list, + const common::ObIArray &addr_list, const ObMigrationOpArg &arg, const bool &must_choose_c_replica, + common::ObAddr &chosen_src_addr) { int ret = OB_SUCCESS; - ObMember src; - if (!addr.is_valid() || !learner_list.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument!", K(ret), K(addr), K(learner_list)); - } else if (!learner_list.contains(addr)) { - } else if (OB_FAIL(learner_list.get_learner_by_addr(addr, src))) { - LOG_WARN("failed to get learner by addr", KR(ret), K(addr)); - } else if (src.is_columnstore() && OB_FAIL(c_replica_list.push_back(addr))) { - LOG_WARN("failed to push back addr", K(ret), K(addr), K(src), K(learner_list)); - } - return ret; -} + UNUSED(leader_addr); -int ObCTypeReplicaSrcProvider::inner_choose_ob_src_( - const common::ObIArray &addr_list, - const common::GlobalLearnerList &learner_list, - const ObMigrationOpArg &arg, common::ObAddr &choosen_src_addr) -{ - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - choosen_src_addr.reset(); - common::ObArray candidate_addr_list; - obrpc::ObFetchLSMetaInfoResp ls_info; - ObArray all_c_replica_list; - ObArray valid_c_replica_list; if (!is_inited_) { ret = OB_NOT_INIT; - LOG_WARN("ObCTypeReplicaSrcProvider is not init.", K(ret)); - } else if (addr_list.empty() || !learner_list.is_valid() || !arg.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument!", K(ret), K(addr_list), K(learner_list), K(arg)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < addr_list.count(); ++i) { - if (OB_FAIL(get_c_replica_(addr_list.at(i), learner_list, all_c_replica_list))) { - LOG_WARN("failed to get all c replica", K(ret), "addr", addr_list.at(i), K(learner_list)); - } else if (OB_TMP_FAIL(check_replica_validity(addr_list.at(i), arg.dst_, learner_list, ls_info))) { - if (OB_DATA_SOURCE_NOT_EXIST == tmp_ret) { - // overwrite ret - ret = tmp_ret; - LOG_WARN("failed to check replica validity", K(ret), K(tmp_ret), "tenant_id", get_tenant_id(), - "ls_id", get_ls_id(), "addr", addr_list.at(i), K(arg.dst_), K(learner_list)); - break; - } else { - LOG_WARN("this address is not suitable.", K(ret), K(tmp_ret), "tenant_id", get_tenant_id(), - "ls_id", get_ls_id(), "addr", addr_list.at(i), K(arg.dst_), K(learner_list)); - } - } else if (OB_FAIL(candidate_addr_list.push_back(i))) { - LOG_WARN("failed to push back to candidate_addr_list", K(ret), "index", i, "addr", addr_list.at(i)); - } else if (OB_FAIL(get_c_replica_(addr_list.at(i), learner_list, valid_c_replica_list))) { - LOG_WARN("failed to get valid c replica", K(ret), "addr", addr_list.at(i), K(learner_list)); - } - } - if (OB_SUCC(ret)) { - if (candidate_addr_list.empty() && valid_c_replica_list.empty()) { - ret = OB_DATA_SOURCE_NOT_EXIST; - LOG_INFO("no available data source exist in this area", K(ret), "tenant_id", get_tenant_id(), - "ls_id", get_ls_id(), K(arg.dst_), K(addr_list), K(learner_list)); - } else if (candidate_addr_list.empty() && !valid_c_replica_list.empty()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("this case is unexpected", K(ret), K(valid_c_replica_list), "tenant_id", get_tenant_id(), - "ls_id", get_ls_id(), K(arg.dst_), K(addr_list), K(learner_list)); - } else if (!valid_c_replica_list.empty()) { - // if C replica exist, choose one of them - int64_t num = valid_c_replica_list.count(); - choosen_src_addr = valid_c_replica_list.at(rand() % num); - LOG_INFO("found available C replica source in this area", "tenant_id", get_tenant_id(), - "ls_id", get_ls_id(), K(arg.dst_), K(learner_list), K(choosen_src_addr), K(addr_list), K(valid_c_replica_list)); - } else if (!all_c_replica_list.empty() && !is_first_c_replica_) { - // C replica exist, but all C replica is not available. If dst is not first C replica, fail this migration - ret = OB_DATA_SOURCE_NOT_EXIST; - LOG_INFO("C replica exist, but all C replica is not available", K(ret), "tenant_id", get_tenant_id(), - "ls_id", get_ls_id(), K(arg.dst_), K(learner_list), K(addr_list), K(all_c_replica_list), K(valid_c_replica_list), K(is_first_c_replica_)); - } else { - // If no C replica exist or dst is first C replica and other C replica invalid, choose other F/R replica - int64_t num = candidate_addr_list.count(); - choosen_src_addr = addr_list.at(candidate_addr_list.at(rand() % num)); - LOG_INFO("no available C replica, but found other available source in this area", "tenant_id", get_tenant_id(), - "ls_id", get_ls_id(), K(arg.dst_), K(learner_list), K(choosen_src_addr), K(addr_list), K(is_first_c_replica_)); - } - } - } - return ret; -} - -int ObCTypeReplicaSrcProvider::choose_ob_src( - const ObMigrationOpArg &arg, common::ObAddr &chosen_src_addr) -{ - int ret = OB_SUCCESS; - chosen_src_addr.reset(); - if (!is_inited_) { - ret = OB_NOT_INIT; - LOG_WARN("ObCTypeReplicaSrcProvider is not init.", K(ret)); + LOG_WARN("ObRSRecommendSrcProvider is not init.", K(ret)); } else if (!arg.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument!", K(ret), K(arg)); - } else if (OB_FAIL(inner_choose_ob_src_( - member_list_info_.member_list_, member_list_info_.learner_list_, arg, chosen_src_addr))) { - LOG_WARN("failed to inner choose ob src", K(ret), "tenant_id", get_tenant_id(), "ls_id", get_ls_id(), - K(member_list_info_), K(arg)); + } else if (OB_FAIL(check_replica_validity_(arg.cluster_id_, addr_list, + arg.data_src_.get_server(), arg.dst_, learner_list, must_choose_c_replica))) { + LOG_WARN("failed to check replica validity", K(ret), K(member_list_info_), K(arg)); + } else { + chosen_src_addr = arg.data_src_.get_server(); + chosen_policy_type_ = ChooseSourcePolicy::RECOMMEND; } return ret; } + +int ObRSRecommendSrcProvider::check_replica_validity_( + const int64_t cluster_id, + const common::ObIArray &addr_list, + const common::ObAddr &addr, const common::ObReplicaMember &dst, + const common::GlobalLearnerList &learner_list, const bool &must_choose_c_replica) +{ + int ret = OB_SUCCESS; + obrpc::ObFetchLSMetaInfoResp ls_info; + int64_t gconf_cluster_id = GCONF.cluster_id; + if (addr_list.empty() || !addr.is_valid() || !dst.is_valid() || cluster_id != gconf_cluster_id) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument!", K(ret), K(addr_list), K(addr), K(dst), + K(cluster_id), K(gconf_cluster_id)); + } else { + if (OB_FAIL(check_replica_validity(addr, dst, learner_list, must_choose_c_replica, ls_info))) { + LOG_WARN("failed to check replica validity", K(ret), K(addr), K(dst), K(learner_list)); + } else { + bool is_exist = false; + for (int64_t i = 0; OB_SUCC(ret) && i < addr_list.count(); i++) { + if (addr_list.at(i) == addr) { + is_exist = true; + break; + } + } + if (!is_exist) { + ret = OB_DATA_SOURCE_NOT_EXIST; + LOG_WARN("addr not in addr_list", K(ret), K(addr), K(addr_list)); + } + } + } + return ret; +} + /** * ------------------------------ObStorageHAChooseSrcHelper--------------------- */ @@ -1321,8 +1261,9 @@ ObStorageHAChooseSrcHelper::~ObStorageHAChooseSrcHelper() storage_rpc_ = nullptr; } -int ObStorageHAChooseSrcHelper::init(const ObMigrationChooseSrcHelperInitParam ¶m, - const ObStorageHASrcProvider::ChooseSourcePolicy policy, storage::ObStorageRpc *storage_rpc, +int ObStorageHAChooseSrcHelper::init( + const ObMigrationChooseSrcHelperInitParam ¶m, + storage::ObStorageRpc *storage_rpc, ObStorageHAGetMemberHelper *member_helper) { int ret = OB_SUCCESS; @@ -1330,46 +1271,35 @@ int ObStorageHAChooseSrcHelper::init(const ObMigrationChooseSrcHelperInitParam & ret = OB_INIT_TWICE; LOG_WARN("storage ha src helper init twice", K(ret)); } else if (!param.is_valid() - || policy >= ObStorageHASrcProvider::ChooseSourcePolicy::MAX_POLICY - || policy < ObStorageHASrcProvider::ChooseSourcePolicy::IDC || OB_ISNULL(storage_rpc) || OB_ISNULL(member_helper)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument!", K(ret), K(param), KP(storage_rpc), KP(member_helper)); } else { - switch (policy) { + switch (param.policy_) { + case ObStorageHASrcProvider::ChooseSourcePolicy::ZONE: case ObStorageHASrcProvider::ChooseSourcePolicy::IDC: case ObStorageHASrcProvider::ChooseSourcePolicy::REGION: { - if (OB_FAIL(init_choose_source_by_location_provider_(param, policy, - storage_rpc, member_helper))) { - LOG_WARN("failed to init choose source by location provider", K(ret), K(param), K(policy)); + if (OB_FAIL(init_choose_source_by_location_provider_(param, storage_rpc, member_helper))) { + LOG_WARN("failed to init choose source by location provider", K(ret), K(param)); } break; } case ObStorageHASrcProvider::ChooseSourcePolicy::CHECKPOINT: { - if (OB_FAIL(init_choose_source_by_checkpoint_provider_(param, - storage_rpc, member_helper))) { - LOG_WARN("failed to init choose source by checkpoint provider", K(ret), K(param), K(policy)); + if (OB_FAIL(init_choose_source_by_checkpoint_provider_(param, storage_rpc, member_helper))) { + LOG_WARN("failed to init choose source by checkpoint provider", K(ret), K(param)); } break; } case ObStorageHASrcProvider::ChooseSourcePolicy::RECOMMEND: { - if (OB_FAIL(init_rs_recommend_source_provider_(param, - storage_rpc, member_helper))) { - LOG_WARN("failed to init choose source by rs recommend provider", K(ret), K(param), K(policy)); - } - break; - } - case ObStorageHASrcProvider::ChooseSourcePolicy::C_TYPE_REPLICA: { - if (OB_FAIL(init_c_type_replica_source_provider_(param, - storage_rpc, member_helper))) { - LOG_WARN("failed to init c replica choose source provider", K(ret), K(param), K(policy)); + if (OB_FAIL(init_rs_recommend_source_provider_(param, storage_rpc, member_helper))) { + LOG_WARN("failed to init choose source by rs recommend provider", K(ret), K(param)); } break; } default: { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected choose source policy", K(ret), K(param), K(policy)); + LOG_WARN("unexpected choose source policy", K(ret), K(param)); } } } @@ -1398,18 +1328,27 @@ int ObStorageHAChooseSrcHelper::get_available_src(const ObMigrationOpArg &arg, O LOG_INFO("succeed to choose src", K(src_info)); errsim_test_(arg, src_info); } + + int tmp_ret = OB_SUCCESS; + char policy_info_str[MAX_ROOTSERVICE_EVENT_VALUE_LENGTH] = {0}; + if (OB_TMP_FAIL(provider_->get_policy_detailed_info_str(policy_info_str, MAX_ROOTSERVICE_EVENT_VALUE_LENGTH))) { + LOG_WARN("failed to get detailed policy str", K(tmp_ret), K(policy_info_str)); + } + SERVER_EVENT_ADD("storage_ha", "choose_src", "tenant_id", provider_->get_tenant_id(), "ls_id", provider_->get_ls_id().id(), "src_addr", src_info.src_addr_, "dst_addr", arg.dst_.get_server(), - "op_type", ObMigrationOpType::get_str(provider_->get_type())); + "op_type", ObMigrationOpType::get_str(provider_->get_migration_op_type()), + "policy_info", policy_info_str); return ret; } int ObStorageHAChooseSrcHelper::init_rs_recommend_source_provider_( const ObMigrationChooseSrcHelperInitParam ¶m, - storage::ObStorageRpc *storage_rpc, ObStorageHAGetMemberHelper *member_helper) + storage::ObStorageRpc *storage_rpc, + ObStorageHAGetMemberHelper *member_helper) { int ret = OB_SUCCESS; void *buf = nullptr; @@ -1421,8 +1360,8 @@ int ObStorageHAChooseSrcHelper::init_rs_recommend_source_provider_( } else if (OB_ISNULL(recommend_src_provider = (new (buf) ObRSRecommendSrcProvider()))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("provider is nullptr", K(ret)); - } else if (OB_FAIL(recommend_src_provider->init(param, policy, storage_rpc, member_helper))) { - LOG_WARN("failed to init rs recommend src provider", K(ret), K(param), K(policy), KP(storage_rpc), KP(member_helper)); + } else if (OB_FAIL(recommend_src_provider->init(param, storage_rpc, member_helper))) { + LOG_WARN("failed to init rs recommend src provider", K(ret), K(param), KP(storage_rpc), KP(member_helper)); } else { provider_ = recommend_src_provider; recommend_src_provider = nullptr; @@ -1437,8 +1376,8 @@ int ObStorageHAChooseSrcHelper::init_rs_recommend_source_provider_( int ObStorageHAChooseSrcHelper::init_choose_source_by_location_provider_( const ObMigrationChooseSrcHelperInitParam ¶m, - const ObStorageHASrcProvider::ChooseSourcePolicy policy, - storage::ObStorageRpc *storage_rpc, ObStorageHAGetMemberHelper *member_helper) + storage::ObStorageRpc *storage_rpc, + ObStorageHAGetMemberHelper *member_helper) { int ret = OB_SUCCESS; void *buf = nullptr; @@ -1449,8 +1388,8 @@ int ObStorageHAChooseSrcHelper::init_choose_source_by_location_provider_( } else if (OB_ISNULL(choose_src_by_location_provider = (new (buf) ObMigrationSrcByLocationProvider()))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("provider is nullptr", K(ret)); - } else if (OB_FAIL(choose_src_by_location_provider->init(param, policy, storage_rpc, member_helper))) { - LOG_WARN("failed to init src by location provider", K(ret), K(param), K(policy), KP(storage_rpc), KP(member_helper)); + } else if (OB_FAIL(choose_src_by_location_provider->init(param, storage_rpc, member_helper))) { + LOG_WARN("failed to init src by location provider", K(ret), K(param), KP(storage_rpc), KP(member_helper)); } else { provider_ = choose_src_by_location_provider; choose_src_by_location_provider = nullptr; @@ -1465,7 +1404,8 @@ int ObStorageHAChooseSrcHelper::init_choose_source_by_location_provider_( int ObStorageHAChooseSrcHelper::init_choose_source_by_checkpoint_provider_( const ObMigrationChooseSrcHelperInitParam ¶m, - storage::ObStorageRpc *storage_rpc, ObStorageHAGetMemberHelper *member_helper) + storage::ObStorageRpc *storage_rpc, + ObStorageHAGetMemberHelper *member_helper) { int ret = OB_SUCCESS; void *buf = nullptr; @@ -1477,7 +1417,7 @@ int ObStorageHAChooseSrcHelper::init_choose_source_by_checkpoint_provider_( } else if (OB_ISNULL(choose_src_by_checkpoint_provider = (new (buf) ObMigrationSrcByCheckpointProvider()))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("provider is nullptr", K(ret)); - } else if (OB_FAIL(choose_src_by_checkpoint_provider->init(param, policy, storage_rpc, member_helper))) { + } else if (OB_FAIL(choose_src_by_checkpoint_provider->init(param, storage_rpc, member_helper))) { LOG_WARN("failed to init src by checkpoint provider", K(ret), K(param), K(policy), KP(storage_rpc), KP(member_helper)); } else { provider_ = choose_src_by_checkpoint_provider; @@ -1491,69 +1431,42 @@ int ObStorageHAChooseSrcHelper::init_choose_source_by_checkpoint_provider_( return ret; } -int ObStorageHAChooseSrcHelper::init_c_type_replica_source_provider_( - const ObMigrationChooseSrcHelperInitParam ¶m, - storage::ObStorageRpc *storage_rpc, ObStorageHAGetMemberHelper *member_helper) -{ - int ret = OB_SUCCESS; - void *buf = nullptr; - ObCTypeReplicaSrcProvider *c_replica_choose_src_provider = nullptr; - const ObStorageHASrcProvider::ChooseSourcePolicy policy = ObStorageHASrcProvider::C_TYPE_REPLICA; - if (OB_ISNULL(buf = allocator_.alloc(sizeof(ObCTypeReplicaSrcProvider)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc memory", K(ret)); - } else if (OB_ISNULL(c_replica_choose_src_provider = (new (buf) ObCTypeReplicaSrcProvider()))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("provider is nullptr", K(ret)); - } else if (OB_FAIL(c_replica_choose_src_provider->init(param, policy, storage_rpc, member_helper))) { - LOG_WARN("failed to init c replica src provider", K(ret), K(param), K(policy), KP(storage_rpc), KP(member_helper)); - } else { - provider_ = c_replica_choose_src_provider; - c_replica_choose_src_provider = nullptr; - } - if (OB_NOT_NULL(c_replica_choose_src_provider)) { - c_replica_choose_src_provider->~ObCTypeReplicaSrcProvider(); - c_replica_choose_src_provider = nullptr; - } - buf = nullptr; - return ret; -} - int ObStorageHAChooseSrcHelper::get_policy_type( const ObMigrationOpArg &arg, const uint64_t tenant_id, - bool enable_choose_source_policy, + const bool enable_choose_source_policy, const char *policy_str, const common::GlobalLearnerList &learner_list, - ObStorageHASrcProvider::ChooseSourcePolicy &policy) + ObStorageHASrcProvider::ChooseSourcePolicy &policy, + bool &use_c_replica_policy) { int ret = OB_SUCCESS; policy = ObStorageHASrcProvider::ChooseSourcePolicy::IDC; - bool use_c_replica_policy = false; + use_c_replica_policy = false; if (!arg.is_valid() || OB_INVALID_TENANT_ID == tenant_id) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(arg), K(tenant_id), K(learner_list)); } else if (OB_FAIL(check_c_replica_migration_policy_(tenant_id, arg.ls_id_, arg.dst_, learner_list, use_c_replica_policy))) { LOG_WARN("failed to check c replica choose source", K(ret), K(tenant_id), K(arg.ls_id_), K(arg.dst_), K(learner_list)); - } else if (use_c_replica_policy) { - policy = ObStorageHASrcProvider::ChooseSourcePolicy::C_TYPE_REPLICA; - LOG_INFO("c replica choose source", K(tenant_id), K(arg.ls_id_)); } else if (arg.data_src_.is_valid() && ObMigrationOpType::TYPE::REBUILD_LS_OP != arg.type_) { // TODO (zhixing.yh) modify condition after repairing compat policy = ObStorageHASrcProvider::ChooseSourcePolicy::RECOMMEND; - LOG_INFO("rs recommend source", K(arg.data_src_), K(tenant_id)); + LOG_INFO("rs recommend source", K(arg.data_src_), K(tenant_id), K(use_c_replica_policy)); } else if (!enable_choose_source_policy) { policy = ObStorageHASrcProvider::ChooseSourcePolicy::CHECKPOINT; - LOG_INFO("set checkpoint policy", K(tenant_id)); + LOG_INFO("set checkpoint policy", K(tenant_id), K(use_c_replica_policy)); + } else if (arg.prioritize_same_zone_src_) { + policy = ObStorageHASrcProvider::ChooseSourcePolicy::ZONE; + LOG_INFO("set zone policy", K(tenant_id), K(use_c_replica_policy)); } else if (0 == strcmp(policy_str, ObStorageHASrcProvider::get_policy_str(ObStorageHASrcProvider::ChooseSourcePolicy::IDC))) { policy = ObStorageHASrcProvider::ChooseSourcePolicy::IDC; - LOG_INFO("set idc policy", K(tenant_id)); + LOG_INFO("set idc policy", K(tenant_id), K(use_c_replica_policy)); } else if (0 == strcmp(policy_str, ObStorageHASrcProvider::get_policy_str(ObStorageHASrcProvider::ChooseSourcePolicy::REGION))) { policy = ObStorageHASrcProvider::ChooseSourcePolicy::REGION; - LOG_INFO("set region policy", K(tenant_id)); + LOG_INFO("set region policy", K(tenant_id), K(use_c_replica_policy)); } else { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected type", K(ret), K(tenant_id), K(policy_str)); + LOG_WARN("unexpected type", K(ret), K(tenant_id), K(policy_str), K(use_c_replica_policy)); } return ret; } @@ -1630,5 +1543,62 @@ int ObStorageHAChooseSrcHelper::check_c_replica_migration_policy_(const uint64_t #endif return ret; } +/** + * -----------------------ObMigrationChooseSrcHelperInitParam---------------------------- + */ +ObMigrationChooseSrcHelperInitParam::ObMigrationChooseSrcHelperInitParam() + : tenant_id_(OB_INVALID_ID), + ls_id_(), + local_clog_checkpoint_scn_(), + arg_(), + info_(), + policy_(ObStorageHASrcProvider::ChooseSourcePolicy::MAX_POLICY), + use_c_replica_policy_(false), + is_first_c_replica_(false) +{ +} + +void ObMigrationChooseSrcHelperInitParam::reset() +{ + tenant_id_ = OB_INVALID_ID; + ls_id_.reset(); + local_clog_checkpoint_scn_.reset(); + arg_.reset(); + info_.reset(); + policy_ = ObStorageHASrcProvider::ChooseSourcePolicy::MAX_POLICY; + use_c_replica_policy_ = false; + is_first_c_replica_ = false; +} + +bool ObMigrationChooseSrcHelperInitParam::is_valid() const +{ + return OB_INVALID_ID != tenant_id_ + && ls_id_.is_valid() + && local_clog_checkpoint_scn_.is_valid() + && arg_.is_valid() + && info_.is_valid() + && policy_ >= ObStorageHASrcProvider::ChooseSourcePolicy::ZONE + && policy_ < ObStorageHASrcProvider::ChooseSourcePolicy::MAX_POLICY; +} + +int ObMigrationChooseSrcHelperInitParam::assign(const ObMigrationChooseSrcHelperInitParam ¶m) +{ + int ret = OB_SUCCESS; + if (!param.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument!", K(ret), K(param)); + } else if (OB_FAIL(info_.assign(param.info_))) { + LOG_WARN("failed to assign param", K(ret), K(param)); + } else { + tenant_id_ = param.tenant_id_; + ls_id_ = param.ls_id_; + local_clog_checkpoint_scn_ = param.local_clog_checkpoint_scn_; + arg_ = param.arg_; + policy_ = param.policy_; + use_c_replica_policy_ = param.use_c_replica_policy_; + is_first_c_replica_ = param.is_first_c_replica_; + } + return ret; +} } // namespace storage } // namespace oceanbase diff --git a/src/storage/high_availability/ob_storage_ha_src_provider.h b/src/storage/high_availability/ob_storage_ha_src_provider.h old mode 100644 new mode 100755 index b63b2bb5e..205901a45 --- a/src/storage/high_availability/ob_storage_ha_src_provider.h +++ b/src/storage/high_availability/ob_storage_ha_src_provider.h @@ -21,6 +21,7 @@ namespace oceanbase { namespace storage { +struct ObMigrationChooseSrcHelperInitParam; class ObStorageHAGetMemberHelper { @@ -68,51 +69,91 @@ class ObStorageHASrcProvider { public: enum ChooseSourcePolicy : uint8_t { - IDC = 0, - REGION = 1, - CHECKPOINT = 2, - RECOMMEND = 3, - C_TYPE_REPLICA = 4, + ZONE = 0, + IDC = 1, + REGION = 2, + DIFFERENT_REGION = 3, // cannot set manually + // above policies are used for choosing source by location + CHECKPOINT = 4, + RECOMMEND = 5, MAX_POLICY }; + static const int64_t LOCATION_POLICY_COUNT = DIFFERENT_REGION + 1; + + struct ChooseSourcePolicyDetailedInfo + { + public: + ChooseSourcePolicyDetailedInfo(): + policy_type_(ChooseSourcePolicy::MAX_POLICY), + chosen_policy_type_(ChooseSourcePolicy::MAX_POLICY), + use_c_replica_policy_(false), + is_first_c_replica_(false) + {} + ~ChooseSourcePolicyDetailedInfo() {} + + TO_STRING_KV( + "policy_type", get_policy_str(policy_type_), + "chosen_policy_type_", get_policy_str(chosen_policy_type_), + K_(use_c_replica_policy), + K_(is_first_c_replica)); + ChooseSourcePolicy policy_type_; + ChooseSourcePolicy chosen_policy_type_; + bool use_c_replica_policy_; + bool is_first_c_replica_; + }; + ObStorageHASrcProvider(); virtual ~ObStorageHASrcProvider(); int init(const ObMigrationChooseSrcHelperInitParam ¶m, - const ChooseSourcePolicy policy_type, storage::ObStorageRpc *storage_rpc, ObStorageHAGetMemberHelper *member_helper); virtual int choose_ob_src( - const ObMigrationOpArg &arg, common::ObAddr &chosen_src_addr) = 0; + const ObMigrationOpArg &arg, common::ObAddr &chosen_src_addr); inline uint64_t get_tenant_id() const { return tenant_id_; } const share::ObLSID &get_ls_id() const { return ls_id_; } - ObMigrationOpType::TYPE get_type() const { return type_; } + ObMigrationOpType::TYPE get_migration_op_type() const { return type_; } + ChooseSourcePolicy get_policy_type() const { return policy_type_; } storage::ObStorageRpc *get_storage_rpc() const { return storage_rpc_; } const share::SCN &get_local_clog_checkpoint_scn() const { return local_clog_checkpoint_scn_; } const share::SCN &get_palf_parent_checkpoint_scn() const { return palf_parent_checkpoint_scn_; } - ChooseSourcePolicy get_policy_type() const { return policy_type_; } bool is_first_c_replica() const { return is_first_c_replica_; } const static char *ObChooseSourcePolicyStr[static_cast(ChooseSourcePolicy::MAX_POLICY)]; const static char *get_policy_str(const ChooseSourcePolicy policy_type); + int get_policy_detailed_info_str(char *buf, const int64_t buf_len); int check_tenant_primary(bool &is_primary); protected: - // The validity assessment of replicas includes: - // server_version: dest server_version >= src server_version - // restore_status: if restore_status of ls is fail, migration needs to wait. - // migration_status: OB_MIGRATION_STATUS_NONE - // replica type: F replica could serve as the source of F replica and R replica, - // while R replica could only serve as the source of R replica - // source checkpoint scn must be greater than or equal than palf_parent_checkpoint_scn_ and local_clog_checkpoint_scn_ + /* + * The validity assessment of replicas includes: + * server_version: dest server_version >= src server_version + * restore_status: if restore_status of ls is fail, migration needs to wait. + * migration_status: OB_MIGRATION_STATUS_NONE + * replica type: + * 1. F replica could serve as the source of F/R/C replica + * 2. R replica could only serve as the source of R/C + * 3. C replica could only serve as the source of C replica + * clog_checkpoint: source checkpoint scn must be greater than or equal than palf_parent_checkpoint_scn_ and local_clog_checkpoint_scn_ + * + * If must_choose_c_replica is true, the source must be C replica. + */ int check_replica_validity( const common::ObAddr &addr, const common::ObReplicaMember &dst, - const common::GlobalLearnerList &learner_list, obrpc::ObFetchLSMetaInfoResp &ls_info); + const common::GlobalLearnerList &learner_list, const bool &must_choose_c_replica, + obrpc::ObFetchLSMetaInfoResp &ls_info); + + virtual int inner_choose_ob_src( + const common::ObAddr &leader_addr, const common::GlobalLearnerList &learner_list, + const common::ObIArray &addr_list, const ObMigrationOpArg &arg, const bool &must_choose_c_replica, + common::ObAddr &chosen_src_addr) = 0; protected: bool is_inited_; ObLSMemberListInfo member_list_info_; bool is_first_c_replica_; + ChooseSourcePolicy chosen_policy_type_; // the policy type finally chosen after checking + bool use_c_replica_policy_; // true if dst is c replica and there already exists c replica in the cluster private: int fetch_ls_meta_info_(const uint64_t tenant_id, const share::ObLSID &ls_id, const common::ObAddr &member_addr, obrpc::ObFetchLSMetaInfoResp &ls_meta_info); @@ -120,6 +161,7 @@ private: const common::ObAddr &addr, const common::ObReplicaMember &dst, const common::GlobalLearnerList &learner_list, + const bool &must_choose_c_replica, bool &is_replica_type_valid); int init_palf_parent_checkpoint_scn_(const uint64_t tenant_id, const share::ObLSID &ls_id, const share::SCN &local_clog_checkpoint_scn, const common::ObReplicaType replica_type); @@ -147,7 +189,7 @@ private: share::SCN palf_parent_checkpoint_scn_; ObStorageHAGetMemberHelper *member_helper_; storage::ObStorageRpc *storage_rpc_; - ChooseSourcePolicy policy_type_; + ChooseSourcePolicy policy_type_; // the policy type chosen by the user DISALLOW_COPY_AND_ASSIGN(ObStorageHASrcProvider); }; @@ -156,36 +198,49 @@ class ObMigrationSrcByLocationProvider : public ObStorageHASrcProvider public: ObMigrationSrcByLocationProvider(); virtual ~ObMigrationSrcByLocationProvider(); - int init(const ObMigrationChooseSrcHelperInitParam ¶m, - const ChooseSourcePolicy policy_type, + int init( + const ObMigrationChooseSrcHelperInitParam ¶m, storage::ObStorageRpc *storage_rpc, ObStorageHAGetMemberHelper *member_helper); - virtual int choose_ob_src( - const ObMigrationOpArg &arg, common::ObAddr &chosen_src_addr) override; - protected: - int inner_choose_ob_src( + virtual int inner_choose_ob_src( const common::ObAddr &leader_addr, const common::GlobalLearnerList &learner_list, - const common::ObIArray &addr_list, const ObMigrationOpArg &arg, - common::ObAddr &choosen_src_addr); + const common::ObIArray &addr_list, const ObMigrationOpArg &arg, const bool &must_choose_c_replica, + common::ObAddr &chosen_src_addr) override; +private: + void set_locality_manager_(ObLocalityManager *locality_manager); + int get_server_geography_info_(const common::ObAddr &addr, common::ObRegion ®ion, common::ObIDC &idc, common::ObZone &zone); + /* + * Divide (sort) the addr_list by zone, idc, region, different region. + * The layout of the sorted_addr_list is as follows: + * |<---- same zone ----->|<------ same idc ------>|<----- same region ---->|<-- different region -->| + * {addr[0], ..., addr[p], addr[p+1], ..., addr[q], addr[q+1], ..., addr[s], addr[s+1], ..., addr[t]} + * | | | + * zone_end_index idc_end_index region_end_index + */ int divide_addr_list( const common::ObIArray &addr_list, const common::ObReplicaMember &dst, common::ObIArray &sorted_addr_list, + int64_t &zone_end_index, int64_t &idc_end_index, int64_t ®ion_end_index); - int find_src( + /* + * Find source from the sorted addr list. + * Will only choose the src in [start_index, end_index] + * The chosen source must be valid (see ObStorageHASrcProvider::check_replica_validity). + */ + int find_src_in_sorted_addr_list_( const common::ObIArray &sorted_addr_list, const int64_t start_index, const int64_t end_index, const common::GlobalLearnerList &learner_list, const common::ObAddr &leader_addr, const common::ObReplicaMember &dst, - common::ObAddr &choosen_src_addr); -private: - void set_locality_manager_(ObLocalityManager *locality_manager); - int get_server_region_and_idc_( - const common::ObAddr &addr, common::ObRegion ®ion, common::ObIDC &idc); + const bool &must_choose_c_replica, + common::ObAddr &chosen_src_addr); + + private: ObLocalityManager *locality_manager_; @@ -197,18 +252,15 @@ class ObMigrationSrcByCheckpointProvider : public ObStorageHASrcProvider public: ObMigrationSrcByCheckpointProvider(); virtual ~ObMigrationSrcByCheckpointProvider(); - int init(const ObMigrationChooseSrcHelperInitParam ¶m, - const ChooseSourcePolicy policy_type, + int init( + const ObMigrationChooseSrcHelperInitParam ¶m, storage::ObStorageRpc *storage_rpc, ObStorageHAGetMemberHelper *member_helper); - virtual int choose_ob_src( - const ObMigrationOpArg &arg, common::ObAddr &chosen_src_addr) override; - -private: - int inner_choose_ob_src_( +protected: + virtual int inner_choose_ob_src( const common::ObAddr &leader_addr, const common::GlobalLearnerList &learner_list, - const common::ObIArray &addr_list, const ObMigrationOpArg &arg, - common::ObAddr &choosen_src_addr); + const common::ObIArray &addr_list, const ObMigrationOpArg &arg, const bool &must_choose_c_replica, + common::ObAddr &chosen_src_addr) override; DISALLOW_COPY_AND_ASSIGN(ObMigrationSrcByCheckpointProvider); }; @@ -218,71 +270,59 @@ class ObRSRecommendSrcProvider : public ObStorageHASrcProvider public: ObRSRecommendSrcProvider(); virtual ~ObRSRecommendSrcProvider(); - int init(const ObMigrationChooseSrcHelperInitParam ¶m, - const ChooseSourcePolicy policy_type, + int init( + const ObMigrationChooseSrcHelperInitParam ¶m, storage::ObStorageRpc *storage_rpc, ObStorageHAGetMemberHelper *member_helper); - int choose_ob_src( - const ObMigrationOpArg &arg, common::ObAddr &chosen_src_addr); + + // overwrite choose_ob_src, if use specified policy, only need to call inner_choose_ob_src once (no need to retry when is_first_c_replica) + virtual int choose_ob_src( + const ObMigrationOpArg &arg, common::ObAddr &chosen_src_addr) override; +protected: + virtual int inner_choose_ob_src( + const common::ObAddr &leader_addr, const common::GlobalLearnerList &learner_list, + const common::ObIArray &addr_list, const ObMigrationOpArg &arg, const bool &must_choose_c_replica, + common::ObAddr &chosen_src_addr) override; private: int check_replica_validity_(const int64_t cluster_id, const common::ObIArray &addr_list, const common::ObAddr &addr, const common::ObReplicaMember &dst, - const common::GlobalLearnerList &learner_list); + const common::GlobalLearnerList &learner_list, const bool &must_choose_c_replica); DISALLOW_COPY_AND_ASSIGN(ObRSRecommendSrcProvider); }; -class ObCTypeReplicaSrcProvider : public ObStorageHASrcProvider -{ -public: - ObCTypeReplicaSrcProvider(); - virtual ~ObCTypeReplicaSrcProvider(); - int init(const ObMigrationChooseSrcHelperInitParam ¶m, - const ChooseSourcePolicy policy_type, - storage::ObStorageRpc *storage_rpc, - ObStorageHAGetMemberHelper *member_helper); - int choose_ob_src( - const ObMigrationOpArg &arg, common::ObAddr &chosen_src_addr); - -private: - int inner_choose_ob_src_( - const common::ObIArray &addr_list, - const common::GlobalLearnerList &learner_list, - const ObMigrationOpArg &arg, common::ObAddr &choosen_src_addr); - int get_c_replica_( - const common::ObAddr &addr, - const common::GlobalLearnerList &learner_list, - ObIArray &c_replica_list) const; - DISALLOW_COPY_AND_ASSIGN(ObCTypeReplicaSrcProvider); -}; - class ObStorageHAChooseSrcHelper final { public: ObStorageHAChooseSrcHelper(); ~ObStorageHAChooseSrcHelper(); - int init(const ObMigrationChooseSrcHelperInitParam ¶m, const ObStorageHASrcProvider::ChooseSourcePolicy policy, - storage::ObStorageRpc *storage_rpc, ObStorageHAGetMemberHelper *member_helper); + int init( + const ObMigrationChooseSrcHelperInitParam ¶m, + storage::ObStorageRpc *storage_rpc, + ObStorageHAGetMemberHelper *member_helper); int get_available_src(const ObMigrationOpArg &arg, ObStorageHASrcInfo &src_info); - static int get_policy_type(const ObMigrationOpArg &arg, const uint64_t tenant_id, - bool enable_choose_source_policy, const char *policy_str, + static int get_policy_type( + const ObMigrationOpArg &arg, + const uint64_t tenant_id, + const bool enable_choose_source_policy, + const char *policy_str, const common::GlobalLearnerList &learner_list, - ObStorageHASrcProvider::ChooseSourcePolicy &policy); + ObStorageHASrcProvider::ChooseSourcePolicy &policy, + bool &use_c_replica_policy); private: - int init_rs_recommend_source_provider_(const ObMigrationChooseSrcHelperInitParam ¶m, + int init_rs_recommend_source_provider_( + const ObMigrationChooseSrcHelperInitParam ¶m, storage::ObStorageRpc *storage_rpc, ObStorageHAGetMemberHelper *member_helper); int init_choose_source_by_location_provider_( const ObMigrationChooseSrcHelperInitParam ¶m, - const ObStorageHASrcProvider::ChooseSourcePolicy policy, storage::ObStorageRpc *storage_rpc, + storage::ObStorageRpc *storage_rpc, ObStorageHAGetMemberHelper *member_helper); int init_choose_source_by_checkpoint_provider_( const ObMigrationChooseSrcHelperInitParam ¶m, storage::ObStorageRpc *storage_rpc, ObStorageHAGetMemberHelper *member_helper); - int init_c_type_replica_source_provider_( - const ObMigrationChooseSrcHelperInitParam ¶m, - storage::ObStorageRpc *storage_rpc, ObStorageHAGetMemberHelper *member_helper); + void errsim_test_(const ObMigrationOpArg &arg, ObStorageHASrcInfo &src_info); ObStorageHASrcProvider * get_provider() const { return provider_; } static int check_c_replica_migration_policy_(const uint64_t tenant_id, const share::ObLSID &ls_id, @@ -297,6 +337,28 @@ private: DISALLOW_COPY_AND_ASSIGN(ObStorageHAChooseSrcHelper); }; +struct ObMigrationChooseSrcHelperInitParam final +{ +public: + ObMigrationChooseSrcHelperInitParam(); + ~ObMigrationChooseSrcHelperInitParam() = default; + void reset(); + bool is_valid() const; + int assign(const ObMigrationChooseSrcHelperInitParam ¶m); + + TO_STRING_KV( + K_(tenant_id), K_(ls_id), K_(local_clog_checkpoint_scn), K_(arg), K_(info), K_(policy), K_(use_c_replica_policy), K_(is_first_c_replica)); + uint64_t tenant_id_; + share::ObLSID ls_id_; + share::SCN local_clog_checkpoint_scn_; + ObMigrationOpArg arg_; + ObLSMemberListInfo info_; + ObStorageHASrcProvider::ChooseSourcePolicy policy_; + bool use_c_replica_policy_; + bool is_first_c_replica_; +private: + DISALLOW_COPY_AND_ASSIGN(ObMigrationChooseSrcHelperInitParam); +}; } // namespace storage } // namespace oceanbase #endif diff --git a/src/storage/high_availability/ob_storage_ha_struct.cpp b/src/storage/high_availability/ob_storage_ha_struct.cpp index aef13c608..95e06ff1e 100644 --- a/src/storage/high_availability/ob_storage_ha_struct.cpp +++ b/src/storage/high_availability/ob_storage_ha_struct.cpp @@ -1101,7 +1101,8 @@ ObMigrationOpArg::ObMigrationOpArg() src_(), dst_(), data_src_(), - paxos_replica_number_(0) + paxos_replica_number_(0), + prioritize_same_zone_src_(false) { } @@ -1127,6 +1128,7 @@ void ObMigrationOpArg::reset() dst_.reset(); data_src_.reset(); paxos_replica_number_ = 0; + prioritize_same_zone_src_ = false; } /******************ObTabletsTransferArg*********************/ @@ -2499,53 +2501,6 @@ int ObLSMemberListInfo::assign(const ObLSMemberListInfo &info) return ret; } -ObMigrationChooseSrcHelperInitParam::ObMigrationChooseSrcHelperInitParam() - : tenant_id_(OB_INVALID_ID), - ls_id_(), - local_clog_checkpoint_scn_(), - arg_(), - info_(), - is_first_c_replica_(false) -{ -} - -void ObMigrationChooseSrcHelperInitParam::reset() -{ - tenant_id_ = OB_INVALID_ID; - ls_id_.reset(); - local_clog_checkpoint_scn_.reset(); - arg_.reset(); - info_.reset(); - is_first_c_replica_ = false; -} - -bool ObMigrationChooseSrcHelperInitParam::is_valid() const -{ - return OB_INVALID_ID != tenant_id_ - && ls_id_.is_valid() - && local_clog_checkpoint_scn_.is_valid() - && arg_.is_valid() - && info_.is_valid(); -} - -int ObMigrationChooseSrcHelperInitParam::assign(const ObMigrationChooseSrcHelperInitParam ¶m) -{ - int ret = OB_SUCCESS; - if (!param.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument!", K(ret), K(param)); - } else if (OB_FAIL(info_.assign(param.info_))) { - LOG_WARN("failed to assign param", K(ret), K(param)); - } else { - tenant_id_ = param.tenant_id_; - ls_id_ = param.ls_id_; - local_clog_checkpoint_scn_ = param.local_clog_checkpoint_scn_; - arg_ = param.arg_; - is_first_c_replica_ = param.is_first_c_replica_; - } - return ret; -} - int ObMacroBlockReuseMgr::get_macro_block_reuse_info( const ObITable::TableKey &table_key, const blocksstable::ObLogicMacroBlockId &logic_id, diff --git a/src/storage/high_availability/ob_storage_ha_struct.h b/src/storage/high_availability/ob_storage_ha_struct.h index d97a8ed33..4a06facdf 100644 --- a/src/storage/high_availability/ob_storage_ha_struct.h +++ b/src/storage/high_availability/ob_storage_ha_struct.h @@ -187,6 +187,7 @@ struct ObMigrationOpArg common::ObReplicaMember dst_; common::ObReplicaMember data_src_; int64_t paxos_replica_number_; + bool prioritize_same_zone_src_; }; struct ObTabletsTransferArg @@ -723,26 +724,6 @@ private: DISALLOW_COPY_AND_ASSIGN(ObLSMemberListInfo); }; -struct ObMigrationChooseSrcHelperInitParam final -{ -public: - ObMigrationChooseSrcHelperInitParam(); - ~ObMigrationChooseSrcHelperInitParam() = default; - void reset(); - bool is_valid() const; - int assign(const ObMigrationChooseSrcHelperInitParam ¶m); - - TO_STRING_KV(K_(tenant_id), K_(ls_id), K_(local_clog_checkpoint_scn), K_(arg), K_(info), K_(is_first_c_replica)); - uint64_t tenant_id_; - share::ObLSID ls_id_; - share::SCN local_clog_checkpoint_scn_; - ObMigrationOpArg arg_; - ObLSMemberListInfo info_; - bool is_first_c_replica_; // whether the dst is the first C replica in the learner list -private: - DISALLOW_COPY_AND_ASSIGN(ObMigrationChooseSrcHelperInitParam); -}; - } } #endif diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result index c2ba22908..b6fe71e35 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result @@ -221,6 +221,7 @@ recover_table_concurrency recover_table_dop recyclebin_object_expire_time redundancy_level +replica_parallel_migration_mode resource_hard_limit rootservice_async_task_queue_size rootservice_async_task_thread_count diff --git a/unittest/storage/migration/test_choose_migration_source_policy.cpp b/unittest/storage/migration/test_choose_migration_source_policy.cpp index 1d1c5b879..84c69a821 100644 --- a/unittest/storage/migration/test_choose_migration_source_policy.cpp +++ b/unittest/storage/migration/test_choose_migration_source_policy.cpp @@ -560,10 +560,9 @@ TEST_F(TestChooseMigrationSourcePolicy, get_available_src_with_checkpoint_policy EXPECT_EQ(OB_SUCCESS, mock_migrate_choose_helper_param(tenant_id, ls_id, local_ls_checkpoint_scn, mock_arg, param)); EXPECT_EQ(OB_SUCCESS, member_helper_.get_member_list_by_replica_type(tenant_id, ls_id, mock_arg.dst_, param.info_, param.is_first_c_replica_)); - ObStorageHASrcProvider::ChooseSourcePolicy policy; - EXPECT_EQ(OB_SUCCESS, get_checkpoint_policy(mock_arg, tenant_id, param.info_.learner_list_, policy)); + EXPECT_EQ(OB_SUCCESS, get_checkpoint_policy(mock_arg, tenant_id, param.info_.learner_list_, param.policy_, param.use_c_replica_policy_)); ObStorageHASrcInfo src_info; - EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, policy, &storage_rpc_, &member_helper_)); + EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, &storage_rpc_, &member_helper_)); EXPECT_EQ(OB_SUCCESS, choose_src_helper_.get_available_src(mock_arg, src_info)); common::ObAddr expect_addr; EXPECT_EQ(OB_SUCCESS, mock_addr("192.168.1.1:1234", expect_addr)); @@ -597,11 +596,9 @@ TEST_F(TestChooseMigrationSourcePolicy, get_available_src_with_rs_recommend) ObMigrationChooseSrcHelperInitParam param; EXPECT_EQ(OB_SUCCESS, mock_migrate_choose_helper_param(tenant_id, ls_id, local_ls_checkpoint_scn, mock_arg, param)); EXPECT_EQ(OB_SUCCESS, member_helper_.get_member_list_by_replica_type(tenant_id, ls_id, - mock_arg.dst_, param.info_, param.is_first_c_replica_)); - ObStorageHASrcProvider::ChooseSourcePolicy policy; - EXPECT_EQ(OB_SUCCESS, get_recommand_policy(mock_arg, tenant_id, param.info_.learner_list_, policy)); + mock_arg.dst_, param.info_, param.is_first_c_replica_)); EXPECT_EQ(OB_SUCCESS, get_recommand_policy(mock_arg, tenant_id, param.info_.learner_list_, param.policy_, param.use_c_replica_policy_)); ObStorageHASrcInfo src_info; - EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, policy, &storage_rpc_, &member_helper_)); + EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, &storage_rpc_, &member_helper_)); EXPECT_EQ(OB_SUCCESS, choose_src_helper_.get_available_src(mock_arg, src_info)); common::ObAddr expect_addr; EXPECT_EQ(OB_SUCCESS, mock_addr("192.168.1.4:1234", expect_addr)); @@ -636,11 +633,9 @@ TEST_F(TestChooseMigrationSourcePolicy, idc_mode_idc_leader) ObMigrationChooseSrcHelperInitParam param; EXPECT_EQ(OB_SUCCESS, mock_migrate_choose_helper_param(tenant_id, ls_id, local_ls_checkpoint_scn, mock_arg, param)); EXPECT_EQ(OB_SUCCESS, member_helper_.get_member_list_by_replica_type(tenant_id, ls_id, - mock_arg.dst_, param.info_, param.is_first_c_replica_)); - ObStorageHASrcProvider::ChooseSourcePolicy policy; - EXPECT_EQ(OB_SUCCESS, get_idc_policy(mock_arg, tenant_id, param.info_.learner_list_, policy)); + mock_arg.dst_, param.info_, param.is_first_c_replica_)); EXPECT_EQ(OB_SUCCESS, get_idc_policy(mock_arg, tenant_id, param.info_.learner_list_, param.policy_, param.use_c_replica_policy_)); ObStorageHASrcInfo src_info; - EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, policy, &storage_rpc_, &member_helper_)); + EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, &storage_rpc_, &member_helper_)); EXPECT_EQ(OB_SUCCESS, mock_locality_manager(MOCKLOCALITY::IDC_MODE_IDC_LEADER, locality_manager_)); EXPECT_EQ(ObStorageHASrcProvider::ChooseSourcePolicy::IDC, choose_src_helper_.get_provider()->get_policy_type()); static_cast(choose_src_helper_.get_provider())->set_locality_manager_(&locality_manager_); @@ -678,11 +673,9 @@ TEST_F(TestChooseMigrationSourcePolicy, idc_mode_idc_follower) ObMigrationChooseSrcHelperInitParam param; EXPECT_EQ(OB_SUCCESS, mock_migrate_choose_helper_param(tenant_id, ls_id, local_ls_checkpoint_scn, mock_arg, param)); EXPECT_EQ(OB_SUCCESS, member_helper_.get_member_list_by_replica_type(tenant_id, ls_id, - mock_arg.dst_, param.info_, param.is_first_c_replica_)); - ObStorageHASrcProvider::ChooseSourcePolicy policy; - EXPECT_EQ(OB_SUCCESS, get_idc_policy(mock_arg, tenant_id, param.info_.learner_list_, policy)); + mock_arg.dst_, param.info_, param.is_first_c_replica_)); EXPECT_EQ(OB_SUCCESS, get_idc_policy(mock_arg, tenant_id, param.info_.learner_list_, param.policy_, param.use_c_replica_policy_)); ObStorageHASrcInfo src_info; - EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, policy, &storage_rpc_, &member_helper_)); + EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, &storage_rpc_, &member_helper_)); EXPECT_EQ(OB_SUCCESS, mock_locality_manager(MOCKLOCALITY::IDC_MODE_IDC_FOLLOWER, locality_manager_)); EXPECT_EQ(ObStorageHASrcProvider::ChooseSourcePolicy::IDC, choose_src_helper_.get_provider()->get_policy_type()); static_cast(choose_src_helper_.get_provider())->set_locality_manager_(&locality_manager_); @@ -718,11 +711,9 @@ TEST_F(TestChooseMigrationSourcePolicy, idc_mode_region_leader) ObMigrationChooseSrcHelperInitParam param; EXPECT_EQ(OB_SUCCESS, mock_migrate_choose_helper_param(tenant_id, ls_id, local_ls_checkpoint_scn, mock_arg, param)); EXPECT_EQ(OB_SUCCESS, member_helper_.get_member_list_by_replica_type(tenant_id, ls_id, - mock_arg.dst_, param.info_, param.is_first_c_replica_)); - ObStorageHASrcProvider::ChooseSourcePolicy policy; - EXPECT_EQ(OB_SUCCESS, get_idc_policy(mock_arg, tenant_id, param.info_.learner_list_, policy)); + mock_arg.dst_, param.info_, param.is_first_c_replica_)); EXPECT_EQ(OB_SUCCESS, get_idc_policy(mock_arg, tenant_id, param.info_.learner_list_, param.policy_, param.use_c_replica_policy_)); ObStorageHASrcInfo src_info; - EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, policy, &storage_rpc_, &member_helper_)); + EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, &storage_rpc_, &member_helper_)); EXPECT_EQ(OB_SUCCESS, mock_locality_manager(MOCKLOCALITY::IDC_MODE_REGION_LEADER, locality_manager_)); EXPECT_EQ(ObStorageHASrcProvider::ChooseSourcePolicy::IDC, choose_src_helper_.get_provider()->get_policy_type()); static_cast(choose_src_helper_.get_provider())->set_locality_manager_(&locality_manager_); @@ -758,11 +749,9 @@ TEST_F(TestChooseMigrationSourcePolicy, idc_mode_region_follower) ObMigrationChooseSrcHelperInitParam param; EXPECT_EQ(OB_SUCCESS, mock_migrate_choose_helper_param(tenant_id, ls_id, local_ls_checkpoint_scn, mock_arg, param)); EXPECT_EQ(OB_SUCCESS, member_helper_.get_member_list_by_replica_type(tenant_id, ls_id, - mock_arg.dst_, param.info_, param.is_first_c_replica_)); - ObStorageHASrcProvider::ChooseSourcePolicy policy; - EXPECT_EQ(OB_SUCCESS, get_idc_policy(mock_arg, tenant_id, param.info_.learner_list_, policy)); + mock_arg.dst_, param.info_, param.is_first_c_replica_)); EXPECT_EQ(OB_SUCCESS, get_idc_policy(mock_arg, tenant_id, param.info_.learner_list_, param.policy_, param.use_c_replica_policy_)); ObStorageHASrcInfo src_info; - EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, policy, &storage_rpc_, &member_helper_)); + EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, &storage_rpc_, &member_helper_)); EXPECT_EQ(OB_SUCCESS, mock_locality_manager(MOCKLOCALITY::IDC_MODE_REGION_FOLLOWER, locality_manager_)); EXPECT_EQ(ObStorageHASrcProvider::ChooseSourcePolicy::IDC, choose_src_helper_.get_provider()->get_policy_type()); static_cast(choose_src_helper_.get_provider())->set_locality_manager_(&locality_manager_); @@ -796,11 +785,9 @@ TEST_F(TestChooseMigrationSourcePolicy, idc_mode_diff_region_leader) ObMigrationChooseSrcHelperInitParam param; EXPECT_EQ(OB_SUCCESS, mock_migrate_choose_helper_param(tenant_id, ls_id, local_ls_checkpoint_scn, mock_arg, param)); EXPECT_EQ(OB_SUCCESS, member_helper_.get_member_list_by_replica_type(tenant_id, ls_id, - mock_arg.dst_, param.info_, param.is_first_c_replica_)); - ObStorageHASrcProvider::ChooseSourcePolicy policy; - EXPECT_EQ(OB_SUCCESS, get_idc_policy(mock_arg, tenant_id, param.info_.learner_list_, policy)); + mock_arg.dst_, param.info_, param.is_first_c_replica_)); EXPECT_EQ(OB_SUCCESS, get_idc_policy(mock_arg, tenant_id, param.info_.learner_list_, param.policy_, param.use_c_replica_policy_)); ObStorageHASrcInfo src_info; - EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, policy, &storage_rpc_, &member_helper_)); + EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, &storage_rpc_, &member_helper_)); EXPECT_EQ(OB_SUCCESS, mock_locality_manager(MOCKLOCALITY::IDC_MODE_DIFF_REGION_LEADER, locality_manager_)); EXPECT_EQ(ObStorageHASrcProvider::ChooseSourcePolicy::IDC, choose_src_helper_.get_provider()->get_policy_type()); static_cast(choose_src_helper_.get_provider())->set_locality_manager_(&locality_manager_); @@ -835,11 +822,9 @@ TEST_F(TestChooseMigrationSourcePolicy, idc_mode_diff_region_follower) ObMigrationChooseSrcHelperInitParam param; EXPECT_EQ(OB_SUCCESS, mock_migrate_choose_helper_param(tenant_id, ls_id, local_ls_checkpoint_scn, mock_arg, param)); EXPECT_EQ(OB_SUCCESS, member_helper_.get_member_list_by_replica_type(tenant_id, ls_id, - mock_arg.dst_, param.info_, param.is_first_c_replica_)); - ObStorageHASrcProvider::ChooseSourcePolicy policy; - EXPECT_EQ(OB_SUCCESS, get_idc_policy(mock_arg, tenant_id, param.info_.learner_list_, policy)); + mock_arg.dst_, param.info_, param.is_first_c_replica_)); EXPECT_EQ(OB_SUCCESS, get_idc_policy(mock_arg, tenant_id, param.info_.learner_list_, param.policy_, param.use_c_replica_policy_)); ObStorageHASrcInfo src_info; - EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, policy, &storage_rpc_, &member_helper_)); + EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, &storage_rpc_, &member_helper_)); EXPECT_EQ(OB_SUCCESS, mock_locality_manager(MOCKLOCALITY::IDC_MODE_DIFF_REGION_FOLLOWER, locality_manager_)); EXPECT_EQ(ObStorageHASrcProvider::ChooseSourcePolicy::IDC, choose_src_helper_.get_provider()->get_policy_type()); static_cast(choose_src_helper_.get_provider())->set_locality_manager_(&locality_manager_); @@ -875,11 +860,9 @@ TEST_F(TestChooseMigrationSourcePolicy, region_mode_region_follower) ObMigrationChooseSrcHelperInitParam param; EXPECT_EQ(OB_SUCCESS, mock_migrate_choose_helper_param(tenant_id, ls_id, local_ls_checkpoint_scn, mock_arg, param)); EXPECT_EQ(OB_SUCCESS, member_helper_.get_member_list_by_replica_type(tenant_id, ls_id, - mock_arg.dst_, param.info_, param.is_first_c_replica_)); - ObStorageHASrcProvider::ChooseSourcePolicy policy; - EXPECT_EQ(OB_SUCCESS, get_region_policy(mock_arg, tenant_id, param.info_.learner_list_, policy)); + mock_arg.dst_, param.info_, param.is_first_c_replica_)); EXPECT_EQ(OB_SUCCESS, get_region_policy(mock_arg, tenant_id, param.info_.learner_list_, param.policy_, param.use_c_replica_policy_)); ObStorageHASrcInfo src_info; - EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, policy, &storage_rpc_, &member_helper_)); + EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, &storage_rpc_, &member_helper_)); EXPECT_EQ(OB_SUCCESS, mock_locality_manager(MOCKLOCALITY::REGION_MODE_REGION_FOLLOWER, locality_manager_)); EXPECT_EQ(ObStorageHASrcProvider::ChooseSourcePolicy::REGION, choose_src_helper_.get_provider()->get_policy_type()); static_cast(choose_src_helper_.get_provider())->set_locality_manager_(&locality_manager_); @@ -916,10 +899,9 @@ TEST_F(TestChooseMigrationSourcePolicy, region_mode_region_leader) EXPECT_EQ(OB_SUCCESS, mock_migrate_choose_helper_param(tenant_id, ls_id, local_ls_checkpoint_scn, mock_arg, param)); EXPECT_EQ(OB_SUCCESS, member_helper_.get_member_list_by_replica_type(tenant_id, ls_id, mock_arg.dst_, param.info_, param.is_first_c_replica_)); - ObStorageHASrcProvider::ChooseSourcePolicy policy; - EXPECT_EQ(OB_SUCCESS, get_region_policy(mock_arg, tenant_id, param.info_.learner_list_, policy)); + EXPECT_EQ(OB_SUCCESS, get_region_policy(mock_arg, tenant_id, param.info_.learner_list_, param.policy_, param.use_c_replica_policy_)); ObStorageHASrcInfo src_info; - EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, policy, &storage_rpc_, &member_helper_)); + EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, &storage_rpc_, &member_helper_)); EXPECT_EQ(OB_SUCCESS, mock_locality_manager(MOCKLOCALITY::REGION_MODE_REGION_LEADER, locality_manager_)); EXPECT_EQ(ObStorageHASrcProvider::ChooseSourcePolicy::REGION, choose_src_helper_.get_provider()->get_policy_type()); static_cast(choose_src_helper_.get_provider())->set_locality_manager_(&locality_manager_); @@ -955,10 +937,9 @@ TEST_F(TestChooseMigrationSourcePolicy, region_mode_diff_region_follower) EXPECT_EQ(OB_SUCCESS, mock_migrate_choose_helper_param(tenant_id, ls_id, local_ls_checkpoint_scn, mock_arg, param)); EXPECT_EQ(OB_SUCCESS, member_helper_.get_member_list_by_replica_type(tenant_id, ls_id, mock_arg.dst_, param.info_, param.is_first_c_replica_)); - ObStorageHASrcProvider::ChooseSourcePolicy policy; - EXPECT_EQ(OB_SUCCESS, get_region_policy(mock_arg, tenant_id, param.info_.learner_list_, policy)); + EXPECT_EQ(OB_SUCCESS, get_region_policy(mock_arg, tenant_id, param.info_.learner_list_, param.policy_, param.use_c_replica_policy_)); ObStorageHASrcInfo src_info; - EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, policy, &storage_rpc_, &member_helper_)); + EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, &storage_rpc_, &member_helper_)); EXPECT_EQ(OB_SUCCESS, mock_locality_manager(MOCKLOCALITY::REGION_MODE_DIFF_REGION_FOLLOWER, locality_manager_)); EXPECT_EQ(ObStorageHASrcProvider::ChooseSourcePolicy::REGION, choose_src_helper_.get_provider()->get_policy_type()); static_cast(choose_src_helper_.get_provider())->set_locality_manager_(&locality_manager_); @@ -993,10 +974,9 @@ TEST_F(TestChooseMigrationSourcePolicy, region_mode_diff_region_leader) EXPECT_EQ(OB_SUCCESS, mock_migrate_choose_helper_param(tenant_id, ls_id, local_ls_checkpoint_scn, mock_arg, param)); EXPECT_EQ(OB_SUCCESS, member_helper_.get_member_list_by_replica_type(tenant_id, ls_id, mock_arg.dst_, param.info_, param.is_first_c_replica_)); - ObStorageHASrcProvider::ChooseSourcePolicy policy; - EXPECT_EQ(OB_SUCCESS, get_region_policy(mock_arg, tenant_id, param.info_.learner_list_, policy)); + EXPECT_EQ(OB_SUCCESS, get_region_policy(mock_arg, tenant_id, param.info_.learner_list_, param.policy_, param.use_c_replica_policy_)); ObStorageHASrcInfo src_info; - EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, policy, &storage_rpc_, &member_helper_)); + EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, &storage_rpc_, &member_helper_)); EXPECT_EQ(OB_SUCCESS, mock_locality_manager(MOCKLOCALITY::REGION_MODE_DIFF_REGION_LEADER, locality_manager_)); EXPECT_EQ(ObStorageHASrcProvider::ChooseSourcePolicy::REGION, choose_src_helper_.get_provider()->get_policy_type()); static_cast(choose_src_helper_.get_provider())->set_locality_manager_(&locality_manager_); @@ -1035,10 +1015,9 @@ TEST_F(TestChooseMigrationSourcePolicy, get_available_src_with_rebuild) EXPECT_EQ(OB_SUCCESS, mock_migrate_choose_helper_param(tenant_id, ls_id, local_ls_checkpoint_scn, mock_arg, param)); EXPECT_EQ(OB_SUCCESS, member_helper_.get_member_list_by_replica_type(tenant_id, ls_id, mock_arg.dst_, param.info_, param.is_first_c_replica_)); - ObStorageHASrcProvider::ChooseSourcePolicy policy; - EXPECT_EQ(OB_SUCCESS, get_idc_policy(mock_arg, tenant_id, param.info_.learner_list_, policy)); + EXPECT_EQ(OB_SUCCESS, get_idc_policy(mock_arg, tenant_id, param.info_.learner_list_, param.policy_, param.use_c_replica_policy_)); ObStorageHASrcInfo src_info; - EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, policy, &storage_rpc_, &member_helper_)); + EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, &storage_rpc_, &member_helper_)); EXPECT_EQ(OB_SUCCESS, mock_locality_manager(MOCKLOCALITY::IDC_MODE_IDC_LEADER, locality_manager_)); EXPECT_EQ(ObStorageHASrcProvider::ChooseSourcePolicy::IDC, choose_src_helper_.get_provider()->get_policy_type()); static_cast(choose_src_helper_.get_provider())->set_locality_manager_(&locality_manager_); @@ -1084,10 +1063,9 @@ TEST_F(TestChooseMigrationSourcePolicy, src_provider_init_idc_fail) param.arg_ = mock_arg; EXPECT_EQ(OB_SUCCESS, member_helper_.get_member_list_by_replica_type(tenant_id, ls_id, mock_arg.dst_, param.info_, param.is_first_c_replica_)); - ObStorageHASrcProvider::ChooseSourcePolicy policy; - EXPECT_EQ(OB_INVALID_ARGUMENT, get_idc_policy(mock_arg, tenant_id, param.info_.learner_list_, policy)); + EXPECT_EQ(OB_INVALID_ARGUMENT, get_idc_policy(mock_arg, tenant_id, param.info_.learner_list_, param.policy_, param.use_c_replica_policy_)); ObStorageHASrcInfo src_info; - EXPECT_EQ(OB_INVALID_ARGUMENT, choose_src_helper_.init(param, policy, &storage_rpc_, &member_helper_)); + EXPECT_EQ(OB_INVALID_ARGUMENT, choose_src_helper_.init(param, &storage_rpc_, &member_helper_)); } // test ObMigrationSrcByLocationProvider init fail TEST_F(TestChooseMigrationSourcePolicy, src_provider_init_region_fail) @@ -1112,10 +1090,9 @@ TEST_F(TestChooseMigrationSourcePolicy, src_provider_init_region_fail) param.arg_ = mock_arg; EXPECT_EQ(OB_SUCCESS, member_helper_.get_member_list_by_replica_type(tenant_id, ls_id, mock_arg.dst_, param.info_, param.is_first_c_replica_)); - ObStorageHASrcProvider::ChooseSourcePolicy policy; - EXPECT_EQ(OB_INVALID_ARGUMENT, get_region_policy(mock_arg, tenant_id, param.info_.learner_list_, policy)); + EXPECT_EQ(OB_INVALID_ARGUMENT, get_region_policy(mock_arg, tenant_id, param.info_.learner_list_, param.policy_, param.use_c_replica_policy_)); ObStorageHASrcInfo src_info; - EXPECT_EQ(OB_INVALID_ARGUMENT, choose_src_helper_.init(param, policy, &storage_rpc_, &member_helper_)); + EXPECT_EQ(OB_INVALID_ARGUMENT, choose_src_helper_.init(param, &storage_rpc_, &member_helper_)); } // test ObRSRecommendSrcProvider init fail TEST_F(TestChooseMigrationSourcePolicy, src_provider_init_recommand_fail) @@ -1140,10 +1117,9 @@ TEST_F(TestChooseMigrationSourcePolicy, src_provider_init_recommand_fail) param.arg_ = mock_arg; EXPECT_EQ(OB_SUCCESS, member_helper_.get_member_list_by_replica_type(tenant_id, ls_id, mock_arg.dst_, param.info_, param.is_first_c_replica_)); - ObStorageHASrcProvider::ChooseSourcePolicy policy; - EXPECT_EQ(OB_INVALID_ARGUMENT, get_recommand_policy(mock_arg, tenant_id, param.info_.learner_list_, policy)); + EXPECT_EQ(OB_INVALID_ARGUMENT, get_recommand_policy(mock_arg, tenant_id, param.info_.learner_list_, param.policy_, param.use_c_replica_policy_)); ObStorageHASrcInfo src_info; - EXPECT_EQ(OB_INVALID_ARGUMENT, choose_src_helper_.init(param, policy, &storage_rpc_, &member_helper_)); + EXPECT_EQ(OB_INVALID_ARGUMENT, choose_src_helper_.init(param, &storage_rpc_, &member_helper_)); } // test ObMigrationSrcByCheckpointProvider init fail TEST_F(TestChooseMigrationSourcePolicy, src_provider_init_checkpoint_fail) @@ -1168,10 +1144,9 @@ TEST_F(TestChooseMigrationSourcePolicy, src_provider_init_checkpoint_fail) param.arg_ = mock_arg; EXPECT_EQ(OB_SUCCESS, member_helper_.get_member_list_by_replica_type(tenant_id, ls_id, mock_arg.dst_, param.info_, param.is_first_c_replica_)); - ObStorageHASrcProvider::ChooseSourcePolicy policy; - EXPECT_EQ(OB_INVALID_ARGUMENT, get_checkpoint_policy(mock_arg, tenant_id, param.info_.learner_list_, policy)); + EXPECT_EQ(OB_INVALID_ARGUMENT, get_checkpoint_policy(mock_arg, tenant_id, param.info_.learner_list_, param.policy_, param.use_c_replica_policy_)); ObStorageHASrcInfo src_info; - EXPECT_EQ(OB_INVALID_ARGUMENT, choose_src_helper_.init(param, policy, &storage_rpc_, &member_helper_)); + EXPECT_EQ(OB_INVALID_ARGUMENT, choose_src_helper_.init(param, &storage_rpc_, &member_helper_)); } // test check replica valid fail // candidate addr: ["192.168.1.1:1234", "192.168.1.2:1234", "192.168.1.3:1234", "192.168.1.4:1234", "192.168.1.5:1234"] @@ -1214,10 +1189,9 @@ TEST_F(TestChooseMigrationSourcePolicy, get_available_src_condition_fail) EXPECT_EQ(OB_SUCCESS, mock_migrate_choose_helper_param(tenant_id, ls_id, local_ls_checkpoint_scn, mock_arg, param)); EXPECT_EQ(OB_SUCCESS, member_helper_.get_member_list_by_replica_type(tenant_id, ls_id, mock_arg.dst_, param.info_, param.is_first_c_replica_)); - ObStorageHASrcProvider::ChooseSourcePolicy policy; - EXPECT_EQ(OB_SUCCESS, get_checkpoint_policy(mock_arg, tenant_id, param.info_.learner_list_, policy)); + EXPECT_EQ(OB_SUCCESS, get_checkpoint_policy(mock_arg, tenant_id, param.info_.learner_list_, param.policy_, param.use_c_replica_policy_)); ObStorageHASrcInfo src_info; - EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, policy, &storage_rpc_, &member_helper_)); + EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, &storage_rpc_, &member_helper_)); EXPECT_EQ(OB_SUCCESS, choose_src_helper_.get_available_src(mock_arg, src_info)); common::ObAddr expect_addr; EXPECT_EQ(OB_SUCCESS, mock_addr("192.168.1.4:1234", expect_addr)); @@ -1264,10 +1238,9 @@ TEST_F(TestChooseMigrationSourcePolicy, idc_mode_check_replica_fail) EXPECT_EQ(OB_SUCCESS, mock_migrate_choose_helper_param(tenant_id, ls_id, local_ls_checkpoint_scn, mock_arg, param)); EXPECT_EQ(OB_SUCCESS, member_helper_.get_member_list_by_replica_type(tenant_id, ls_id, mock_arg.dst_, param.info_, param.is_first_c_replica_)); - ObStorageHASrcProvider::ChooseSourcePolicy policy; - EXPECT_EQ(OB_SUCCESS, get_idc_policy(mock_arg, tenant_id, param.info_.learner_list_, policy)); + EXPECT_EQ(OB_SUCCESS, get_idc_policy(mock_arg, tenant_id, param.info_.learner_list_, param.policy_, param.use_c_replica_policy_)); ObStorageHASrcInfo src_info; - EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, policy, &storage_rpc_, &member_helper_)); + EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, &storage_rpc_, &member_helper_)); EXPECT_EQ(OB_SUCCESS, mock_locality_manager(MOCKLOCALITY::IDC_MODE_IDC_LEADER, locality_manager_)); EXPECT_EQ(ObStorageHASrcProvider::ChooseSourcePolicy::IDC, choose_src_helper_.get_provider()->get_policy_type()); static_cast(choose_src_helper_.get_provider())->set_locality_manager_(&locality_manager_); @@ -1316,10 +1289,9 @@ TEST_F(TestChooseMigrationSourcePolicy, idc_mode_r_replica_init) EXPECT_EQ(OB_SUCCESS, mock_migrate_choose_helper_param(tenant_id, ls_id, local_ls_checkpoint_scn, mock_arg, param)); EXPECT_EQ(OB_SUCCESS, member_helper_.get_member_list_by_replica_type(tenant_id, ls_id, mock_arg.dst_, param.info_, param.is_first_c_replica_)); - ObStorageHASrcProvider::ChooseSourcePolicy policy; - EXPECT_EQ(OB_SUCCESS, get_idc_policy(mock_arg, tenant_id, param.info_.learner_list_, policy)); + EXPECT_EQ(OB_SUCCESS, get_idc_policy(mock_arg, tenant_id, param.info_.learner_list_, param.policy_, param.use_c_replica_policy_)); ObStorageHASrcInfo src_info; - EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, policy, &storage_rpc_, &member_helper_)); + EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, &storage_rpc_, &member_helper_)); EXPECT_EQ(OB_SUCCESS, mock_locality_manager(MOCKLOCALITY::IDC_MODE_IDC_LEADER, locality_manager_)); EXPECT_EQ(ObStorageHASrcProvider::ChooseSourcePolicy::IDC, choose_src_helper_.get_provider()->get_policy_type()); static_cast(choose_src_helper_.get_provider())->set_locality_manager_(&locality_manager_); @@ -1369,10 +1341,9 @@ TEST_F(TestChooseMigrationSourcePolicy, c_replica_no_other) EXPECT_EQ(OB_SUCCESS, mock_migrate_choose_helper_param(tenant_id, ls_id, local_ls_checkpoint_scn, mock_arg, param)); EXPECT_EQ(OB_SUCCESS, member_helper_.get_member_list_by_replica_type(tenant_id, ls_id, mock_arg.dst_, param.info_, param.is_first_c_replica_)); - ObStorageHASrcProvider::ChooseSourcePolicy policy; - EXPECT_EQ(OB_SUCCESS, get_idc_policy(mock_arg, tenant_id, param.info_.learner_list_, policy)); + EXPECT_EQ(OB_SUCCESS, get_idc_policy(mock_arg, tenant_id, param.info_.learner_list_, param.policy_, param.use_c_replica_policy_)); ObStorageHASrcInfo src_info; - EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, policy, &storage_rpc_, &member_helper_)); + EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, &storage_rpc_, &member_helper_)); EXPECT_EQ(OB_SUCCESS, mock_locality_manager(MOCKLOCALITY::IDC_MODE_IDC_LEADER, locality_manager_)); EXPECT_EQ(ObStorageHASrcProvider::ChooseSourcePolicy::IDC, choose_src_helper_.get_provider()->get_policy_type()); static_cast(choose_src_helper_.get_provider())->set_locality_manager_(&locality_manager_); @@ -1396,9 +1367,10 @@ TEST_F(TestChooseMigrationSourcePolicy, c_replica_no_other) TEST_F(TestChooseMigrationSourcePolicy, c_replica_have_other) { MockLsMetaInfo ls_meta; + // when replica type is F, won't fetch ls_info EXPECT_CALL(storage_rpc_, post_ls_meta_info_request(_, _, _, _)) .WillOnce(Invoke(&ls_meta, &MockLsMetaInfo::post_ls_meta_info_request_parent_checkpoint)) - .WillOnce(Invoke(&ls_meta, &MockLsMetaInfo::post_ls_meta_info_request_min_checkpoint)) + // .WillOnce(Invoke(&ls_meta, &MockLsMetaInfo::post_ls_meta_info_request_min_checkpoint)) .WillOnce(Invoke(&ls_meta, &MockLsMetaInfo::post_ls_meta_info_request_min_checkpoint)) .WillOnce(Invoke(&ls_meta, &MockLsMetaInfo::post_ls_meta_info_request_base_checkpoint)) .WillOnce(Invoke(&ls_meta, &MockLsMetaInfo::post_ls_meta_info_request_large_checkpoint)) @@ -1422,11 +1394,11 @@ TEST_F(TestChooseMigrationSourcePolicy, c_replica_have_other) EXPECT_EQ(OB_SUCCESS, mock_migrate_choose_helper_param(tenant_id, ls_id, local_ls_checkpoint_scn, mock_arg, param)); EXPECT_EQ(OB_SUCCESS, member_helper_.get_member_list_by_replica_type(tenant_id, ls_id, mock_arg.dst_, param.info_, param.is_first_c_replica_)); - ObStorageHASrcProvider::ChooseSourcePolicy policy; - EXPECT_EQ(OB_SUCCESS, get_idc_policy(mock_arg, tenant_id, param.info_.learner_list_, policy)); + EXPECT_EQ(OB_SUCCESS, get_idc_policy(mock_arg, tenant_id, param.info_.learner_list_, param.policy_, param.use_c_replica_policy_)); ObStorageHASrcInfo src_info; - EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, policy, &storage_rpc_, &member_helper_)); - EXPECT_EQ(ObStorageHASrcProvider::ChooseSourcePolicy::C_TYPE_REPLICA, choose_src_helper_.get_provider()->get_policy_type()); + EXPECT_EQ(OB_SUCCESS, choose_src_helper_.init(param, &storage_rpc_, &member_helper_)); + EXPECT_EQ(true, param.use_c_replica_policy_); + EXPECT_EQ(OB_SUCCESS, mock_locality_manager(MOCKLOCALITY::IDC_MODE_IDC_LEADER, locality_manager_)); static_cast(choose_src_helper_.get_provider())->set_locality_manager_(&locality_manager_); EXPECT_EQ(OB_SUCCESS, choose_src_helper_.get_available_src(mock_arg, src_info)); common::ObAddr expect_addr; diff --git a/unittest/storage/migration/test_migration.h b/unittest/storage/migration/test_migration.h index d7edbb169..d4d242054 100644 --- a/unittest/storage/migration/test_migration.h +++ b/unittest/storage/migration/test_migration.h @@ -511,58 +511,70 @@ static share::SCN mock_ckpt_inc(share::SCN &local_ls_checkpoint_scn) return result; } -static int get_checkpoint_policy(const ObMigrationOpArg &arg, const uint64_t tenant_id, +static int get_checkpoint_policy( + const ObMigrationOpArg &arg, + const uint64_t tenant_id, const common::GlobalLearnerList &learner_list, - ObStorageHASrcProvider::ChooseSourcePolicy &policy) + ObStorageHASrcProvider::ChooseSourcePolicy &policy, + bool &use_c_replica_policy) { int ret = OB_SUCCESS; bool enable_choose_source_policy = false; const char *str = "idc"; if (OB_FAIL(ObStorageHAChooseSrcHelper::get_policy_type(arg, tenant_id, - enable_choose_source_policy, str, learner_list, policy))) { + enable_choose_source_policy, str, learner_list, policy, use_c_replica_policy))) { LOG_WARN("failed to get policy type", K(ret), K(arg), K(tenant_id), K(learner_list)); } return ret; } -static int get_recommand_policy(const ObMigrationOpArg &arg, const uint64_t tenant_id, +static int get_recommand_policy( + const ObMigrationOpArg &arg, + const uint64_t tenant_id, const common::GlobalLearnerList &learner_list, - ObStorageHASrcProvider::ChooseSourcePolicy &policy) + ObStorageHASrcProvider::ChooseSourcePolicy &policy, + bool &use_c_replica_policy) { int ret = OB_SUCCESS; bool enable_choose_source_policy = true; const char *str = "idc"; if (OB_FAIL(ObStorageHAChooseSrcHelper::get_policy_type(arg, tenant_id, - enable_choose_source_policy, str, learner_list, policy))) { + enable_choose_source_policy, str, learner_list, policy, use_c_replica_policy))) { LOG_WARN("failed to get policy type", K(ret), K(arg), K(tenant_id), K(learner_list)); } return ret; } -static int get_idc_policy(const ObMigrationOpArg &arg, const uint64_t tenant_id, +static int get_idc_policy( + const ObMigrationOpArg &arg, + const uint64_t tenant_id, const common::GlobalLearnerList &learner_list, - ObStorageHASrcProvider::ChooseSourcePolicy &policy) + ObStorageHASrcProvider::ChooseSourcePolicy &policy, + bool &use_c_replica_policy) { int ret = OB_SUCCESS; bool enable_choose_source_policy = true; const char *str = "idc"; if (OB_FAIL(ObStorageHAChooseSrcHelper::get_policy_type(arg, tenant_id, - enable_choose_source_policy, str, learner_list, policy))) { + enable_choose_source_policy, str, learner_list, policy, use_c_replica_policy))) { LOG_WARN("failed to get policy type", K(ret), K(arg), K(tenant_id), K(learner_list)); } return ret; } -static int get_region_policy(const ObMigrationOpArg &arg, const uint64_t tenant_id, +static int get_region_policy( + const ObMigrationOpArg &arg, + const uint64_t tenant_id, const common::GlobalLearnerList &learner_list, - ObStorageHASrcProvider::ChooseSourcePolicy &policy) + ObStorageHASrcProvider::ChooseSourcePolicy &policy, + bool &use_c_replica_policy) { int ret = OB_SUCCESS; bool enable_choose_source_policy = true; const char *str = "region"; if (OB_FAIL(ObStorageHAChooseSrcHelper::get_policy_type(arg, tenant_id, - enable_choose_source_policy, str, learner_list, policy))) { + enable_choose_source_policy, str, learner_list, policy, use_c_replica_policy))) { LOG_WARN("failed to get policy type", K(ret), K(arg), K(tenant_id), K(learner_list)); } return ret;