diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h index ac8f913e5b..93108c6d38 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h @@ -482,7 +482,7 @@ PCODE_DEF(OB_HA_GET_CONFIG_CHANGE_LOCK_STAT, 0x4B8) PCODE_DEF(OB_HA_WAKEUP_TRANSFER_SERVICE, 0x4B9) PCODE_DEF(OB_HA_SUBMIT_TX_LOG, 0x4BA) PCODE_DEF(OB_HA_FETCH_LS_MEMBER_AND_LEARNER_LIST, 0x4BB) - +PCODE_DEF(OB_HA_UNLOCK_MEMBER_LIST, 0x4BC) // sql, including executor // sql diff --git a/src/observer/ob_rpc_processor_simple.cpp b/src/observer/ob_rpc_processor_simple.cpp index 01918ae704..cc7e6ba598 100644 --- a/src/observer/ob_rpc_processor_simple.cpp +++ b/src/observer/ob_rpc_processor_simple.cpp @@ -2798,5 +2798,18 @@ int ObRpcGetLSReplayedScnP::process() return ret; } +int ObAdminUnlockMemberListP::process() +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(gctx_.ob_service_)) { + ret = OB_ERR_UNEXPECTED; + COMMON_LOG(WARN, "ob_service is null", KR(ret)); + } else if (OB_FAIL(gctx_.ob_service_->ob_admin_unlock_member_list(arg_))) { + COMMON_LOG(WARN, "failed to unlock member list", KR(ret), K(arg_)); + } + return ret; +} + + } // end of namespace observer } // end of namespace oceanbase diff --git a/src/observer/ob_rpc_processor_simple.h b/src/observer/ob_rpc_processor_simple.h index 7f90da8e59..38f0d9d18f 100644 --- a/src/observer/ob_rpc_processor_simple.h +++ b/src/observer/ob_rpc_processor_simple.h @@ -253,6 +253,8 @@ OB_DEFINE_PROCESSOR_S(Srv, OB_GET_SERVER_RESOURCE_INFO, ObRpcGetServerResourceIn OB_DEFINE_PROCESSOR_S(Srv, OB_UPDATE_TENANT_INFO_CACHE, ObUpdateTenantInfoCacheP); OB_DEFINE_PROCESSOR_S(Srv, OB_BROADCAST_CONSENSUS_VERSION, ObBroadcastConsensusVersionP); OB_DEFINE_PROCESSOR_S(Srv, OB_GET_LS_REPLAYED_SCN, ObRpcGetLSReplayedScnP); +OB_DEFINE_PROCESSOR_S(Srv, OB_HA_UNLOCK_MEMBER_LIST, ObAdminUnlockMemberListP); + } // end of namespace observer } // end of namespace oceanbase diff --git a/src/observer/ob_service.cpp b/src/observer/ob_service.cpp index 46a01e25f3..9fba903788 100644 --- a/src/observer/ob_service.cpp +++ b/src/observer/ob_service.cpp @@ -84,6 +84,7 @@ #include "share/backup/ob_backup_struct.h" #include "observer/ob_heartbeat_handler.h" #include "storage/slog/ob_storage_logger_manager.h" +#include "storage/high_availability/ob_transfer_lock_utils.h" namespace oceanbase { @@ -3094,5 +3095,33 @@ int ObService::update_tenant_info_cache( return ret; } +int ObService::ob_admin_unlock_member_list( + const obrpc::ObAdminUnlockMemberListOpArg &arg) +{ + LOG_INFO("start ob_admin_unlock_member_list", K(arg)); + int ret = OB_SUCCESS; + MAKE_TENANT_SWITCH_SCOPE_GUARD(guard); + + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (!arg.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("arg is invaild", KR(ret), K(arg)); + } else if (arg.tenant_id_ != MTL_ID() && OB_FAIL(guard.switch_to(arg.tenant_id_))) { + LOG_WARN("switch tenant failed", KR(ret), K(arg)); + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(ObMemberListLockUtils::unlock_for_ob_admin(arg.tenant_id_, arg.ls_id_, arg.lock_id_))) { + LOG_WARN("failed to unlock member list", K(ret), K(arg)); + } else { + LOG_INFO("finish ob_admin_unlock_member_list", K(arg)); + } + } + return ret; +} + + }// end namespace observer }// end namespace oceanbase diff --git a/src/observer/ob_service.h b/src/observer/ob_service.h index 0191bf68cd..437ab634ef 100644 --- a/src/observer/ob_service.h +++ b/src/observer/ob_service.h @@ -257,6 +257,9 @@ public: int handle_heartbeat( const share::ObHBRequest &hb_request, share::ObHBResponse &hb_response); + int ob_admin_unlock_member_list( + const obrpc::ObAdminUnlockMemberListOpArg &arg); + private: int get_role_from_palf_( logservice::ObLogService &log_service, diff --git a/src/observer/ob_srv_xlator_partition.cpp b/src/observer/ob_srv_xlator_partition.cpp index d1dd440c98..c25025baa3 100644 --- a/src/observer/ob_srv_xlator_partition.cpp +++ b/src/observer/ob_srv_xlator_partition.cpp @@ -169,6 +169,7 @@ void oceanbase::observer::init_srv_xlator_for_migration(ObSrvRpcXlator *xlator) RPC_PROCESSOR(ObStorageGetLogConfigStatP, gctx_.bandwidth_throttle_); RPC_PROCESSOR(ObStorageWakeupTransferServiceP, gctx_.bandwidth_throttle_); RPC_PROCESSOR(ObFetchLSMemberAndLearnerListP); + RPC_PROCESSOR(ObAdminUnlockMemberListP, gctx_); } void oceanbase::observer::init_srv_xlator_for_others(ObSrvRpcXlator *xlator) { diff --git a/src/share/ob_rpc_struct.cpp b/src/share/ob_rpc_struct.cpp index c46407aa56..bbf5c231a8 100755 --- a/src/share/ob_rpc_struct.cpp +++ b/src/share/ob_rpc_struct.cpp @@ -9237,5 +9237,49 @@ int ObLoadBaselineRes::assign(const ObLoadBaselineRes &other) } OB_SERIALIZE_MEMBER(ObLoadBaselineRes, load_count_); +ObAdminUnlockMemberListOpArg::ObAdminUnlockMemberListOpArg() + : tenant_id_(OB_INVALID_ID), + ls_id_(), + lock_id_(-1) +{ +} + +ObAdminUnlockMemberListOpArg::~ObAdminUnlockMemberListOpArg() +{ +} + +void ObAdminUnlockMemberListOpArg::reset() +{ + tenant_id_ = OB_INVALID_ID; + ls_id_.reset(); + lock_id_ = -1; +} + +bool ObAdminUnlockMemberListOpArg::is_valid() const +{ + return OB_INVALID_ID != tenant_id_ + && ls_id_.is_valid() + && lock_id_ >= 0; +} + +int ObAdminUnlockMemberListOpArg::set( + const uint64_t tenant_id, + const share::ObLSID &ls_id, + const int64_t lock_id) +{ + int ret = OB_SUCCESS; + if (OB_INVALID_ID == tenant_id || !ls_id.is_valid() || lock_id < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("set unlock mmeber list arg get invalid argument", K(ret), K(tenant_id), K(ls_id), K(lock_id)); + } else { + tenant_id_ = tenant_id; + ls_id_ = ls_id; + lock_id_ = lock_id; + } + return ret; +} + +OB_SERIALIZE_MEMBER(ObAdminUnlockMemberListOpArg, tenant_id_, ls_id_, lock_id_); + }//end namespace obrpc }//end namepsace oceanbase diff --git a/src/share/ob_rpc_struct.h b/src/share/ob_rpc_struct.h index 783ab9e2fa..a09397bffe 100755 --- a/src/share/ob_rpc_struct.h +++ b/src/share/ob_rpc_struct.h @@ -9949,6 +9949,25 @@ public: uint64_t load_count_; }; +struct ObAdminUnlockMemberListOpArg final +{ + OB_UNIS_VERSION(1); +public: + ObAdminUnlockMemberListOpArg(); + ~ObAdminUnlockMemberListOpArg(); + int set(const uint64_t tenant_id, + const share::ObLSID &ls_id, + const int64_t lock_id); + bool is_valid() const; + void reset(); + TO_STRING_KV(K_(tenant_id), K_(ls_id), K_(lock_id)); +public: + uint64_t tenant_id_; + share::ObLSID ls_id_; + int64_t lock_id_; +}; + + }//end namespace obrpc }//end namespace oceanbase #endif diff --git a/src/share/ob_srv_rpc_proxy.h b/src/share/ob_srv_rpc_proxy.h index f0fe791bec..8609fd7308 100755 --- a/src/share/ob_srv_rpc_proxy.h +++ b/src/share/ob_srv_rpc_proxy.h @@ -235,6 +235,7 @@ public: RPC_AP(PR5 update_tenant_info_cache, OB_UPDATE_TENANT_INFO_CACHE, (obrpc::ObUpdateTenantInfoCacheArg), obrpc::ObUpdateTenantInfoCacheRes); RPC_AP(PR5 broadcast_consensus_version, OB_BROADCAST_CONSENSUS_VERSION, (obrpc::ObBroadcastConsensusVersionArg), obrpc::ObBroadcastConsensusVersionRes); RPC_S(PR5 direct_load_control, OB_DIRECT_LOAD_CONTROL, (observer::ObDirectLoadControlRequest), observer::ObDirectLoadControlResult); + RPC_S(PR5 admin_unlock_member_list_op, OB_HA_UNLOCK_MEMBER_LIST, (obrpc::ObAdminUnlockMemberListOpArg)); }; // end of class ObSrvRpcProxy } // end of namespace rpc diff --git a/src/storage/high_availability/ob_transfer_handler.cpp b/src/storage/high_availability/ob_transfer_handler.cpp index 38400ed4ad..2adacf606b 100644 --- a/src/storage/high_availability/ob_transfer_handler.cpp +++ b/src/storage/high_availability/ob_transfer_handler.cpp @@ -1552,6 +1552,7 @@ int ObTransferHandler::update_transfer_status_aborted_( ret = OB_NOT_INIT; LOG_WARN("transfer handler do not init", K(ret)); } else if (!task_info.is_valid()) { + ret = OB_INVALID_ARGUMENT; LOG_WARN("update transfer status aborted get invalid argument", K(ret), K(task_info)); } else if (OB_FAIL(start_trans_(timeout_ctx, trans))) { LOG_WARN("failed to start trans", K(ret), K(task_info)); diff --git a/src/storage/high_availability/ob_transfer_lock_utils.cpp b/src/storage/high_availability/ob_transfer_lock_utils.cpp index 168cd9c16f..ac94dd6f3f 100644 --- a/src/storage/high_availability/ob_transfer_lock_utils.cpp +++ b/src/storage/high_availability/ob_transfer_lock_utils.cpp @@ -630,5 +630,41 @@ void ObMemberListLockUtils::destory_storage_rpc_( storage_rpc.destroy(); } +int ObMemberListLockUtils::unlock_for_ob_admin( + const uint64_t tenant_id, + const share::ObLSID &ls_id, + const int64_t lock_id) +{ + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + const int64_t lock_timeout = CONFIG_CHANGE_TIMEOUT; + ObLSHandle ls_handle; + ObLS *ls = nullptr; + bool palf_is_locked = false; + int64_t palf_lock_owner = -1; + + if (OB_INVALID_ID == tenant_id || !ls_id.is_valid() || lock_id < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid args", K(ret), K(tenant_id), K(ls_id), K(lock_id)); + } else if (OB_FAIL(get_ls_handle(tenant_id, ls_id, ls_handle))) { + LOG_WARN("failed to get ls handle", K(ret), K(tenant_id), K(ls_id)); + } else if (OB_ISNULL(ls = ls_handle.get_ls())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls should not be null", K(ret)); + } else if (OB_FAIL(ls->get_config_change_lock_stat(palf_lock_owner, palf_is_locked))) { + LOG_WARN("failed to get config change lock stat", K(ret)); + } else if (!palf_is_locked) { + //do nothing + } else if (lock_id != palf_lock_owner) { + ret = OB_ERR_UNEXPECTED_LOCK_OWNER; + LOG_WARN("palf lock owner and inner table not same", K(ret), K(palf_lock_owner), K(lock_id)); + } else if (OB_FAIL(ls->unlock_config_change(palf_lock_owner, lock_timeout))) { + LOG_WARN("failed to unlock config change", K(ret), K(palf_lock_owner)); + } + + LOG_INFO("unlock for ob admin", K(ret), K(tenant_id), K(ls_id), K(palf_is_locked), K(palf_lock_owner)); + return ret; +} + } // namespace storage } // namespace oceanbase diff --git a/src/storage/high_availability/ob_transfer_lock_utils.h b/src/storage/high_availability/ob_transfer_lock_utils.h index 5e59ffff34..8fdda729f4 100644 --- a/src/storage/high_availability/ob_transfer_lock_utils.h +++ b/src/storage/high_availability/ob_transfer_lock_utils.h @@ -32,6 +32,7 @@ public: const common::ObMemberList &member_list, const ObTransferLockStatus &status, common::ObMySQLProxy &sql_proxy); static int unlock_ls_member_list(const uint64_t tenant_id, const share::ObLSID &ls_id, const int64_t task_id, const common::ObMemberList &member_list, const ObTransferLockStatus &status, common::ObMySQLProxy &sql_proxy); + static int unlock_for_ob_admin(const uint64_t tenant_id, const share::ObLSID &ls_id, const int64_t lock_id); public: /* interface used for primary switch over to standby */ diff --git a/tools/ob_admin/server_tool/ob_admin_routine.cpp b/tools/ob_admin/server_tool/ob_admin_routine.cpp index a159d2ae16..25656fdf38 100644 --- a/tools/ob_admin/server_tool/ob_admin_routine.cpp +++ b/tools/ob_admin/server_tool/ob_admin_routine.cpp @@ -823,4 +823,43 @@ DEF_COMMAND(TRANS, force_clear_arb_cluster_info, 1, "cluster_id # force_clear_ar COMMON_LOG(INFO, "force_clear_arb_cluster_info", K(cluster_id_to_clean)); return ret; } -#endif \ No newline at end of file +#endif + +// unlock_member_list +// @params [in] tenant_id, which tenant to modify +// @params [in] ls_id, which log stream to modify +DEF_COMMAND(SERVER, unlock_member_list, 1, "tenant_id:ls_id:lock_id # unlock_member_list") +{ + int ret = OB_SUCCESS; + string arg_str; + ObAdminUnlockMemberListOpArg arg; + uint64_t tenant_id_to_set = OB_INVALID_TENANT_ID; + int64_t ls_id_to_set = 0; + int64_t lock_id = -1; + + if (cmd_ == action_name_) { + ret = OB_INVALID_ARGUMENT; + ADMIN_WARN("should provide tenant_id, ls_id"); + } else { + arg_str = cmd_.substr(action_name_.length() + 1); + } + + if (OB_FAIL(ret)) { + } else if (3 != sscanf(arg_str.c_str(), "%ld:%ld:%ld", &tenant_id_to_set, &ls_id_to_set, &lock_id)) { + ret = OB_INVALID_ARGUMENT; + COMMON_LOG(WARN, "invalid arg", K(ret), K(arg_str.c_str()), K(cmd_.c_str()), + K(tenant_id_to_set), K(ls_id_to_set), K(lock_id)); + } else { + share::ObLSID ls_id(ls_id_to_set); + if (OB_INVALID_ID == tenant_id_to_set || !ls_id.is_valid()) { + ret = OB_INVALID_ARGUMENT; + COMMON_LOG(WARN, "argument is invalid", K(ret), K(tenant_id_to_set), K(ls_id)); + } else if (OB_FAIL(arg.set(tenant_id_to_set, ls_id, lock_id))) { + COMMON_LOG(WARN, "failed to set unlock member list op arg", K(ret), K(tenant_id_to_set), K(ls_id), K(lock_id)); + } else if (OB_FAIL(client_->admin_unlock_member_list_op(arg))) { + COMMON_LOG(ERROR, "send req fail", K(ret)); + } + } + COMMON_LOG(INFO, "unlock_member_list", K(ret), K(arg)); + return ret; +}