From e5b7cbc59365fac7aa8c20eb591bc482c863233f Mon Sep 17 00:00:00 2001 From: yinyj17 Date: Thu, 4 Jan 2024 16:17:26 +0000 Subject: [PATCH] rebuild master key version info after restore finished --- deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h | 1 + src/observer/ob_srv_xlator_rootserver.cpp | 1 + src/rootserver/ob_root_service.cpp | 20 ++++++++++ src/rootserver/ob_root_service.h | 2 + src/rootserver/ob_rs_rpc_processor.h | 1 + .../restore/ob_restore_scheduler.cpp | 39 ++++++++++++++++++- src/rootserver/restore/ob_restore_scheduler.h | 2 +- src/share/ob_common_rpc_proxy.h | 1 + src/share/ob_rpc_struct.cpp | 19 +++++++++ src/share/ob_rpc_struct.h | 26 +++++++++++++ 10 files changed, 109 insertions(+), 3 deletions(-) diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h index f646f24f1e..0189d3d871 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h @@ -766,6 +766,7 @@ PCODE_DEF(OB_SET_ROOT_KEY, 0x861) PCODE_DEF(OB_GET_ROOT_KEY, 0x862) PCODE_DEF(OB_CDC_FETCH_RAW_LOG, 0x863) +PCODE_DEF(OB_RELOAD_MASTER_KEY, 0x864) //partition service //PCODE_DEF(OB_PTS_FETCH_INFO, 0x901) diff --git a/src/observer/ob_srv_xlator_rootserver.cpp b/src/observer/ob_srv_xlator_rootserver.cpp index 8257442df0..23c3915c0b 100644 --- a/src/observer/ob_srv_xlator_rootserver.cpp +++ b/src/observer/ob_srv_xlator_rootserver.cpp @@ -284,6 +284,7 @@ void oceanbase::observer::init_srv_xlator_for_rootserver(ObSrvRpcXlator *xlator) RPC_PROCESSOR(rootserver::ObRpcHandleRlsContextDDLP, *gctx_.root_service_); #ifdef OB_BUILD_TDE_SECURITY RPC_PROCESSOR(rootserver::ObGetRootKeyP, *gctx_.root_service_); + RPC_PROCESSOR(rootserver::ObReloadMasterKeyP, *gctx_.root_service_); #endif } diff --git a/src/rootserver/ob_root_service.cpp b/src/rootserver/ob_root_service.cpp index ff2e28c5a6..e7f089d47d 100755 --- a/src/rootserver/ob_root_service.cpp +++ b/src/rootserver/ob_root_service.cpp @@ -11295,6 +11295,26 @@ int ObRootService::get_root_key_from_obs_(const obrpc::ObRootKeyArg &arg, } return ret; } + +int ObRootService::reload_master_key(const obrpc::ObReloadMasterKeyArg &arg, + obrpc::ObReloadMasterKeyResult &result) +{ + int ret = OB_SUCCESS; + uint64_t max_version = 0; + if (!inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_UNLIKELY(!arg.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", K(arg), K(ret)); + } else if (OB_FAIL(master_key_mgr_.reload_tenant_max_key_version(arg.tenant_id_, max_version))) { + LOG_WARN("failed to reload master key version", K(ret), K(arg)); + } else { + result.tenant_id_ = arg.tenant_id_; + result.master_key_id_ = max_version; + } + return ret; +} #endif } // end namespace rootserver diff --git a/src/rootserver/ob_root_service.h b/src/rootserver/ob_root_service.h index 4bde4c1d3c..fbb55a4851 100644 --- a/src/rootserver/ob_root_service.h +++ b/src/rootserver/ob_root_service.h @@ -817,6 +817,8 @@ public: int try_add_dep_infos_for_synonym_batch(const obrpc::ObTryAddDepInofsForSynonymBatchArg &arg); #ifdef OB_BUILD_TDE_SECURITY int handle_get_root_key(const obrpc::ObRootKeyArg &arg, obrpc::ObRootKeyResult &result); + int reload_master_key(const obrpc::ObReloadMasterKeyArg &arg, + obrpc::ObReloadMasterKeyResult &result); #endif private: #ifdef OB_BUILD_TDE_SECURITY diff --git a/src/rootserver/ob_rs_rpc_processor.h b/src/rootserver/ob_rs_rpc_processor.h index 0b3cad52c8..971b6fa1d9 100644 --- a/src/rootserver/ob_rs_rpc_processor.h +++ b/src/rootserver/ob_rs_rpc_processor.h @@ -559,6 +559,7 @@ DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_HANDLE_RLS_GROUP_DDL, ObRpcHandleRlsGroupD DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_HANDLE_RLS_CONTEXT_DDL, ObRpcHandleRlsContextDDLP, handle_rls_context_ddl(arg_)); #ifdef OB_BUILD_TDE_SECURITY DEFINE_RS_RPC_PROCESSOR(obrpc::OB_GET_ROOT_KEY, ObGetRootKeyP, handle_get_root_key(arg_, result_)); +DEFINE_RS_RPC_PROCESSOR(obrpc::OB_RELOAD_MASTER_KEY, ObReloadMasterKeyP, reload_master_key(arg_, result_)); #endif #undef DEFINE_RS_RPC_PROCESSOR_ diff --git a/src/rootserver/restore/ob_restore_scheduler.cpp b/src/rootserver/restore/ob_restore_scheduler.cpp index faf39aa33a..fa57516a10 100644 --- a/src/rootserver/restore/ob_restore_scheduler.cpp +++ b/src/rootserver/restore/ob_restore_scheduler.cpp @@ -467,7 +467,7 @@ int ObRestoreScheduler::fill_restore_statistics(const share::ObPhysicalRestoreJo return ret; } -int ObRestoreScheduler::convert_parameters( +int ObRestoreScheduler::convert_tde_parameters( const ObPhysicalRestoreJob &job_info) { int ret = OB_SUCCESS; @@ -516,6 +516,41 @@ int ObRestoreScheduler::convert_parameters( LOG_WARN("failed to execute", K(ret), K(affected_row), K(sql)); } } + if (OB_SUCC(ret)) { + const int64_t DEFAULT_TIMEOUT = GCONF.internal_sql_execute_timeout; + obrpc::ObReloadMasterKeyArg arg; + obrpc::ObReloadMasterKeyResult result; + arg.tenant_id_ = tenant_id; + if (OB_FAIL(rpc_proxy_->timeout(DEFAULT_TIMEOUT).reload_master_key(arg, result))) { + LOG_WARN("fail to reload master key", K(ret), K(arg), K(DEFAULT_TIMEOUT)); + } else if (result.master_key_id_ > 0 ) { + bool is_active = false; + const int64_t SLEEP_US = 5 * 1000 * 1000L; // 5s + const int64_t MAX_WAIT_US = 60 * 1000 * 1000L; // 60s + const int64_t start = ObTimeUtility::current_time(); + char master_key[OB_MAX_MASTER_KEY_LENGTH]; + int64_t master_key_len = 0; + uint64_t master_key_id = 0; + while (OB_SUCC(ret) && !is_active) { + if (ObTimeUtility::current_time() - start > MAX_WAIT_US) { + ret = OB_TIMEOUT; + LOG_WARN("use too much time", K(ret), "cost_us", ObTimeUtility::current_time() - start); + } else if (OB_FAIL(ObMasterKeyGetter::get_active_master_key(tenant_id, master_key, + OB_MAX_MASTER_KEY_LENGTH, + master_key_len, master_key_id))) { + if (OB_KEYSTORE_OPEN_NO_MASTER_KEY == ret) { + ret = OB_SUCCESS; + LOG_INFO("master key is not active, need wait", K(tenant_id)); + usleep(SLEEP_US); + } else { + LOG_WARN("fail to get active master key", K(tenant_id)); + } + } else { + is_active = true; + } + } + } + } #endif return ret; } @@ -629,7 +664,7 @@ int ObRestoreScheduler::post_check(const ObPhysicalRestoreJob &job_info) LOG_WARN("failed to process schema", KR(ret)); } - if (FAILEDx(convert_parameters(job_info))) { + if (FAILEDx(convert_tde_parameters(job_info))) { LOG_WARN("fail to convert parameters", K(ret), K(job_info)); } diff --git a/src/rootserver/restore/ob_restore_scheduler.h b/src/rootserver/restore/ob_restore_scheduler.h index d623cd3459..1c07313cef 100644 --- a/src/rootserver/restore/ob_restore_scheduler.h +++ b/src/rootserver/restore/ob_restore_scheduler.h @@ -72,7 +72,7 @@ private: int fill_create_tenant_arg(const share::ObPhysicalRestoreJob &job_info, const ObSqlString &pool_list, obrpc::ObCreateTenantArg &arg); - int convert_parameters(const share::ObPhysicalRestoreJob &job_info); + int convert_tde_parameters(const share::ObPhysicalRestoreJob &job_info); int restore_root_key(const share::ObPhysicalRestoreJob &job_info); int restore_keystore(const share::ObPhysicalRestoreJob &job_info); diff --git a/src/share/ob_common_rpc_proxy.h b/src/share/ob_common_rpc_proxy.h index 73d406e31a..8f7dc6aad5 100644 --- a/src/share/ob_common_rpc_proxy.h +++ b/src/share/ob_common_rpc_proxy.h @@ -318,6 +318,7 @@ public: RPC_S(PRD try_add_dep_infos_for_synonym_batch, obrpc::OB_TRY_ADD_DEP_INFOS_FOR_SYNONYM_BATCH, (ObTryAddDepInofsForSynonymBatchArg)); #ifdef OB_BUILD_TDE_SECURITY RPC_S(PR5 get_root_key, obrpc::OB_GET_ROOT_KEY, (obrpc::ObRootKeyArg), obrpc::ObRootKeyResult); + RPC_S(PR5 reload_master_key, obrpc::OB_RELOAD_MASTER_KEY, (obrpc::ObReloadMasterKeyArg), obrpc::ObReloadMasterKeyResult); #endif public: void set_rs_mgr(share::ObRsMgr &rs_mgr) diff --git a/src/share/ob_rpc_struct.cpp b/src/share/ob_rpc_struct.cpp index 7a97f5062e..6ab1fec9d6 100755 --- a/src/share/ob_rpc_struct.cpp +++ b/src/share/ob_rpc_struct.cpp @@ -7743,6 +7743,25 @@ void ObRootKeyResult::reset() key_type_ = RootKeyType::INVALID; root_key_.reset(); } + +OB_SERIALIZE_MEMBER(ObReloadMasterKeyArg, tenant_id_); + +int ObReloadMasterKeyArg::assign(const ObReloadMasterKeyArg &other) +{ + int ret = OB_SUCCESS; + tenant_id_ = other.tenant_id_; + return ret; +} + +OB_SERIALIZE_MEMBER(ObReloadMasterKeyResult, tenant_id_, master_key_id_); + +int ObReloadMasterKeyResult::assign(const ObReloadMasterKeyResult &other) +{ + int ret = OB_SUCCESS; + tenant_id_ = other.tenant_id_; + master_key_id_ = other.master_key_id_; + return ret; +} #endif OB_SERIALIZE_MEMBER(ObTrxToolArg, trans_id_, status_, trans_version_, end_log_ts_, cmd_); diff --git a/src/share/ob_rpc_struct.h b/src/share/ob_rpc_struct.h index 6f88a916aa..7d7399ed22 100755 --- a/src/share/ob_rpc_struct.h +++ b/src/share/ob_rpc_struct.h @@ -9150,6 +9150,32 @@ public: private: common::ObArenaAllocator allocator_; }; + +struct ObReloadMasterKeyArg +{ + OB_UNIS_VERSION(1); +public: + ObReloadMasterKeyArg(): tenant_id_(OB_INVALID_ID) + {} + ~ObReloadMasterKeyArg() {} + int assign(const ObReloadMasterKeyArg &other); + bool is_valid() const { return OB_INVALID_ID != tenant_id_; } + TO_STRING_KV(K_(tenant_id)); + uint64_t tenant_id_; +}; + +struct ObReloadMasterKeyResult +{ + OB_UNIS_VERSION(1); +public: + ObReloadMasterKeyResult(): tenant_id_(OB_INVALID_ID), master_key_id_(OB_INVALID_ID) + {} + ~ObReloadMasterKeyResult() {} + int assign(const ObReloadMasterKeyResult &other); + TO_STRING_KV(K_(tenant_id), K_(master_key_id)); + uint64_t tenant_id_; + uint64_t master_key_id_; +}; #endif struct TenantServerUnitConfig