diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h index b4018f6898..c088053169 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h @@ -720,7 +720,9 @@ PCODE_DEF(OB_DUMP_TENANT_CACHE_MASTER_KEY, 0x85A) //PCODE_DEF(OB_LOG_REQ_UNLOAD_PROXY, 0x85D) //PCODE_DEF(OB_LOG_REQ_LOAD_PROXY_PROGRESS, 0x85E) PCODE_DEF(OB_HANDLE_PART_TRANS_CTX, 0x85F) - +PCODE_DEF(OB_RESTORE_KEY, 0x860) +PCODE_DEF(OB_SET_ROOT_KEY, 0x861) +PCODE_DEF(OB_GET_ROOT_KEY, 0x862) //partition service //PCODE_DEF(OB_PTS_FETCH_INFO, 0x901) diff --git a/src/objit/include/objit/common/ob_item_type.h b/src/objit/include/objit/common/ob_item_type.h index 150c8878de..f7e121cbab 100755 --- a/src/objit/include/objit/common/ob_item_type.h +++ b/src/objit/include/objit/common/ob_item_type.h @@ -2020,6 +2020,7 @@ typedef enum ObItemType T_BACKUP_MANAGE, T_BACKUP_CLEAN, T_DELETE_POLICY, + T_BACKUP_KEY, T_RESTORE_TENANT_2, T_GEN_ROWS, T_LOAD_BATCH_SIZE, diff --git a/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp b/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp index e31d63c9dd..a765fc8069 100644 --- a/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp +++ b/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp @@ -393,7 +393,9 @@ int ObBackupSetTaskMgr::backup_user_meta_() ret = OB_ERR_UNEXPECTED; LOG_WARN("[DATA_BACKUP]no lostream task", K(ret), "job_id", job_attr_->job_id_, "tenant_id", job_attr_->tenant_id_); } else if (OB_FAIL(do_backup_meta_(ls_task, finish_cnt))) { - LOG_WARN("[DATA_BACKUP]failed to do backuo meta", K(ret), K(ls_task)); + LOG_WARN("[DATA_BACKUP]failed to do backup meta", K(ret), K(ls_task)); + } else if (OB_FAIL(do_backup_root_key_())) { + LOG_WARN("[DATA_BACKUP]failed to do backup root key", K(ret)); } else if (ls_task.count() == finish_cnt) { ROOTSERVICE_EVENT_ADD("backup_data", "before_backup_data"); DEBUG_SYNC(BEFORE_BACKUP_DATA); @@ -796,6 +798,15 @@ int ObBackupSetTaskMgr::do_backup_meta_(ObArray &ls_task, in return ret; } +int ObBackupSetTaskMgr::do_backup_root_key_() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(store_.write_root_key_info(job_attr_->tenant_id_))) { + LOG_WARN("failed to write root key info"); + } + return ret; +} + int ObBackupSetTaskMgr::get_next_status_(const share::ObBackupStatus &cur_status, share::ObBackupStatus &next_status) { int ret = OB_SUCCESS; diff --git a/src/rootserver/backup/ob_backup_data_set_task_mgr.h b/src/rootserver/backup/ob_backup_data_set_task_mgr.h index 33f1b62708..fa72f58db4 100644 --- a/src/rootserver/backup/ob_backup_data_set_task_mgr.h +++ b/src/rootserver/backup/ob_backup_data_set_task_mgr.h @@ -56,6 +56,7 @@ private: int disable_transfer_(); int enable_transfer_(); int do_backup_meta_(ObArray &ls_task, int64_t &finish_cnt); + int do_backup_root_key_(); int merge_tablet_to_ls_info_(const ObIArray &ls_tasks); int construct_ls_task_map_(const ObIArray &ls_tasks, hash::ObHashMap &ls_map); diff --git a/src/rootserver/ob_bootstrap.cpp b/src/rootserver/ob_bootstrap.cpp index a2e92714b1..cb9d718297 100644 --- a/src/rootserver/ob_bootstrap.cpp +++ b/src/rootserver/ob_bootstrap.cpp @@ -253,6 +253,8 @@ int ObPreBootstrap::prepare_bootstrap(ObAddr &master_rs) } else if (!is_empty) { ret = OB_INIT_TWICE; LOG_WARN("cannot do bootstrap on not empty server", KR(ret)); + } else if (OB_FAIL(notify_sys_tenant_root_key())) { + LOG_WARN("fail to notify sys tenant root key", KR(ret)); } else if (OB_FAIL(notify_sys_tenant_server_unit_resource())) { LOG_WARN("fail to notify sys tenant server unit resource", KR(ret)); } else if (OB_FAIL(notify_sys_tenant_config_())) { @@ -266,6 +268,12 @@ int ObPreBootstrap::prepare_bootstrap(ObAddr &master_rs) return ret; } +int ObPreBootstrap::notify_sys_tenant_root_key() +{ + int ret = OB_SUCCESS; + return ret; +} + int ObPreBootstrap::notify_sys_tenant_server_unit_resource() { int ret = OB_SUCCESS; diff --git a/src/rootserver/ob_bootstrap.h b/src/rootserver/ob_bootstrap.h index c2d87de6ae..595f5829a9 100644 --- a/src/rootserver/ob_bootstrap.h +++ b/src/rootserver/ob_bootstrap.h @@ -102,6 +102,7 @@ private: virtual int check_is_all_server_empty(bool &is_empty); virtual int check_all_server_bootstrap_mode_match(bool &match); + virtual int notify_sys_tenant_root_key(); virtual int notify_sys_tenant_server_unit_resource(); virtual int create_ls(); virtual int wait_elect_ls(common::ObAddr &master_rs); diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index c4dbc850ea..8a334d9265 100644 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -20721,6 +20721,8 @@ int ObDDLService::create_tenant_schema( LOG_INFO("[CREATE_TENANT] STEP 1.4. finish change pool owners", KR(ret), K(user_tenant_id), "cost", ObTimeUtility::fast_current_time() - tmp_start_time); } + + if (OB_SUCC(ret)) { ObArray addrs; ObZone zone; // empty means get all zone's servers diff --git a/src/rootserver/ob_root_service.cpp b/src/rootserver/ob_root_service.cpp index ab98b567fe..187a0ada30 100644 --- a/src/rootserver/ob_root_service.cpp +++ b/src/rootserver/ob_root_service.cpp @@ -10115,5 +10115,6 @@ int ObRootService::try_add_dep_infos_for_synonym_batch(const obrpc::ObTryAddDepI return ret; } + } // end namespace rootserver } // end namespace oceanbase diff --git a/src/rootserver/restore/ob_restore_scheduler.cpp b/src/rootserver/restore/ob_restore_scheduler.cpp index b25c796fe2..70ef262cff 100644 --- a/src/rootserver/restore/ob_restore_scheduler.cpp +++ b/src/rootserver/restore/ob_restore_scheduler.cpp @@ -13,6 +13,8 @@ #define USING_LOG_PREFIX RS_RESTORE #include "ob_restore_scheduler.h" +#include "rootserver/ob_ddl_service.h" +#include "rootserver/ob_rs_async_rpc_proxy.h" #include "rootserver/ob_rs_event_history_table_operator.h" #include "rootserver/ob_unit_manager.h"//convert_pool_name_lis #include "rootserver/ob_tenant_role_transition_service.h"//failover_to_primary @@ -33,7 +35,6 @@ #include "logservice/palf/log_define.h"//scn #include "share/scn.h" - namespace oceanbase { namespace rootserver @@ -451,8 +452,10 @@ int ObRestoreService::restore_pre(const ObPhysicalRestoreJob &job_info) LOG_WARN("invalid tenant id", K(ret), K(tenant_id_)); } else if (OB_FAIL(check_stop())) { LOG_WARN("restore scheduler stopped", K(ret)); - } else if (OB_FAIL(convert_parameters(job_info))) { - LOG_WARN("fail to convert parameters", K(ret), K(job_info)); + } else if (OB_FAIL(restore_root_key(job_info))) { + LOG_WARN("fail to restore root key", K(ret)); + } else if (OB_FAIL(restore_keystore(job_info))) { + LOG_WARN("fail to restore keystore", K(ret), K(job_info)); } else { if (OB_FAIL(fill_restore_statistics(job_info))) { LOG_WARN("fail to fill restore statistics", K(ret), K(job_info)); @@ -513,41 +516,18 @@ int ObRestoreService::convert_parameters( const ObPhysicalRestoreJob &job_info) { int ret = OB_SUCCESS; - uint64_t tenant_id = gen_meta_tenant_id(tenant_id_); - if (!inited_) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret)); - } else if (OB_INVALID_TENANT_ID == tenant_id - || OB_SYS_TENANT_ID == tenant_id) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid tenant id", K(ret), K(tenant_id)); - } else if (OB_FAIL(check_stop())) { - LOG_WARN("restore scheduler stopped", K(ret)); - } else if (job_info.get_kms_encrypt()) { - int64_t affected_row = 0; - ObSqlString sql; - // set tde_method - if (OB_FAIL(sql.assign_fmt("ALTER SYSTEM SET tde_method = 'bkmi'"))) { - LOG_WARN("failed to assign fmt", K(ret), K(sql)); - } else if (OB_FAIL(sql_proxy_->write(tenant_id, sql.ptr(), affected_row))) { - LOG_WARN("failed to execute", K(ret), K(affected_row), K(sql)); - } - // set external_kms_info - if (OB_SUCC(ret)) { - sql.reset(); - if (OB_FAIL(sql.assign_fmt("ALTER SYSTEM SET external_kms_info= '%s'", job_info.get_kms_info().ptr()))) { - LOG_WARN("failed to assign fmt", K(ret), K(sql)); - } else if (OB_FAIL(sql_proxy_->write(tenant_id, sql.ptr(), affected_row))) { - LOG_WARN("failed to execute", K(ret), K(affected_row), K(sql)); - } - } - } + return ret; +} - if (OB_SUCC(ret)) { - // Broadcast tenant's config version after system tables are restored. - // bugfix: - // TODO check all config is valid on observer - } +int ObRestoreService::restore_root_key(const share::ObPhysicalRestoreJob &job_info) +{ + int ret = OB_SUCCESS; + return ret; +} + +int ObRestoreService::restore_keystore(const share::ObPhysicalRestoreJob &job_info) +{ + int ret = OB_SUCCESS; return ret; } @@ -605,6 +585,10 @@ int ObRestoreService::post_check(const ObPhysicalRestoreJob &job_info) } } + if (FAILEDx(convert_parameters(job_info))) { + LOG_WARN("fail to convert parameters", K(ret), K(job_info)); + } + if (OB_SUCC(ret)) { int tmp_ret = OB_SUCCESS; if (OB_SUCCESS != (tmp_ret = try_update_job_status(ret, job_info))) { diff --git a/src/rootserver/restore/ob_restore_scheduler.h b/src/rootserver/restore/ob_restore_scheduler.h index 46040a8892..013f492f29 100644 --- a/src/rootserver/restore/ob_restore_scheduler.h +++ b/src/rootserver/restore/ob_restore_scheduler.h @@ -106,6 +106,8 @@ private: const ObSqlString &pool_list, obrpc::ObCreateTenantArg &arg); int convert_parameters(const share::ObPhysicalRestoreJob &job_info); + int restore_root_key(const share::ObPhysicalRestoreJob &job_info); + int restore_keystore(const share::ObPhysicalRestoreJob &job_info); int check_locality_valid(const share::schema::ZoneLocalityIArray &locality); int try_update_job_status( diff --git a/src/rootserver/restore/ob_restore_util.cpp b/src/rootserver/restore/ob_restore_util.cpp index 2836b77001..cfa91ee530 100644 --- a/src/rootserver/restore/ob_restore_util.cpp +++ b/src/rootserver/restore/ob_restore_util.cpp @@ -18,6 +18,7 @@ #include "share/schema/ob_schema_mgr.h" #include "share/schema/ob_schema_getter_guard.h" #include "share/backup/ob_backup_struct.h" +#include "share/backup/ob_backup_io_adapter.h" #include "share/backup/ob_backup_path.h" #include "rootserver/ob_rs_event_history_table_operator.h" #include "storage/backup/ob_backup_restore_util.h" @@ -75,6 +76,12 @@ int ObRestoreUtil::fill_physical_restore_job( } } + if (OB_SUCC(ret)) { + if (OB_FAIL(fill_encrypt_info_(arg, job))) { + LOG_WARN("failed to fill encrypt info", KR(ret), K(arg), K(job)); + } + } + if (FAILEDx(job.set_passwd_array(arg.passwd_array_))) { LOG_WARN("failed to copy passwd array", K(ret), K(arg)); } @@ -364,6 +371,14 @@ int ObRestoreUtil::fill_restore_scn_(const obrpc::ObPhysicalRestoreTenantArg &ar return ret; } +int ObRestoreUtil::fill_encrypt_info_( + const obrpc::ObPhysicalRestoreTenantArg &arg, + share::ObPhysicalRestoreJob &job) +{ + int ret = OB_SUCCESS; + return ret; +} + int ObRestoreUtil::get_restore_source( const ObIArray& tenant_path_array, const common::ObString &passwd_array, diff --git a/src/rootserver/restore/ob_restore_util.h b/src/rootserver/restore/ob_restore_util.h index 888f394b82..a14012db0f 100644 --- a/src/rootserver/restore/ob_restore_util.h +++ b/src/rootserver/restore/ob_restore_util.h @@ -117,6 +117,9 @@ private: const obrpc::ObPhysicalRestoreTenantArg &arg, const ObIArray &tenant_path_array, share::ObPhysicalRestoreJob &job); + static int fill_encrypt_info_( + const obrpc::ObPhysicalRestoreTenantArg &arg, + share::ObPhysicalRestoreJob &job); DISALLOW_COPY_AND_ASSIGN(ObRestoreUtil); }; diff --git a/src/share/backup/ob_backup_data_store.cpp b/src/share/backup/ob_backup_data_store.cpp index 6bdd7c2ef6..d996e9cbd9 100644 --- a/src/share/backup/ob_backup_data_store.cpp +++ b/src/share/backup/ob_backup_data_store.cpp @@ -731,6 +731,18 @@ int ObBackupDataStore::read_backup_set_info(ObExternBackupSetInfoDesc &backup_se return ret; } +int ObBackupDataStore::write_root_key_info(const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + return ret; +} + +int ObBackupDataStore::read_root_key_info(const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + return ret; +} + int ObBackupDataStore::get_backup_set_placeholder_path_( const bool is_inner, const bool is_start, diff --git a/src/share/backup/ob_backup_data_store.h b/src/share/backup/ob_backup_data_store.h index bbc42adc6d..69b2921edf 100644 --- a/src/share/backup/ob_backup_data_store.h +++ b/src/share/backup/ob_backup_data_store.h @@ -318,6 +318,8 @@ public: int get_backup_set_array(const common::ObString &passwd_array, const SCN &restore_scn, SCN &restore_start_scn, common::ObIArray &backup_set_list); int get_max_sys_ls_retry_id(const share::ObBackupPath &backup_path, const ObLSID &ls_id, int64_t &retry_id); + int write_root_key_info(const uint64_t tenant_id); + int read_root_key_info(const uint64_t tenant_id); TO_STRING_KV(K_(backup_desc)); diff --git a/src/share/backup/ob_backup_path.cpp b/src/share/backup/ob_backup_path.cpp index 0124146b67..f563156ac1 100644 --- a/src/share/backup/ob_backup_path.cpp +++ b/src/share/backup/ob_backup_path.cpp @@ -892,6 +892,18 @@ int ObBackupPathUtil::get_ls_meta_infos_path(const share::ObBackupDest &backup_s return ret; } +// file:///obbackup/backup_set_1_full/infos/meta_info/root_key.obbak +int ObBackupPathUtil::get_backup_root_key_path(const share::ObBackupDest &backup_set_dest, ObBackupPath &backup_path) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(get_tenant_meta_info_dir_path(backup_set_dest, backup_path))) { + LOG_WARN("failed to get backup set dir path", K(ret), K(backup_set_dest)); + } else if (OB_FAIL(backup_path.join(OB_STR_ROOT_KEY, ObBackupFileSuffix::BACKUP))) { + LOG_WARN("failed to join ls meta infos", K(ret)); + } + return ret; +} + // file:///obbackup/backup_set_1_full/tenant_backup_set_infos.obbak int ObBackupPathUtil::get_tenant_backup_set_infos_path(const share::ObBackupDest &backup_set_dest, ObBackupPath &backup_path) diff --git a/src/share/backup/ob_backup_path.h b/src/share/backup/ob_backup_path.h index 27179b95d9..e61c36157f 100644 --- a/src/share/backup/ob_backup_path.h +++ b/src/share/backup/ob_backup_path.h @@ -185,6 +185,10 @@ struct ObBackupPathUtil // file:///obbackup/backup_set_1_full/infos/meta_info/ls_meta_infos.obbak static int get_ls_meta_infos_path(const share::ObBackupDest &backup_set_dest, share::ObBackupPath &backup_path); + // file:///obbackup/backup_set_1_full/infos/meta_info/root_key.obbak + static int get_backup_root_key_path(const share::ObBackupDest &backup_set_dest, + share::ObBackupPath &backup_path); + // file:///obbackup/backup_set_1_full/tenant_backup_set_infos.obbak static int get_tenant_backup_set_infos_path(const share::ObBackupDest &backup_set_dest, share::ObBackupPath &backup_path); diff --git a/src/share/backup/ob_backup_store.cpp b/src/share/backup/ob_backup_store.cpp index 5b1584fce4..e39c09320f 100644 --- a/src/share/backup/ob_backup_store.cpp +++ b/src/share/backup/ob_backup_store.cpp @@ -25,7 +25,8 @@ using namespace share; static const char *type_strs[] = { "backup_data", - "archive_log" + "archive_log", + "backup_key" }; const char *ObBackupDestType::get_str(const TYPE &type) diff --git a/src/share/backup/ob_backup_store.h b/src/share/backup/ob_backup_store.h index b21783acdc..67633cef1c 100644 --- a/src/share/backup/ob_backup_store.h +++ b/src/share/backup/ob_backup_store.h @@ -28,6 +28,7 @@ struct ObBackupDestType final { DEST_TYPE_BACKUP_DATA = 0, DEST_TYPE_ARCHIVE_LOG, + DEST_TYPE_BACKUP_KEY, DEST_TYPE_MAX }; static const char *get_str(const TYPE &type); diff --git a/src/share/backup/ob_backup_struct.h b/src/share/backup/ob_backup_struct.h index 31e318f4ad..2712caf5e7 100644 --- a/src/share/backup/ob_backup_struct.h +++ b/src/share/backup/ob_backup_struct.h @@ -54,7 +54,7 @@ const int64_t OB_MAX_BACKUP_STORAGE_INFO_LENGTH = 1536; const int64_t OB_MAX_BACKUP_ENDPOINT_LENGTH = 256; const int64_t OB_MAX_BACKUP_ACCESSID_LENGTH = 256; const int64_t OB_MAX_BACKUP_ACCESSKEY_LENGTH = 256; -const int64_t OB_MAX_BACKUP_ENCRYPTKEY_LENGTH = OB_MAX_BACKUP_ACCESSKEY_LENGTH + 16; +const int64_t OB_MAX_BACKUP_ENCRYPTKEY_LENGTH = OB_MAX_BACKUP_ACCESSKEY_LENGTH + 32; const int64_t OB_MAX_BACKUP_SERIALIZEKEY_LENGTH = OB_MAX_BACKUP_ENCRYPTKEY_LENGTH * 2; const int64_t OB_MAX_BACKUP_AUTHORIZATION_LENGTH = 1024; const int64_t OB_MAX_BACKUP_EXTENSION_LENGTH = 512; @@ -383,6 +383,7 @@ const char *const OB_STR_BACKUP_SET_LIST = "backup_set_list"; const char *const OB_STR_BACKUP_PIECE_LIST = "backup_piece_list"; const char *const OB_STR_LOG_PATH_LIST = "log_path_list"; const char *const OB_STR_LS_META_INFOS = "ls_meta_infos"; +const char *const OB_STR_ROOT_KEY = "root_key"; const char *const OB_STR_BACKUP_DATA_VERSION = "backup_data_version"; const char *const OB_STR_CLUSTER_VERSION = "cluster_version"; const char *const OB_BACKUP_SUFFIX=".obbak"; diff --git a/src/share/ob_encryption_util.h b/src/share/ob_encryption_util.h index e68fb835e7..d8785bff6e 100644 --- a/src/share/ob_encryption_util.h +++ b/src/share/ob_encryption_util.h @@ -94,6 +94,7 @@ public: static const int OB_AES_BLOCK_SIZE = 16; }; +const int64_t OB_ROOT_KEY_LEN = 16; const int64_t OB_ORIGINAL_TABLE_KEY_LEN = 15; const int64_t OB_ENCRYPTED_TABLE_KEY_LEN = 16; const int64_t OB_MAX_MASTER_KEY_LENGTH = 16; diff --git a/src/share/ob_rpc_struct.cpp b/src/share/ob_rpc_struct.cpp index 33cef0230a..eb3383cb37 100644 --- a/src/share/ob_rpc_struct.cpp +++ b/src/share/ob_rpc_struct.cpp @@ -4251,7 +4251,10 @@ OB_SERIALIZE_MEMBER((ObPhysicalRestoreTenantArg, ObCmdArg), table_items_, multi_uri_, description_, - with_restore_scn_); + with_restore_scn_, + encrypt_key_, + kms_uri_, + kms_encrypt_key_); ObPhysicalRestoreTenantArg::ObPhysicalRestoreTenantArg() : ObCmdArg(), @@ -4264,7 +4267,10 @@ ObPhysicalRestoreTenantArg::ObPhysicalRestoreTenantArg() table_items_(), multi_uri_(), description_(), - with_restore_scn_(false) + with_restore_scn_(false), + encrypt_key_(), + kms_uri_(), + kms_encrypt_key_() { } @@ -4284,6 +4290,9 @@ int ObPhysicalRestoreTenantArg::assign(const ObPhysicalRestoreTenantArg &other) passwd_array_ = other.passwd_array_; description_ = other.description_; with_restore_scn_ = other.with_restore_scn_; + encrypt_key_ = other.encrypt_key_; + kms_uri_ = other.kms_uri_; + kms_encrypt_key_ = other.kms_encrypt_key_; } return ret; } diff --git a/src/share/ob_rpc_struct.h b/src/share/ob_rpc_struct.h index 10a8dbbb30..b27ad2281b 100644 --- a/src/share/ob_rpc_struct.h +++ b/src/share/ob_rpc_struct.h @@ -4811,7 +4811,10 @@ public: K_(kms_info), K_(table_items), K_(multi_uri), - K_(with_restore_scn)); + K_(with_restore_scn), + K_(encrypt_key), + K_(kms_uri), + K_(kms_encrypt_key)); common::ObString tenant_name_; common::ObString uri_; @@ -4823,6 +4826,9 @@ public: common::ObString multi_uri_; // 备份拆分用 common::ObString description_; bool with_restore_scn_; + common::ObString encrypt_key_; + common::ObString kms_uri_; + common::ObString kms_encrypt_key_; }; struct ObServerZoneArg diff --git a/src/share/restore/ob_physical_restore_info.cpp b/src/share/restore/ob_physical_restore_info.cpp index f86d3b71e5..ecc69687cc 100644 --- a/src/share/restore/ob_physical_restore_info.cpp +++ b/src/share/restore/ob_physical_restore_info.cpp @@ -305,6 +305,12 @@ int ObPhysicalRestoreJob::assign(const ObPhysicalRestoreJob &other) LOG_WARN("failed to copy string", KR(ret), K(other)); } else if (OB_FAIL(deep_copy_ob_string(allocator_, other.kms_info_, kms_info_))) { LOG_WARN("failed to copy string", KR(ret), K(other)); + } else if (OB_FAIL(deep_copy_ob_string(allocator_, other.encrypt_key_, encrypt_key_))) { + LOG_WARN("failed to copy string", KR(ret), K(other)); + } else if (OB_FAIL(deep_copy_ob_string(allocator_, other.kms_dest_, kms_dest_))) { + LOG_WARN("failed to copy string", KR(ret), K(other)); + } else if (OB_FAIL(deep_copy_ob_string(allocator_, other.kms_encrypt_key_, kms_encrypt_key_))) { + LOG_WARN("failed to copy string", KR(ret), K(other)); } else if (OB_FAIL(deep_copy_ob_string(allocator_, other.passwd_array_, passwd_array_))) { LOG_WARN("failed to copy string", KR(ret), K(other)); } else if (OB_FAIL(deep_copy_ob_string(allocator_, other.backup_tenant_name_, backup_tenant_name_))) { @@ -349,6 +355,9 @@ void ObPhysicalRestoreJob::reset() compatible_ = 0; kms_info_.reset(); kms_encrypt_ = false; + encrypt_key_.reset(); + kms_dest_.reset(); + kms_encrypt_key_.reset(); concurrency_ = 0; diff --git a/src/share/restore/ob_physical_restore_info.h b/src/share/restore/ob_physical_restore_info.h index d4235baf9e..713c002098 100644 --- a/src/share/restore/ob_physical_restore_info.h +++ b/src/share/restore/ob_physical_restore_info.h @@ -179,6 +179,9 @@ public: //for kms Property_declare_ObString(kms_info) Property_declare_int(bool, kms_encrypt) + Property_declare_ObString(encrypt_key) + Property_declare_ObString(kms_dest) + Property_declare_ObString(kms_encrypt_key) Property_declare_ObString(passwd_array) Property_declare_int(int64_t, concurrency) diff --git a/src/share/restore/ob_physical_restore_table_operator.cpp b/src/share/restore/ob_physical_restore_table_operator.cpp index 5839428158..60d8323c56 100644 --- a/src/share/restore/ob_physical_restore_table_operator.cpp +++ b/src/share/restore/ob_physical_restore_table_operator.cpp @@ -256,6 +256,9 @@ int ObPhysicalRestoreTableOperator::fill_dml_splicer( ADD_COLUMN_MACRO_IN_TABLE_OPERATOR(job_info, locality); ADD_COLUMN_MACRO_IN_TABLE_OPERATOR(job_info, kms_encrypt); ADD_COLUMN_MACRO_IN_TABLE_OPERATOR(job_info, kms_info); + ADD_COLUMN_MACRO_IN_TABLE_OPERATOR(job_info, encrypt_key); + ADD_COLUMN_MACRO_IN_TABLE_OPERATOR(job_info, kms_dest); + ADD_COLUMN_MACRO_IN_TABLE_OPERATOR(job_info, kms_encrypt_key); ADD_COLUMN_MACRO_IN_TABLE_OPERATOR(job_info, compat_mode); ADD_COLUMN_MACRO_IN_TABLE_OPERATOR(job_info, compatible); ADD_COLUMN_MACRO_IN_TABLE_OPERATOR(job_info, passwd_array); @@ -496,6 +499,9 @@ int ObPhysicalRestoreTableOperator::retrieve_restore_option( RETRIEVE_STR_VALUE(passwd_array, job); RETRIEVE_INT_VALUE(compatible, job); RETRIEVE_STR_VALUE(kms_info, job); + RETRIEVE_STR_VALUE(encrypt_key, job); + RETRIEVE_STR_VALUE(kms_dest, job); + RETRIEVE_STR_VALUE(kms_encrypt_key, job); RETRIEVE_INT_VALUE(concurrency, job); if (OB_SUCC(ret)) { diff --git a/src/sql/engine/cmd/ob_alter_system_executor.cpp b/src/sql/engine/cmd/ob_alter_system_executor.cpp index ac6f05ab06..5380fd9dca 100644 --- a/src/sql/engine/cmd/ob_alter_system_executor.cpp +++ b/src/sql/engine/cmd/ob_alter_system_executor.cpp @@ -2105,6 +2105,12 @@ int ObDeletePolicyExecutor::execute(ObExecContext &ctx, ObDeletePolicyStmt &stmt return ret; } +int ObBackupKeyExecutor::execute(ObExecContext &ctx, ObBackupKeyStmt &stmt) +{ + int ret = OB_SUCCESS; + return ret; +} + int ObBackupBackupsetExecutor::execute(ObExecContext &ctx, ObBackupBackupsetStmt &stmt) { int ret = OB_SUCCESS; diff --git a/src/sql/engine/cmd/ob_alter_system_executor.h b/src/sql/engine/cmd/ob_alter_system_executor.h index 24cbf9f2e2..2ba4f7f6d9 100644 --- a/src/sql/engine/cmd/ob_alter_system_executor.h +++ b/src/sql/engine/cmd/ob_alter_system_executor.h @@ -122,6 +122,7 @@ DEF_SIMPLE_EXECUTOR(ObBackupDatabase); DEF_SIMPLE_EXECUTOR(ObBackupManage); DEF_SIMPLE_EXECUTOR(ObBackupClean); DEF_SIMPLE_EXECUTOR(ObDeletePolicy); +DEF_SIMPLE_EXECUTOR(ObBackupKey); DEF_SIMPLE_EXECUTOR(ObBackupBackupset); DEF_SIMPLE_EXECUTOR(ObBackupArchiveLog); DEF_SIMPLE_EXECUTOR(ObBackupBackupPiece); diff --git a/src/sql/executor/ob_cmd_executor.cpp b/src/sql/executor/ob_cmd_executor.cpp index 9edb5507ae..37c0e464a5 100644 --- a/src/sql/executor/ob_cmd_executor.cpp +++ b/src/sql/executor/ob_cmd_executor.cpp @@ -850,6 +850,10 @@ int ObCmdExecutor::execute(ObExecContext &ctx, ObICmd &cmd) DEFINE_EXECUTE_CMD(ObDeletePolicyStmt, ObDeletePolicyExecutor); break; } + case stmt::T_BACKUP_KEY: { + DEFINE_EXECUTE_CMD(ObBackupKeyStmt, ObBackupKeyExecutor); + break; + } case stmt::T_CREATE_DBLINK: { DEFINE_EXECUTE_CMD(ObCreateDbLinkStmt, ObCreateDbLinkExecutor); break; diff --git a/src/sql/parser/non_reserved_keywords_mysql_mode.c b/src/sql/parser/non_reserved_keywords_mysql_mode.c index 483884b24d..e8754aa4a4 100644 --- a/src/sql/parser/non_reserved_keywords_mysql_mode.c +++ b/src/sql/parser/non_reserved_keywords_mysql_mode.c @@ -236,6 +236,7 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] = {"enable_arbitration_service", ENABLE_ARBITRATION_SERVICE}, {"enable_extended_rowid", ENABLE_EXTENDED_ROWID}, {"enclosed", ENCLOSED}, + {"encrypted", ENCRYPTED}, {"encryption", ENCRYPTION}, {"end", END}, {"ends", ENDS}, diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index 96c15bd7e3..e2f5aadf01 100755 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -266,7 +266,7 @@ END_P SET_VAR DELIMITER DIRECTORY DISABLE DISCARD DISK DISKGROUP DO DUMP DUMPFILE DUPLICATE DUPLICATE_SCOPE DYNAMIC DATABASE_ID DEFAULT_TABLEGROUP DISCONNECT - EFFECTIVE EMPTY ENABLE ENABLE_ARBITRATION_SERVICE ENABLE_EXTENDED_ROWID ENCRYPTION END ENDS ENFORCED ENGINE_ ENGINES ENUM ENTITY ERROR_CODE ERROR_P ERRORS ESTIMATE + EFFECTIVE EMPTY ENABLE ENABLE_ARBITRATION_SERVICE ENABLE_EXTENDED_ROWID ENCRYPTED ENCRYPTION END ENDS ENFORCED ENGINE_ ENGINES ENUM ENTITY ERROR_CODE ERROR_P ERRORS ESTIMATE ESCAPE EVENT EVENTS EVERY EXCHANGE EXECUTE EXPANSION EXPIRE EXPIRE_INFO EXPORT OUTLINE EXTENDED EXTENDED_NOADDR EXTENT_SIZE EXTRACT EXCEPT EXPIRED @@ -489,7 +489,7 @@ END_P SET_VAR DELIMITER %type alter_tablespace_stmt %type permanent_tablespace permanent_tablespace_options permanent_tablespace_option alter_tablespace_actions alter_tablespace_action opt_force_purge %type opt_sql_throttle_for_priority opt_sql_throttle_using_cond sql_throttle_one_or_more_metrics sql_throttle_metric -%type opt_copy_id opt_backup_dest opt_preview opt_backup_backup_dest opt_tenant_info opt_with_active_piece get_format_unit opt_backup_tenant_list opt_backup_to opt_description policy_name opt_recovery_window opt_redundancy opt_backup_copies opt_restore_until +%type opt_copy_id opt_backup_dest opt_preview opt_backup_backup_dest opt_tenant_info opt_with_active_piece get_format_unit opt_backup_tenant_list opt_backup_to opt_description policy_name opt_recovery_window opt_redundancy opt_backup_copies opt_restore_until opt_backup_key_info opt_encrypt_key %type new_or_old new_or_old_column_ref diagnostics_info_ref %type on_empty on_error json_on_response opt_returning_type opt_on_empty_or_error json_value_expr opt_ascii %type ws_nweights opt_ws_as_char opt_ws_levels ws_level_flag_desc ws_level_flag_reverse ws_level_flags ws_level_list ws_level_list_item ws_level_number ws_level_range ws_level_list_or_range @@ -14435,16 +14435,16 @@ ALTER SYSTEM CLEAR RESTORE SOURCE malloc_terminal_node($$, result->malloc_pool_, T_CLEAR_RESTORE_SOURCE); } | -ALTER SYSTEM RESTORE table_list FOR relation_name opt_backup_dest opt_restore_until WITH STRING_VALUE opt_description +ALTER SYSTEM RESTORE table_list FOR relation_name opt_backup_dest opt_restore_until WITH STRING_VALUE opt_encrypt_key opt_backup_key_info opt_description { ParseNode *tables = NULL; merge_nodes(tables, result, T_TABLE_LIST, $4); - malloc_non_terminal_node($$, result->malloc_pool_, T_PHYSICAL_RESTORE_TENANT, 6, $6, $7, $8, $10, $11, tables); + malloc_non_terminal_node($$, result->malloc_pool_, T_PHYSICAL_RESTORE_TENANT, 8, $6, $7, $8, $10, $11, $12, $13, tables); } | -ALTER SYSTEM RESTORE relation_name opt_backup_dest opt_restore_until WITH STRING_VALUE opt_description opt_preview +ALTER SYSTEM RESTORE relation_name opt_backup_dest opt_restore_until WITH STRING_VALUE opt_encrypt_key opt_backup_key_info opt_description opt_preview { - malloc_non_terminal_node($$, result->malloc_pool_, T_PHYSICAL_RESTORE_TENANT, 6, $4, $5, $6, $8, $9, $10); + malloc_non_terminal_node($$, result->malloc_pool_, T_PHYSICAL_RESTORE_TENANT, 8, $4, $5, $6, $8, $9, $10, $11, $12); } | ALTER SYSTEM CHANGE TENANT change_tenant_name_or_tenant_id @@ -14601,6 +14601,22 @@ ALTER SYSTEM BACKUP INCREMENTAL opt_backup_tenant_list opt_backup_to PLUS ARCHIV malloc_non_terminal_node($$, result->malloc_pool_, T_BACKUP_DATABASE, 6, tenant, compl_log, incremental, $5, $6, $9); } | +ALTER SYSTEM BACKUP KEY opt_backup_to opt_encrypt_key +{ + ParseNode *tenant = NULL; + malloc_terminal_node(tenant, result->malloc_pool_, T_INT); + tenant->value_ = 0; + malloc_non_terminal_node($$, result->malloc_pool_, T_BACKUP_KEY, 3, tenant, $5, $6); +} +| +ALTER SYSTEM BACKUP KEY tenant_list_tuple opt_backup_to opt_encrypt_key +{ + ParseNode *tenant = NULL; + malloc_terminal_node(tenant, result->malloc_pool_, T_INT); + tenant->value_ = 1; + malloc_non_terminal_node($$, result->malloc_pool_, T_BACKUP_KEY, 4, tenant, $5, $6, $7); +} +| ALTER SYSTEM CANCEL BACKUP opt_backup_tenant_list { ParseNode *type = NULL; @@ -16242,6 +16258,21 @@ opt_restore_until: } ; +opt_backup_key_info: +/*EMPTY*/ { $$ = NULL; } +| WITH KEY FROM STRING_VALUE opt_encrypt_key +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_BACKUP_KEY, 2, $4, $5); +} +; + +opt_encrypt_key: +/*EMPTY*/ { $$ = NULL; } +| ENCRYPTED BY STRING_VALUE +{ + $$ = $3; +} +; /*=========================================================== * savepoint @@ -16992,6 +17023,7 @@ ACCOUNT | ENABLE | ENABLE_ARBITRATION_SERVICE | ENABLE_EXTENDED_ROWID +| ENCRYPTED | ENCRYPTION | END | ENDS diff --git a/src/sql/privilege_check/ob_privilege_check.cpp b/src/sql/privilege_check/ob_privilege_check.cpp index 373ceb5cf6..2ba3c912e3 100644 --- a/src/sql/privilege_check/ob_privilege_check.cpp +++ b/src/sql/privilege_check/ob_privilege_check.cpp @@ -1711,6 +1711,7 @@ int get_sys_tenant_alter_system_priv( stmt::T_BACKUP_MANAGE != basic_stmt->get_stmt_type() && stmt::T_BACKUP_CLEAN != basic_stmt->get_stmt_type() && stmt::T_DELETE_POLICY != basic_stmt->get_stmt_type() && + stmt::T_BACKUP_KEY != basic_stmt->get_stmt_type() && stmt::T_RECOVER != basic_stmt->get_stmt_type()) { ret = OB_ERR_NO_PRIVILEGE; LOG_WARN("Only sys tenant can do this operation", diff --git a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp index 3ed6adb4d3..b556d95947 100644 --- a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp +++ b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp @@ -2470,7 +2470,7 @@ int ObPhysicalRestoreTenantResolver::resolve(const ParseNode &parse_tree) } else if (OB_UNLIKELY(NULL == parse_tree.children_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("children should not be null"); - } else if (OB_UNLIKELY(6 != parse_tree.num_child_)) { + } else if (OB_UNLIKELY(8 != parse_tree.num_child_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("num of children not match", K(ret), "child_num", parse_tree.num_child_); } else { @@ -2495,7 +2495,31 @@ int ObPhysicalRestoreTenantResolver::resolve(const ParseNode &parse_tree) } } } - ParseNode *description_node = parse_tree.children_[4]; + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_NOT_NULL(parse_tree.children_[4]) + && OB_FAIL(Util::resolve_string(parse_tree.children_[4], + stmt->get_rpc_arg().encrypt_key_))) { + LOG_WARN("failed to resolve encrypt key", K(ret)); + } else if (OB_NOT_NULL(parse_tree.children_[5])) { + ParseNode *kms_node = parse_tree.children_[5]; + if (2 != kms_node->num_child_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("num of children not match", K(ret), "child_num", kms_node->num_child_); + } else if (OB_ISNULL(kms_node->children_[0])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("kms uri should not be NULL", K(ret)); + } else if (OB_FAIL(Util::resolve_string(kms_node->children_[0], + stmt->get_rpc_arg().kms_uri_))) { + LOG_WARN("failed to resolve kms uri", K(ret)); + } else if (OB_NOT_NULL(kms_node->children_[1]) + && OB_FAIL(Util::resolve_string(kms_node->children_[1], + stmt->get_rpc_arg().kms_encrypt_key_))) { + LOG_WARN("failed to resolve kms encrypt key", K(ret)); + } + } + + ParseNode *description_node = parse_tree.children_[6]; if (OB_FAIL(ret)) { // do nothing } else if (OB_FAIL(Util::resolve_string(parse_tree.children_[3], @@ -2520,8 +2544,8 @@ int ObPhysicalRestoreTenantResolver::resolve(const ParseNode &parse_tree) } if (OB_SUCC(ret)) { - if (6 == parse_tree.num_child_) { // resolve table_list - const ParseNode *node = parse_tree.children_[5]; + if (8 == parse_tree.num_child_) { // resolve table_list + const ParseNode *node = parse_tree.children_[7]; if (OB_ISNULL(node)) { stmt->set_is_preview(false); } else { @@ -4046,6 +4070,12 @@ int ObDeletePolicyResolver::resolve(const ParseNode &parse_tree) return ret; } +int ObBackupKeyResolver::resolve(const ParseNode &parse_tree) +{ + int ret = OB_ERR_PARSE_SQL; + return ret; +} + int ObBackupArchiveLogResolver::resolve(const ParseNode &parse_tree) { int ret = OB_SUCCESS; diff --git a/src/sql/resolver/cmd/ob_alter_system_resolver.h b/src/sql/resolver/cmd/ob_alter_system_resolver.h index 3d5fe56f0b..c66654fd35 100644 --- a/src/sql/resolver/cmd/ob_alter_system_resolver.h +++ b/src/sql/resolver/cmd/ob_alter_system_resolver.h @@ -228,6 +228,7 @@ DEF_SIMPLE_CMD_RESOLVER(ObBackupDatabaseResolver); DEF_SIMPLE_CMD_RESOLVER(ObBackupManageResolver); DEF_SIMPLE_CMD_RESOLVER(ObBackupCleanResolver); DEF_SIMPLE_CMD_RESOLVER(ObDeletePolicyResolver); +DEF_SIMPLE_CMD_RESOLVER(ObBackupKeyResolver); DEF_SIMPLE_CMD_RESOLVER(ObEnableSqlThrottleResolver); DEF_SIMPLE_CMD_RESOLVER(ObDisableSqlThrottleResolver); DEF_SIMPLE_CMD_RESOLVER(ObSetRegionBandwidthResolver); diff --git a/src/sql/resolver/cmd/ob_alter_system_stmt.h b/src/sql/resolver/cmd/ob_alter_system_stmt.h index 1e010f2899..2da9e1903a 100644 --- a/src/sql/resolver/cmd/ob_alter_system_stmt.h +++ b/src/sql/resolver/cmd/ob_alter_system_stmt.h @@ -1093,6 +1093,45 @@ private: DISALLOW_COPY_AND_ASSIGN(ObDeletePolicyStmt); }; +class ObBackupKeyStmt : public ObSystemCmdStmt +{ +public: + ObBackupKeyStmt() + : ObSystemCmdStmt(stmt::T_BACKUP_KEY), + tenant_id_(OB_INVALID_TENANT_ID), + backup_dest_() + { + } + virtual ~ObBackupKeyStmt() {} + + uint64 get_tenant_id() const { return tenant_id_; } + const share::ObBackupPathString &get_backup_dest() const { return backup_dest_; } + const ObString &get_encrypt_key() const { return encrypt_key_; } + int set_param(const uint64_t tenant_id, + const share::ObBackupPathString &backup_dest, + const ObString &encrypt_key) + { + int ret = common::OB_SUCCESS; + if (OB_INVALID_ID == tenant_id) { + ret = OB_INVALID_ARGUMENT; + COMMON_LOG(WARN, "invalid args", K(tenant_id)); + } else if (OB_FAIL(backup_dest_.assign(backup_dest))) { + COMMON_LOG(WARN, "set backup dest failed", K(backup_dest)); + } else { + tenant_id_ = tenant_id; + encrypt_key_ = encrypt_key; + } + return ret; + } + + TO_STRING_KV(N_STMT_TYPE, ((int)stmt_type_), K_(tenant_id), K_(backup_dest)); + +private: + uint64_t tenant_id_; + share::ObBackupPathString backup_dest_; + ObString encrypt_key_; +}; + class ObBackupSetEncryptionStmt : public ObSystemCmdStmt { public: diff --git a/src/sql/resolver/ob_resolver.cpp b/src/sql/resolver/ob_resolver.cpp index 22b36e769e..68e365a107 100644 --- a/src/sql/resolver/ob_resolver.cpp +++ b/src/sql/resolver/ob_resolver.cpp @@ -986,6 +986,10 @@ int ObResolver::resolve(IsPrepared if_prepared, const ParseNode &parse_tree, ObS REGISTER_STMT_RESOLVER(BackupDatabase); break; } + case T_BACKUP_KEY: { + REGISTER_STMT_RESOLVER(BackupKey); + break; + } case T_BACKUP_MANAGE: { REGISTER_STMT_RESOLVER(BackupManage); break; diff --git a/src/sql/resolver/ob_stmt_type.h b/src/sql/resolver/ob_stmt_type.h index 5e6deb8e5d..5d99a211ba 100644 --- a/src/sql/resolver/ob_stmt_type.h +++ b/src/sql/resolver/ob_stmt_type.h @@ -280,6 +280,7 @@ OB_STMT_TYPE_DEF_UNKNOWN_AT(T_ADD_ARBITRATION_SERVICE, get_sys_tenant_alter_syst OB_STMT_TYPE_DEF_UNKNOWN_AT(T_REMOVE_ARBITRATION_SERVICE, get_sys_tenant_alter_system_priv, 281) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_REPLACE_ARBITRATION_SERVICE, get_sys_tenant_alter_system_priv, 282) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_SHOW_SEQUENCES, err_stmt_type_priv, 283) +OB_STMT_TYPE_DEF_UNKNOWN_AT(T_BACKUP_KEY, get_sys_tenant_alter_system_priv, 284) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_MAX, err_stmt_type_priv, 500) #endif diff --git a/unittest/share/backup/test_backup_struct.cpp b/unittest/share/backup/test_backup_struct.cpp index 6cc3b8ec9a..4570512c20 100644 --- a/unittest/share/backup/test_backup_struct.cpp +++ b/unittest/share/backup/test_backup_struct.cpp @@ -17,6 +17,7 @@ #define private public #include "share/backup/ob_backup_path.h" #include "share/backup/ob_log_archive_backup_info_mgr.h" +#include "share/ob_master_key_getter.h" using namespace oceanbase; using namespace common; using namespace share; @@ -87,6 +88,9 @@ TEST(ObBackupDest, oss) ASSERT_EQ(0, strcmp(dest.root_path_, "oss://backup_dir")); ASSERT_TRUE(dest.storage_info_->device_type_ == 0); + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().init(NULL)); + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().set_root_key(OB_SYS_TENANT_ID, + obrpc::RootKeyType::DEFAULT, ObString())); ASSERT_EQ(OB_SUCCESS, dest.get_backup_dest_str(backup_dest_str, sizeof(backup_dest_str))); ASSERT_EQ(0, strcmp(backup_dest_str, "oss://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F&delete_mode=tagging")); ASSERT_EQ(OB_SUCCESS, dest.get_backup_path_str(backup_path_str, sizeof(backup_path_str))); @@ -99,12 +103,16 @@ TEST(ObBackupDest, oss) dest1.reset(); ASSERT_EQ(OB_SUCCESS, dest1.set(dest.get_root_path().ptr(), dest.get_storage_info())); ASSERT_TRUE(dest == dest1); + ObMasterKeyGetter::instance().destroy(); } TEST(ObBackupDest, oss_encrypt) { const char *backup_test = "oss://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F"; ObBackupDest dest; + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().init(NULL)); + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().set_root_key(OB_SYS_TENANT_ID, + obrpc::RootKeyType::DEFAULT, ObString())); ASSERT_EQ(OB_SUCCESS, dest.set(backup_test)); LOG_INFO("dump backup dest", K(dest.get_root_path()), K(*(dest.get_storage_info()))); ASSERT_EQ(0, strcmp(dest.root_path_, "oss://backup_dir")); @@ -127,6 +135,7 @@ TEST(ObBackupDest, oss_encrypt) dest1.reset(); ASSERT_EQ(OB_SUCCESS, dest1.set(path, endpoint, authorization, extension)); ASSERT_TRUE(dest == dest1); + ObMasterKeyGetter::instance().destroy(); } int main(int argc, char **argv) diff --git a/unittest/share/test_encryption_util.cpp b/unittest/share/test_encryption_util.cpp index 8e088edc73..2af5a257fb 100644 --- a/unittest/share/test_encryption_util.cpp +++ b/unittest/share/test_encryption_util.cpp @@ -15,6 +15,7 @@ #include #define private public #include "share/ob_encryption_util.h" +#include "share/ob_master_key_getter.h" #undef private namespace oceanbase @@ -175,18 +176,24 @@ TEST(TestEncryptionUtil, encrypt_master_key) int64_t encrypt_len = 0; char out_buf[buf_len] = {0}; int64_t out_len = 0; + uint64_t tenant_id = 123; + system("rm -rf wallet"); + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().init(NULL)); + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().set_root_key(tenant_id, + obrpc::RootKeyType::DEFAULT, ObString())); EXPECT_EQ(OB_SUCCESS, ObKeyGenerator::generate_encrypt_key(data, data_len)); - EXPECT_EQ(OB_SUCCESS, ObEncryptionUtil::encrypt_master_key(data, data_len, + EXPECT_EQ(OB_SUCCESS, ObEncryptionUtil::encrypt_master_key(tenant_id, data, data_len, encrypt_buf, buf_len, encrypt_len)); EXPECT_LE(encrypt_len, OB_MAX_ENCRYPTED_KEY_LENGTH); encrypt_buf[encrypt_len] = '\0'; EXPECT_STRNE(data, encrypt_buf); - EXPECT_EQ(OB_SUCCESS, ObEncryptionUtil::decrypt_master_key(encrypt_buf, encrypt_len, + EXPECT_EQ(OB_SUCCESS, ObEncryptionUtil::decrypt_master_key(tenant_id, encrypt_buf, encrypt_len, out_buf, buf_len, out_len)); EXPECT_EQ(data_len, out_len); out_buf[out_len] = '\0'; EXPECT_STREQ(data, out_buf); + ObMasterKeyGetter::instance().destroy(); } //TEST(TestWebService, store) diff --git a/unittest/share/test_master_key_getter.cpp b/unittest/share/test_master_key_getter.cpp index b7889152b0..1a112f5635 100644 --- a/unittest/share/test_master_key_getter.cpp +++ b/unittest/share/test_master_key_getter.cpp @@ -32,6 +32,7 @@ public: void TestMasterKeyGetter::SetUp() { + system("rm -rf wallet"); int ret = ObMasterKeyGetter::instance().init(NULL); ASSERT_EQ(OB_SUCCESS, ret); } @@ -128,6 +129,9 @@ TEST_F(TestMasterKeyGetter, key_getter_serialize) char data[OB_MAX_MASTER_KEY_LENGTH] = {0}; int64_t data_len = 0; int tenant_num = sizeof(orig_key_list) / sizeof(orig_key_list[0]); + ObString root_key("123456"); + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().set_root_key(OB_SYS_TENANT_ID, + obrpc::RootKeyType::NORMAL, root_key, false)); for (int i = 0; i < tenant_num; ++i) { for (int j = 0; j < key_num; ++j) { cur_key = orig_key_list[i][j]; @@ -169,6 +173,8 @@ TEST_F(TestMasterKeyGetter, dump2file) int tenant_num = sizeof(orig_key_list) / sizeof(orig_key_list[0]); uint64_t key_version = 0; ObAesOpMode key_algorithm = ObAesOpMode::ob_invalid_mode; + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().set_root_key(OB_SYS_TENANT_ID, + obrpc::RootKeyType::DEFAULT, ObString(), false)); for (int i = 0; i < tenant_num; ++i) { for (int j = 0; j < key_num; ++j) { cur_key = orig_key_list[i][j]; @@ -187,6 +193,7 @@ TEST_F(TestMasterKeyGetter, dump2file) ObMasterKeyGetter::instance().id_value_map_.reuse(); ObMasterKeyGetter::instance().tenant_key_version_map_.reuse(); ObMasterKeyGetter::instance().tenant_table_key_algorithm_map_.reuse(); + ObMasterKeyGetter::instance().root_key_map_.reuse(); EXPECT_EQ(0, ObMasterKeyGetter::instance().id_value_map_.size()); EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().load_key(keystore_file)); EXPECT_EQ(tenant_num * key_num, ObMasterKeyGetter::instance().id_value_map_.size()); @@ -197,7 +204,7 @@ TEST_F(TestMasterKeyGetter, dump2file) EXPECT_STREQ(data, orig_key_list[i][j]); } } - EXPECT_EQ(tenant_num, ObMasterKeyGetter::instance().tenant_key_version_map_.size()); + EXPECT_EQ(1, ObMasterKeyGetter::instance().tenant_key_version_map_.size()); EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().get_max_active_version(0, key_version)); EXPECT_EQ(key_num - 2, key_version); EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().get_max_stored_version(0, key_version)); @@ -221,6 +228,162 @@ TEST_F(TestMasterKeyGetter, dump2file) // EXPECT_EQ(0, ObMasterKeyGetter::instance().tenant_table_key_algorithm_map_.size()); // } +TEST_F(TestMasterKeyGetter, dump_tenant_keys) +{ + int64_t key_num = 5; + char key[OB_MAX_MASTER_KEY_LENGTH] = {0}; + const int64_t key_len = OB_MAX_MASTER_KEY_LENGTH; + int64_t tmp_len = 0; + uint64_t src_tenant_id = 1; + uint64_t dst_tenant_id = 2; + ObMasterKeyBackup key_backup; + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().set_root_key(OB_SYS_TENANT_ID, + obrpc::RootKeyType::DEFAULT, ObString(), false)); + for (int64_t i = 0; i < key_num; ++i) { + EXPECT_EQ(OB_SUCCESS, ObKeyGenerator::generate_encrypt_key(key, key_len)); + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().set_master_key(src_tenant_id, i + 1, key, key_len)); + } + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().dump_tenant_keys(src_tenant_id, key_backup.master_key_list_)); + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().load_tenant_keys(dst_tenant_id, ObAesOpMode::ob_aes_128_ecb, key_backup.master_key_list_)); + EXPECT_EQ(key_num * 2, ObMasterKeyGetter::instance().id_value_map_.size()); + + char src_key[OB_MAX_MASTER_KEY_LENGTH + 1] = {0}; + char dst_key[OB_MAX_MASTER_KEY_LENGTH + 1] = {0}; + for (int64_t i = 0; i < key_num; ++i) { + memset(src_key, 0, OB_MAX_MASTER_KEY_LENGTH + 1); + memset(dst_key, 0, OB_MAX_MASTER_KEY_LENGTH + 1); + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::get_master_key(src_tenant_id, i + 1, src_key, OB_MAX_MASTER_KEY_LENGTH, tmp_len)); + EXPECT_EQ(tmp_len, key_len); + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::get_master_key(src_tenant_id, i + 1, dst_key, OB_MAX_MASTER_KEY_LENGTH, tmp_len)); + EXPECT_EQ(tmp_len, key_len); + EXPECT_STREQ(src_key, dst_key); + } +} + +TEST_F(TestMasterKeyGetter, backup_keys) +{ + int64_t key_num = 5; + char key[OB_MAX_MASTER_KEY_LENGTH] = {0}; + char encrypt_key[OB_MAX_MASTER_KEY_LENGTH] = {0}; + const int64_t key_len = OB_MAX_MASTER_KEY_LENGTH; + int64_t tmp_len = 0; + uint64_t src_tenant_id = 1; + uint64_t dst_tenant_id = 2; + ObString backup_path("file://wallet/key.bak"); + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().set_root_key(OB_SYS_TENANT_ID, + obrpc::RootKeyType::DEFAULT, ObString(), false)); + EXPECT_EQ(OB_SUCCESS, ObKeyGenerator::generate_encrypt_key(encrypt_key, key_len)); + for (int64_t i = 0; i < key_num; ++i) { + EXPECT_EQ(OB_SUCCESS, ObKeyGenerator::generate_encrypt_key(key, key_len)); + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().set_master_key(src_tenant_id, i + 1, key, key_len)); + } + ObString encrypt_key_str(key_len, encrypt_key); + EXPECT_EQ(OB_SUCCESS, ObMasterKeyUtil::backup_key(src_tenant_id, backup_path, encrypt_key_str)); + EXPECT_EQ(OB_SUCCESS, ObMasterKeyUtil::restore_key(dst_tenant_id, backup_path, encrypt_key_str)); + EXPECT_EQ(OB_SUCCESS, ObMasterKeyUtil::restore_key(dst_tenant_id, backup_path, encrypt_key_str)); + EXPECT_EQ(key_num * 2, ObMasterKeyGetter::instance().id_value_map_.size()); + + char src_key[OB_MAX_MASTER_KEY_LENGTH + 1] = {0}; + char dst_key[OB_MAX_MASTER_KEY_LENGTH + 1] = {0}; + for (int64_t i = 0; i < key_num; ++i) { + memset(src_key, 0, OB_MAX_MASTER_KEY_LENGTH + 1); + memset(dst_key, 0, OB_MAX_MASTER_KEY_LENGTH + 1); + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::get_master_key(src_tenant_id, i + 1, src_key, OB_MAX_MASTER_KEY_LENGTH, tmp_len)); + EXPECT_EQ(tmp_len, key_len); + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::get_master_key(dst_tenant_id, i + 1, dst_key, OB_MAX_MASTER_KEY_LENGTH, tmp_len)); + EXPECT_EQ(tmp_len, key_len); + EXPECT_STREQ(src_key, dst_key); + } +} + +TEST_F(TestMasterKeyGetter, dump_root_key) +{ + const int key_num = 3; + ObRootKey key_list[key_num]; + key_list[0].key_type_ = obrpc::RootKeyType::NORMAL; + key_list[0].key_ = ObString("123456"); + key_list[1].key_type_ = obrpc::RootKeyType::DEFAULT; + key_list[2].key_type_ = obrpc::RootKeyType::NORMAL; + key_list[2].key_ = ObString("abcde"); + + for (int i = 0; i < key_num; ++i) { + ObRootKey &root_key = key_list[i]; + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().set_root_key(i + 1, root_key.key_type_, root_key.key_)); + } + EXPECT_EQ(key_num, ObMasterKeyGetter::instance().root_key_map_.size()); + ObMasterKeyGetter::instance().root_key_map_.reuse(); + + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().load_root_key()); + EXPECT_EQ(key_num, ObMasterKeyGetter::instance().root_key_map_.size()); + for (int i = 0; i < key_num; ++i) { + ObRootKey &old_key = key_list[i]; + ObRootKey cur_key; + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().get_root_key(i + 1, cur_key.key_type_, cur_key.key_)); + EXPECT_EQ(old_key.key_type_, cur_key.key_type_); + EXPECT_EQ(0, old_key.key_.compare(cur_key.key_)); + } +} + +TEST_F(TestMasterKeyGetter, backup_root_key) +{ + const int key_num = 3; + char encrypt_key_buf[OB_MAX_MASTER_KEY_LENGTH] = {0}; + const int64_t key_len = OB_MAX_MASTER_KEY_LENGTH; + const char *path_format = "file://wallet/key%d.bak"; + ObString path_not_exist("file://wallet/not_exist.bak"); + char root_key_buf[key_num][OB_MAX_MASTER_KEY_LENGTH] = {}; + ObRootKey key_list[key_num]; + + EXPECT_EQ(OB_SUCCESS, ObKeyGenerator::generate_encrypt_key(encrypt_key_buf, key_len)); + ObString encrypt_key(key_len, encrypt_key_buf); + for (int i = 0; i < key_num; ++i) { + ObRootKey &root_key = key_list[i]; + if (i == 1) { + root_key.key_type_ = obrpc::RootKeyType::DEFAULT; + root_key.key_.reset(); + } else { + EXPECT_EQ(OB_SUCCESS, ObKeyGenerator::generate_encrypt_key(root_key_buf[i], key_len)); + root_key.key_type_ = obrpc::RootKeyType::NORMAL; + root_key.key_.assign_ptr(root_key_buf[i], key_len); + } + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().set_root_key(i, root_key.key_type_, root_key.key_)); + } + EXPECT_EQ(key_num, ObMasterKeyGetter::instance().root_key_map_.size()); + for (int i = 0; i < key_num; ++i) { + ObBackupDest backup_dest; + char bak_path[MAX_PATH_SIZE] = {}; + snprintf(bak_path, MAX_PATH_SIZE, path_format, i); + EXPECT_EQ(OB_SUCCESS, backup_dest.set(bak_path)); + EXPECT_EQ(OB_SUCCESS, ObMasterKeyUtil::backup_root_key(i, backup_dest.get_root_path(), + backup_dest.get_storage_info(), encrypt_key)); + } + ObMasterKeyGetter::instance().root_key_map_.reuse(); + + for (int i = 0; i < key_num; ++i) { + ObBackupDest backup_dest; + char bak_path[MAX_PATH_SIZE] = {}; + snprintf(bak_path, MAX_PATH_SIZE, path_format, i); + EXPECT_EQ(OB_SUCCESS, backup_dest.set(bak_path)); + EXPECT_EQ(OB_SUCCESS, ObMasterKeyUtil::restore_root_key(i + key_num, backup_dest.get_root_path(), + backup_dest.get_storage_info(), encrypt_key)); + } + + EXPECT_EQ(key_num, ObMasterKeyGetter::instance().root_key_map_.size()); + for (int i = 0; i < key_num; ++i) { + ObRootKey &old_key = key_list[i]; + ObRootKey cur_key; + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().get_root_key(i + key_num, cur_key.key_type_, cur_key.key_)); + EXPECT_EQ(old_key.key_type_, cur_key.key_type_); + EXPECT_EQ(0, old_key.key_.compare(cur_key.key_)); + } + ObMasterKeyGetter::instance().root_key_map_.reuse(); + ObBackupDest backup_dest; + EXPECT_EQ(OB_SUCCESS, backup_dest.set(path_not_exist)); + EXPECT_EQ(OB_SUCCESS, ObMasterKeyUtil::restore_root_key(1, backup_dest.get_root_path(), + backup_dest.get_storage_info(), encrypt_key)); + EXPECT_EQ(0, ObMasterKeyGetter::instance().root_key_map_.size()); +} + } // end namespace share } // end namespace oceanbase @@ -229,6 +392,7 @@ int main(int argc, char **argv) system("rm -rf test_master_key_getter.log* wallet"); oceanbase::common::ObLogger::get_logger().set_file_name("test_master_key_getter.log", true); oceanbase::common::ObLogger::get_logger().set_log_level("INFO"); + oceanbase::common::ObClusterVersion::get_instance().update_cluster_version(CLUSTER_VERSION_4_2_0_0); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); }