From ca51217d50dd43e34f3909f889f14d3dd24ca4c5 Mon Sep 17 00:00:00 2001 From: obdev Date: Wed, 18 Sep 2024 09:33:50 +0000 Subject: [PATCH] fix shared storage manifest about extension with multiple fields --- .../object_storage/ob_device_config_mgr.cpp | 2 +- .../ob_device_config_parser.cpp | 280 +++++--------- .../object_storage/ob_device_config_parser.h | 13 +- .../object_storage/ob_device_manifest.cpp | 355 +++++++++--------- src/share/object_storage/ob_device_manifest.h | 9 +- .../ob_device_manifest_task.cpp | 43 +-- unittest/share/device/test_device_config.cpp | 242 ++++++------ 7 files changed, 420 insertions(+), 524 deletions(-) diff --git a/src/share/object_storage/ob_device_config_mgr.cpp b/src/share/object_storage/ob_device_config_mgr.cpp index 49f2fdfb9..a2606e048 100644 --- a/src/share/object_storage/ob_device_config_mgr.cpp +++ b/src/share/object_storage/ob_device_config_mgr.cpp @@ -90,7 +90,7 @@ int ObDeviceConfigMgr::load_configs() ret = OB_NOT_INIT; LOG_WARN("ObDeviceConfigMgr not init", KR(ret)); } else if (OB_FAIL(device_manifest_.load(device_configs, head_))) { - if (OB_FILE_NOT_EXIST == ret) { // first start of observer + if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret) { // first start of observer LOG_INFO("device manifest file does not exist", KR(ret)); ret = OB_SUCCESS; } else { diff --git a/src/share/object_storage/ob_device_config_parser.cpp b/src/share/object_storage/ob_device_config_parser.cpp index 0e8db7ec8..ef9311e25 100644 --- a/src/share/object_storage/ob_device_config_parser.cpp +++ b/src/share/object_storage/ob_device_config_parser.cpp @@ -23,17 +23,10 @@ using namespace common; const char ObDeviceConfigParser::USED_FOR[] = "used_for="; const char ObDeviceConfigParser::PATH[] = "path="; const char ObDeviceConfigParser::ENDPOINT[] = "endpoint="; -const char ObDeviceConfigParser::ACCESS_MODE[] = "access_mode="; -const char ObDeviceConfigParser::ACCESS_ID[] = "access_id="; -const char ObDeviceConfigParser::ACCESS_KEY[] = "access_key="; -const char ObDeviceConfigParser::ENCRYPT_KEY[] = "encrypt_key="; +const char ObDeviceConfigParser::ACCESS_INFO[] = "access_info="; const char ObDeviceConfigParser::ENCRYPT_INFO[] = "encrypt_info="; const char ObDeviceConfigParser::EXTENSION[] = "extension="; -const char ObDeviceConfigParser::OLD_ACCESS_MODE[] = "old_access_mode="; -const char ObDeviceConfigParser::OLD_ACCESS_ID[] = "old_access_id="; - -const char ObDeviceConfigParser::OLD_ACCESS_KEY[] = "old_access_key="; -const char ObDeviceConfigParser::OLD_ENCRYPT_KEY[] = "old_encrypt_key="; +const char ObDeviceConfigParser::OLD_ACCESS_INFO[] = "old_access_info="; const char ObDeviceConfigParser::OLD_ENCRYPT_INFO[] = "old_encrypt_info="; const char ObDeviceConfigParser::OLD_EXTENSION[] = "old_extension="; const char ObDeviceConfigParser::RAM_URL[] = "ram_url="; @@ -50,94 +43,86 @@ const char ObDeviceConfigParser::MAX_BANDWIDTH[] = "max_bandwidth="; const char ObDeviceConfigParser::INVALID_OP_ID[] = "-1"; // because op_id in __all_zone_storage is int(20), but in DeviceConfig is uint64_t, // op_id is init UINT64_MAX, so when ObDeviceManifest::write_head_ to manifest op_id is change as -1 -int ObDeviceConfigParser::parse_one_device_config(char *opt_str, ObDeviceConfig &config) +int ObDeviceConfigParser::parse_device_config_field(const char *buf, ObDeviceConfig &device_config) { int ret = OB_SUCCESS; - char *token = opt_str; - char *saved_ptr = NULL; - config.reset(); - - for (char *str = token; OB_SUCC(ret); str = NULL) { - token = ::strtok_r(str, "&", &saved_ptr); - if (OB_ISNULL(token)) { - break; - } else { - // e.g., used_for=clog - if (OB_SUCC(parse_common_field_(token, config.used_for_, FieldType::TYPE_CHAR_ARRAY, USED_FOR, OB_MAX_STORAGE_USED_FOR_LENGTH))) { - } else if (OB_ITEM_NOT_MATCH != ret) { - LOG_WARN("fail to parse used_for", KR(ret), K(token)); - // e.g., path=oss://xxx/yyy - } else if (OB_SUCC(parse_common_field_(token, config.path_, FieldType::TYPE_CHAR_ARRAY, PATH, OB_MAX_BACKUP_DEST_LENGTH))) { - } else if (OB_ITEM_NOT_MATCH != ret) { - LOG_WARN("fail to parse path", KR(ret), K(token)); - // e.g., endpoint=xxx - } else if (OB_SUCC(parse_common_field_(token, config.endpoint_, FieldType::TYPE_CHAR_ARRAY, ENDPOINT, OB_MAX_BACKUP_ENDPOINT_LENGTH))) { - } else if (OB_ITEM_NOT_MATCH != ret) { - LOG_WARN("fail to parse endpoint", KR(ret), K(token)); - // e.g., access_mode=access_by_id or access_id=xxx or ram_url=xxx - } else if (OB_SUCC(parse_access_info_(token, config.access_info_))) { - } else if (OB_ITEM_NOT_MATCH != ret) { - LOG_WARN("fail to parse access_info", KR(ret), K(token)); - // e.g., encrypt_info=xxx - } else if (OB_SUCC(parse_encrypt_info_(token, config.encrypt_info_))) { - } else if (OB_ITEM_NOT_MATCH != ret) { - LOG_WARN("fail to parse encrypt_info", KR(ret), K(token)); - // e.g., extension=xxx - } else if (OB_SUCC(parse_extension_(token, config.extension_))) { - } else if (OB_ITEM_NOT_MATCH != ret) { - LOG_WARN("fail to parse extension", KR(ret), K(token)); - // e.g., old_access_mode=access_by_id or old_access_id=xxx - } else if (OB_SUCC(parse_access_info_(token, config.old_access_info_, true/*is_old*/))) { - } else if (OB_ITEM_NOT_MATCH != ret) { - LOG_WARN("fail to parse access_info", KR(ret), K(token)); - // e.g., old_encrypt_info=xxx - } else if (OB_SUCC(parse_encrypt_info_(token, config.old_encrypt_info_, true/*is_old*/))) { - } else if (OB_ITEM_NOT_MATCH != ret) { - LOG_WARN("fail to parse encrypt_info", KR(ret), K(token)); - // e.g., old_extension=xxx - } else if (OB_SUCC(parse_extension_(token, config.old_extension_, true/*is_old*/))) { - } else if (OB_ITEM_NOT_MATCH != ret) { - LOG_WARN("fail to parse extension", KR(ret), K(token)); - // e.g., state=ADDING - } else if (OB_SUCC(parse_common_field_(token, config.state_, FieldType::TYPE_CHAR_ARRAY, STATE, OB_MAX_STORAGE_STATE_LENGTH))) { - } else if (OB_ITEM_NOT_MATCH != ret) { - LOG_WARN("fail to parse state", KR(ret), K(token)); - // e.g., state_info=xxx - } else if (OB_SUCC(parse_common_field_(token, config.state_info_, FieldType::TYPE_CHAR_ARRAY, STATE_INFO, OB_MAX_STORAGE_STATE_INFO_LENGTH))) { - } else if (OB_ITEM_NOT_MATCH != ret) { - LOG_WARN("fail to parse state_info", KR(ret), K(token)); - // e.g., create_timestamp=1679579590457 - } else if (OB_SUCC(parse_common_field_(token, &(config.create_timestamp_), FieldType::TYPE_INT, CREATE_TIMESTAMP))) { - } else if (OB_ITEM_NOT_MATCH != ret) { - LOG_WARN("fail to parse create_timestamp_", KR(ret), K(token)); - // e.g., last_check_timestamp=1679579590457 - } else if (OB_SUCC(parse_common_field_(token, &(config.last_check_timestamp_), FieldType::TYPE_INT, LAST_CHECK_TIMESTAMP))) { - } else if (OB_ITEM_NOT_MATCH != ret) { - LOG_WARN("fail to parse last_check_timestamp", KR(ret), K(token)); - // e.g., op_id=2 - } else if (OB_SUCC(parse_common_field_(token, &(config.op_id_), FieldType::TYPE_UINT, OP_ID))) { - } else if (OB_ITEM_NOT_MATCH != ret) { - LOG_WARN("fail to parse op_id", KR(ret), K(token)); - // e.g., sub_op_id=1 - } else if (OB_SUCC(parse_common_field_(token, &(config.sub_op_id_), FieldType::TYPE_UINT, SUB_OP_ID))) { - } else if (OB_ITEM_NOT_MATCH != ret) { - LOG_WARN("fail to parse sub_op_id", KR(ret), K(token)); - // e.g., storage_id=10 - } else if (OB_SUCC(parse_common_field_(token, &(config.storage_id_), FieldType::TYPE_UINT, STORAGE_ID))) { - } else if (OB_ITEM_NOT_MATCH != ret) { - LOG_WARN("fail to parse storage_id", KR(ret), K(token)); - // e.g., iops=10000 - } else if (OB_SUCC(parse_common_field_(token, &(config.max_iops_), FieldType::TYPE_INT, MAX_IOPS))) { - } else if (OB_ITEM_NOT_MATCH != ret) { - LOG_WARN("fail to parse iops", KR(ret), K(token)); - // e.g., bandwidth=1000000000 - } else if (OB_SUCC(parse_common_field_(token, &(config.max_bandwidth_), FieldType::TYPE_INT, MAX_BANDWIDTH))) { - } else if (OB_ITEM_NOT_MATCH != ret) { - LOG_WARN("fail to parse bandwidth", KR(ret), K(token)); - } else { - FLOG_WARN("all item not match, unknown token", KR(ret), K(token)); - } - } + if (OB_ISNULL(buf)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("buf is nullptr", KR(ret)); + // e.g., used_for=clog + } else if (OB_SUCC(parse_common_field_(buf, device_config.used_for_, FieldType::TYPE_CHAR_ARRAY, USED_FOR, OB_MAX_STORAGE_USED_FOR_LENGTH))) { + } else if (OB_ITEM_NOT_MATCH != ret) { + LOG_WARN("fail to parse used_for", KR(ret), K(buf)); + // e.g., path=oss://xxx/yyy + } else if (OB_SUCC(parse_common_field_(buf, device_config.path_, FieldType::TYPE_CHAR_ARRAY, PATH, OB_MAX_BACKUP_DEST_LENGTH))) { + } else if (OB_ITEM_NOT_MATCH != ret) { + LOG_WARN("fail to parse path", KR(ret), K(buf)); + // e.g., endpoint=xxx + } else if (OB_SUCC(parse_common_field_(buf, device_config.endpoint_, FieldType::TYPE_CHAR_ARRAY, ENDPOINT, OB_MAX_BACKUP_ENDPOINT_LENGTH))) { + } else if (OB_ITEM_NOT_MATCH != ret) { + LOG_WARN("fail to parse endpoint", KR(ret), K(buf)); + // e.g., access_info=access_id=xxx&encrypt_key=xxx + } else if (OB_SUCC(parse_access_info_(buf, device_config.access_info_))) { + } else if (OB_ITEM_NOT_MATCH != ret) { + LOG_WARN("fail to parse access_info", KR(ret), K(buf)); + // e.g., encrypt_info=xxx + } else if (OB_SUCC(parse_encrypt_info_(buf, device_config.encrypt_info_))) { + } else if (OB_ITEM_NOT_MATCH != ret) { + LOG_WARN("fail to parse encrypt_info", KR(ret), K(buf)); + // e.g., extension=xxx + } else if (OB_SUCC(parse_extension_(buf, device_config.extension_))) { + } else if (OB_ITEM_NOT_MATCH != ret) { + LOG_WARN("fail to parse extension", KR(ret), K(buf)); + // e.g., old_access_info=access_id=xxx&encrypt_key=xxx + } else if (OB_SUCC(parse_access_info_(buf, device_config.old_access_info_, true/*is_old*/))) { + } else if (OB_ITEM_NOT_MATCH != ret) { + LOG_WARN("fail to parse access_info", KR(ret), K(buf)); + // e.g., old_encrypt_info=xxx + } else if (OB_SUCC(parse_encrypt_info_(buf, device_config.old_encrypt_info_, true/*is_old*/))) { + } else if (OB_ITEM_NOT_MATCH != ret) { + LOG_WARN("fail to parse encrypt_info", KR(ret), K(buf)); + // e.g., old_extension=xxx + } else if (OB_SUCC(parse_extension_(buf, device_config.old_extension_, true/*is_old*/))) { + } else if (OB_ITEM_NOT_MATCH != ret) { + LOG_WARN("fail to parse extension", KR(ret), K(buf)); + // e.g., state=ADDING + } else if (OB_SUCC(parse_common_field_(buf, device_config.state_, FieldType::TYPE_CHAR_ARRAY, STATE, OB_MAX_STORAGE_STATE_LENGTH))) { + } else if (OB_ITEM_NOT_MATCH != ret) { + LOG_WARN("fail to parse state", KR(ret), K(buf)); + // e.g., state_info=xxx + } else if (OB_SUCC(parse_common_field_(buf, device_config.state_info_, FieldType::TYPE_CHAR_ARRAY, STATE_INFO, OB_MAX_STORAGE_STATE_INFO_LENGTH))) { + } else if (OB_ITEM_NOT_MATCH != ret) { + LOG_WARN("fail to parse state_info", KR(ret), K(buf)); + // e.g., create_timestamp=1679579590457 + } else if (OB_SUCC(parse_common_field_(buf, &(device_config.create_timestamp_), FieldType::TYPE_INT, CREATE_TIMESTAMP))) { + } else if (OB_ITEM_NOT_MATCH != ret) { + LOG_WARN("fail to parse create_timestamp_", KR(ret), K(buf)); + // e.g., last_check_timestamp=1679579590457 + } else if (OB_SUCC(parse_common_field_(buf, &(device_config.last_check_timestamp_), FieldType::TYPE_INT, LAST_CHECK_TIMESTAMP))) { + } else if (OB_ITEM_NOT_MATCH != ret) { + LOG_WARN("fail to parse last_check_timestamp", KR(ret), K(buf)); + // e.g., op_id=2 + } else if (OB_SUCC(parse_common_field_(buf, &(device_config.op_id_), FieldType::TYPE_UINT, OP_ID))) { + } else if (OB_ITEM_NOT_MATCH != ret) { + LOG_WARN("fail to parse op_id", KR(ret), K(buf)); + // e.g., sub_op_id=1 + } else if (OB_SUCC(parse_common_field_(buf, &(device_config.sub_op_id_), FieldType::TYPE_UINT, SUB_OP_ID))) { + } else if (OB_ITEM_NOT_MATCH != ret) { + LOG_WARN("fail to parse sub_op_id", KR(ret), K(buf)); + // e.g., storage_id=10 + } else if (OB_SUCC(parse_common_field_(buf, &(device_config.storage_id_), FieldType::TYPE_UINT, STORAGE_ID))) { + } else if (OB_ITEM_NOT_MATCH != ret) { + LOG_WARN("fail to parse storage_id", KR(ret), K(buf)); + // e.g., iops=10000 + } else if (OB_SUCC(parse_common_field_(buf, &(device_config.max_iops_), FieldType::TYPE_INT, MAX_IOPS))) { + } else if (OB_ITEM_NOT_MATCH != ret) { + LOG_WARN("fail to parse iops", KR(ret), K(buf)); + // e.g., bandwidth=1000000000 + } else if (OB_SUCC(parse_common_field_(buf, &(device_config.max_bandwidth_), FieldType::TYPE_INT, MAX_BANDWIDTH))) { + } else if (OB_ITEM_NOT_MATCH != ret) { + LOG_WARN("fail to parse bandwidth", KR(ret), K(buf)); + } else { + LOG_WARN("all item not match", KR(ret), K(buf)); } return ret; } @@ -178,43 +163,19 @@ int ObDeviceConfigParser::parse_access_info_( const bool is_old) { int ret = OB_SUCCESS; -#ifndef OB_BUILD_TDE_SECURITY - const char *access_info_keys[] = {ACCESS_MODE, ACCESS_ID, ACCESS_KEY, RAM_URL}; - const char *old_access_info_keys[] = {OLD_ACCESS_MODE, OLD_ACCESS_ID, OLD_ACCESS_KEY, OLD_RAM_URL}; -#else - const char *access_info_keys[] = {ACCESS_MODE, ACCESS_ID, ENCRYPT_KEY, RAM_URL}; - const char *old_access_info_keys[] = {OLD_ACCESS_MODE, OLD_ACCESS_ID, OLD_ENCRYPT_KEY, OLD_RAM_URL}; -#endif - STATIC_ASSERT(ARRAYSIZEOF(access_info_keys) == ARRAYSIZEOF(old_access_info_keys), "key count mismatch"); - const int64_t key_cnt = ARRAYSIZEOF(access_info_keys); - const char *key_name = nullptr; - bool is_item_match = false; - - if (OB_ISNULL(token) || OB_ISNULL(access_info)) { + if (OB_ISNULL(token)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), KP(token), KP(access_info)); - } - - for (int64_t i = 0; OB_SUCC(ret) && (i < key_cnt); ++i) { - key_name = is_old ? old_access_info_keys[i] : access_info_keys[i]; - if (0 == STRNCMP(key_name, token, STRLEN(key_name))) { - const int64_t access_info_len = STRLEN(access_info); - const char *value_str = token + STRLEN(key_name); - if (0 == STRLEN(value_str)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("value is empty", KR(ret), K(key_name), K(token)); - } else if (0 == access_info_len) { // first field of access_info - STRCPY(access_info, token); - } else { // not first field of access_info, need to concat '&' - access_info[access_info_len] = '&'; - STRCPY(access_info + access_info_len + 1, token); - } - is_item_match = true; - break; + LOG_WARN("invalid argument", KR(ret), KP(token)); + } else if (is_old) { // old_access_info=xxx + if (OB_FAIL(parse_config_type_char_array(OLD_ACCESS_INFO, token, OB_MAX_BACKUP_AUTHORIZATION_LENGTH, + access_info)) && (OB_ITEM_NOT_MATCH != ret)) { + LOG_WARN("fail to parse_config_type_char_array", KR(ret), K(OLD_ENCRYPT_INFO), K(token)); + } + } else { // access_info=xxx + if (OB_FAIL(parse_config_type_char_array(ACCESS_INFO, token, OB_MAX_BACKUP_AUTHORIZATION_LENGTH, + access_info)) && (OB_ITEM_NOT_MATCH != ret)) { + LOG_WARN("fail to parse_config_type_char_array", KR(ret), K(ENCRYPT_INFO), K(token)); } - } - if (OB_SUCC(ret) && !is_item_match) { - ret = OB_ITEM_NOT_MATCH; } return ret; } @@ -282,10 +243,6 @@ int ObDeviceConfigParser::parse_config_type_char_array( ret = OB_INVALID_ARGUMENT; LOG_WARN("value str is too long", KR(ret), K(key_name), "value_len", STRLEN(value_str), K(max_value_len)); - // when restart server, because manifest's head.shared_storage_info_ is '\0', so head.shared_storage_info_ must be overwritten - // TODO:当沧溟新的bootstrap共享存储集群的代码合入了后，这里删除掉 @xiaotao.ht - } else if (STRLEN(value_str) == 0) { - MEMSET(value, 0, max_value_len); } else { STRCPY(value, value_str); } @@ -367,60 +324,5 @@ int ObDeviceConfigParser::parse_config_type_uint( return ret; } -// 1. convert from "access_mode=access_by_id&access_id=xxx" to -// "old_access_mode=access_by_id&old_access_id=xxx" -// 2. convert from "access_mode=access_by_ram_url&ram_url=xxx" to -// "old_access_mode=access_by_ram_url&old_ram_url=xxx" -int ObDeviceConfigParser::convert_access_info_to_old(char *access_info, char *old_access_info) -{ - int ret = OB_SUCCESS; -#ifndef OB_BUILD_TDE_SECURITY - const char *access_info_keys[] = {ACCESS_MODE, ACCESS_ID, ACCESS_KEY, RAM_URL}; -#else - const char *access_info_keys[] = {ACCESS_MODE, ACCESS_ID, ENCRYPT_KEY, ACCESS_KEY, RAM_URL}; -#endif - const int64_t key_cnt = ARRAYSIZEOF(access_info_keys); - const char *key_name = nullptr; - char *token = access_info; - char *saved_ptr = NULL; - if (OB_ISNULL(access_info) || OB_ISNULL(old_access_info)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), KP(access_info), KP(old_access_info)); - } else { - for (char *str = token; OB_SUCC(ret); str = NULL) { - token = ::strtok_r(str, "&", &saved_ptr); - if (OB_ISNULL(token)) { - break; - } else { - bool is_item_match = false; - for (int64_t i = 0; OB_SUCC(ret) && (i < key_cnt); ++i) { - key_name = access_info_keys[i]; - if (0 == STRNCMP(key_name, token, STRLEN(key_name))) { - const int64_t old_access_info_len = STRLEN(old_access_info); - const char *value_str = token + STRLEN(key_name); - if (0 == STRLEN(value_str)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("value is empty", KR(ret), K(key_name), K(token)); - } else if (0 == old_access_info_len) { // first field of old_access_info - STRCPY(old_access_info, "old_"); - STRCPY(old_access_info + STRLEN("old_"), token); - } else { // not first field of old_access_info, need to concat '&' - old_access_info[old_access_info_len] = '&'; - STRCPY(old_access_info + old_access_info_len + 1, "old_"); - STRCPY(old_access_info + old_access_info_len + 1 + STRLEN("old_"), token); - } - is_item_match = true; - break; - } - } - if (OB_SUCC(ret) && !is_item_match) { - ret = OB_ITEM_NOT_MATCH; - } - } - } - } - return ret; -} - } // namespace share } // namespace oceanbase diff --git a/src/share/object_storage/ob_device_config_parser.h b/src/share/object_storage/ob_device_config_parser.h index 243971112..767b7dd0d 100644 --- a/src/share/object_storage/ob_device_config_parser.h +++ b/src/share/object_storage/ob_device_config_parser.h @@ -25,16 +25,10 @@ public: static const char USED_FOR[]; static const char PATH[]; static const char ENDPOINT[]; - static const char ACCESS_MODE[]; - static const char ACCESS_ID[]; - static const char ACCESS_KEY[]; - static const char ENCRYPT_KEY[]; + static const char ACCESS_INFO[]; static const char ENCRYPT_INFO[]; static const char EXTENSION[]; - static const char OLD_ACCESS_MODE[]; - static const char OLD_ACCESS_ID[]; - static const char OLD_ACCESS_KEY[]; - static const char OLD_ENCRYPT_KEY[]; + static const char OLD_ACCESS_INFO[]; static const char OLD_ENCRYPT_INFO[]; static const char OLD_EXTENSION[]; static const char RAM_URL[]; @@ -51,14 +45,13 @@ public: static const char INVALID_OP_ID[]; public: - static int parse_one_device_config(char *opt_str, ObDeviceConfig &config); + static int parse_device_config_field(const char *buf, ObDeviceConfig &device_config); static int parse_config_type_int(const char *key_name, const char *token, int64_t &value); static int parse_config_type_uint(const char *key_name, const char *token, uint64_t &value); static int parse_config_type_char_array(const char *key_name, const char *token, const int64_t max_value_len, char *value); - static int convert_access_info_to_old(char *access_info, char *old_access_info); private: enum class FieldType diff --git a/src/share/object_storage/ob_device_manifest.cpp b/src/share/object_storage/ob_device_manifest.cpp index 420171a0e..155d577f5 100644 --- a/src/share/object_storage/ob_device_manifest.cpp +++ b/src/share/object_storage/ob_device_manifest.cpp @@ -30,7 +30,8 @@ const int64_t ObDeviceManifest::MANIFEST_VERSION = 1; const int64_t ObDeviceManifest::MAX_FILE_LINE_LEN = 16384; // 16K const char ObDeviceManifest::MANIFEST_FILE_NAME[] = "manifest"; const char ObDeviceManifest::HEAD_SECTION[] = "[head]"; -const char ObDeviceManifest::DEVICE_SECTION[] = "[device]"; +const char ObDeviceManifest::DEVICE_BEGIN_SECTION[] = "[device begin]"; +const char ObDeviceManifest::DEVICE_END_SECTION[] = "[device end]"; const char ObDeviceManifest::COMMENT_SYMBOL = '#'; const char ObDeviceManifest::VERSION_KEY[] = "version="; const char ObDeviceManifest::CLUSTER_ID_KEY[] = "cluster_id="; @@ -71,17 +72,15 @@ int ObDeviceManifest::load(ObIArray &config_arr, HeadSection &he if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObDeviceManifest not init", KR(ret)); + } else if (OB_ISNULL(data_dir_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("data dir is nullptr", KR(ret)); } else if (OB_FAIL(databuff_printf(manifest_path, OB_MAX_FILE_NAME_LENGTH, "%s/%s", data_dir_, MANIFEST_FILE_NAME))) { LOG_WARN("construct manifest path fail", KR(ret)); } else if (OB_ISNULL(fp = fopen(manifest_path, "r"))) { - if (ENOENT == errno) { - ret = OB_FILE_NOT_EXIST; - LOG_INFO("device manifest file does not exist", KR(ret), K(manifest_path)); - } else { - ret = OB_IO_ERROR; - LOG_WARN("cannot open file", KR(ret), K(manifest_path), K(errno)); - } + ret = ObIODeviceLocalFileOp::convert_sys_errno(); + LOG_WARN("fail to open device manifest file", KR(ret), K(errno), KERRMSG); } else if (OB_FAIL(parse_file_(fp, config_arr, head))) { LOG_WARN("fail to parse file", KR(ret), K(manifest_path)); } else if (OB_FAIL(validate_config_checksum_(config_arr, head))) { @@ -91,7 +90,10 @@ int ObDeviceManifest::load(ObIArray &config_arr, HeadSection &he LOG_WARN("invalid device manifest", K(cluster_id_in_GCONF), "manifest_cluster_id", head.cluster_id_); } if (OB_NOT_NULL(fp)) { - fclose(fp); + if (OB_UNLIKELY(0 != fclose(fp))) { + ret = ObIODeviceLocalFileOp::convert_sys_errno(); + LOG_ERROR("fail to close file", KR(ret), K(errno), KERRMSG); + } } LOG_INFO("finish to load device manifest", KR(ret), K(config_arr), K(head)); return ret; @@ -101,23 +103,35 @@ int ObDeviceManifest::parse_file_(FILE *fp, ObIArray &config_arr { int ret = OB_SUCCESS; SectionType curr_section_type = SECTION_TYPE_INVLAID; - SMART_VAR(char[MAX_FILE_LINE_LEN + 2], line_buf) { + SMART_VARS_2((char[MAX_FILE_LINE_LEN + 2], line_buf), (ObDeviceConfig, device_config)) { while (OB_SUCC(ret)) { if (0 != feof(fp)) { break; // end of file - } else if (0 != ferror(fp)) { + } else if (OB_UNLIKELY(0 != ferror(fp))) { ret = OB_IO_ERROR; LOG_WARN("fail to read manifest file", KR(ret), K(errno)); } else if (OB_ISNULL(fgets(line_buf, sizeof(line_buf), fp))) { // do nothing - } else if (STRLEN(line_buf) > MAX_FILE_LINE_LEN) { + } else if (OB_UNLIKELY(STRLEN(line_buf) > MAX_FILE_LINE_LEN)) { ret = OB_SIZE_OVERFLOW; LOG_WARN("file line len is too long", KR(ret), K(MAX_FILE_LINE_LEN)); } else if (FALSE_IT(line_buf[STRLEN(line_buf) - 1] = '\0')) { // remove the '\n' } else if (0 == STRLEN(line_buf) || (COMMENT_SYMBOL == line_buf[0])) { // skip blank line or comment line } else if (OB_SUCC(parse_section_type_(line_buf, curr_section_type))) { - // this line is a section name + // this line is a section name. + // SECTION_TYPE_DEVICE_END means finishing parsing one device config, push back the + // parsed device config and reset device_config to parse the next device config. + if (SECTION_TYPE_DEVICE_END == curr_section_type) { + if (OB_UNLIKELY(!device_config.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid device config", KR(ret), K(device_config)); + } else if (OB_FAIL(config_arr.push_back(device_config))) { + LOG_WARN("fail to push back", KR(ret), K(device_config)); + } else { + device_config.reset(); + } + } } else if (OB_ITEM_NOT_MATCH != ret) { LOG_WARN("fail to parse section type", KR(ret), K(line_buf)); } else if (SECTION_TYPE_HEAD == curr_section_type) { @@ -125,23 +139,17 @@ int ObDeviceManifest::parse_file_(FILE *fp, ObIArray &config_arr if (OB_FAIL(parse_head_section_(line_buf, head))) { LOG_WARN("fail to parse head section", KR(ret), K(line_buf), K(head)); } - } else if (SECTION_TYPE_DEVICE == curr_section_type) { + } else if (SECTION_TYPE_DEVICE_BEGIN == curr_section_type) { ret = OB_SUCCESS; // ignore ret, and then parse device section - SMART_VAR(ObDeviceConfig, device_config) { - if (OB_FAIL(parse_device_section_(line_buf, device_config))) { - LOG_WARN("fail to parse device section", KR(ret), K(line_buf)); - } else if (OB_FAIL(config_arr.push_back(device_config))) { - LOG_WARN("fail to push back", KR(ret), K(device_config)); - } else { - LOG_INFO("succ to parse device section", K(device_config)); - } + if (OB_FAIL(parse_device_section_(line_buf, device_config))) { + LOG_WARN("fail to parse device section", KR(ret), K(line_buf)); } } else { LOG_WARN("unknow manifest section", KR(ret), K(curr_section_type), K(line_buf)); } } } - if (OB_SUCC(ret) && (config_arr.count() != head.device_num_)) { + if (OB_SUCC(ret) && OB_UNLIKELY((config_arr.count() != head.device_num_))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("device num does not match between head and device sections", KR(ret), K(head), K(config_arr)); } @@ -154,8 +162,10 @@ int ObDeviceManifest::parse_section_type_(const char *buf, SectionType &type) int ret = OB_SUCCESS; if (0 == STRCMP(buf, HEAD_SECTION)) { type = SECTION_TYPE_HEAD; - } else if (0 == STRCMP(buf, DEVICE_SECTION)) { - type = SECTION_TYPE_DEVICE; + } else if (0 == STRCMP(buf, DEVICE_BEGIN_SECTION)) { + type = SECTION_TYPE_DEVICE_BEGIN; + } else if (0 == STRCMP(buf, DEVICE_END_SECTION)) { + type = SECTION_TYPE_DEVICE_END; } else { ret = OB_ITEM_NOT_MATCH; } @@ -165,7 +175,10 @@ int ObDeviceManifest::parse_section_type_(const char *buf, SectionType &type) int ObDeviceManifest::parse_head_section_(const char *buf, HeadSection &head) { int ret = OB_SUCCESS; - if (OB_SUCC(ObDeviceConfigParser::parse_config_type_int(VERSION_KEY, buf, head.version_))) { + if (OB_ISNULL(buf)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("buf is nullptr", KR(ret)); + } else if (OB_SUCC(ObDeviceConfigParser::parse_config_type_int(VERSION_KEY, buf, head.version_))) { } else if (OB_ITEM_NOT_MATCH != ret) { LOG_WARN("fail to parse_config_type_int", KR(ret), K(VERSION_KEY), K(buf)); } else if (OB_SUCC(ObDeviceConfigParser::parse_config_type_int(CLUSTER_ID_KEY, buf, @@ -202,20 +215,18 @@ int ObDeviceManifest::parse_head_section_(const char *buf, HeadSection &head) return ret; } -int ObDeviceManifest::parse_device_section_(char *buf, ObDeviceConfig &device_config) +int ObDeviceManifest::parse_device_section_(const char *buf, ObDeviceConfig &device_config) { int ret = OB_SUCCESS; - if (OB_SUCC(ObDeviceConfigParser::parse_one_device_config(buf, device_config))) { - if (!device_config.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid non-initial device config", KR(ret), K(device_config)); + if (OB_ISNULL(buf)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("buf is nullptr", KR(ret)); + } else if (OB_FAIL(ObDeviceConfigParser::parse_device_config_field(buf, device_config))) { + if (OB_ITEM_NOT_MATCH == ret) { + LOG_WARN("unknow manifest device section line", KR(ret), K(buf)); } else { - LOG_INFO("succ to parse one device config", K(device_config)); + LOG_WARN("fail to parse device config field", KR(ret), K(buf)); } - } else if (OB_ITEM_NOT_MATCH != ret) { - LOG_WARN("fail to parse_one_device_config", KR(ret), K(buf)); - } else { - LOG_WARN("unknow manifest device section line", KR(ret), K(buf)); } return ret; } @@ -256,6 +267,9 @@ int ObDeviceManifest::dump2file( allocator.alloc(OB_MAX_FILE_NAME_LENGTH)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to malloc memory for his_manifest_path", KR(ret), K(OB_MAX_FILE_NAME_LENGTH)); + } else if (OB_ISNULL(data_dir_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("data dir is nullptr", KR(ret)); } else if (OB_FAIL(databuff_printf(manifest_path, OB_MAX_FILE_NAME_LENGTH, "%s/%s", data_dir_, MANIFEST_FILE_NAME))) { LOG_WARN("fail to construct manifest path", KR(ret)); @@ -272,11 +286,8 @@ int ObDeviceManifest::dump2file( } else if (OB_ISNULL(fp = ::fdopen(fd, "w"))) { ret = ObIODeviceLocalFileOp::convert_sys_errno(); LOG_WARN("fail to fdopen", KR(ret), K(fd), K(errno), KERRMSG); - } else if (FALSE_IT(pret = - fprintf(fp, "# THIS FILE IS AUTOMATICALLY GENERATED BY OBSERVER. PLEASE DO NOT MODIFY IT MANUALLY!!!\n"))) { - } else if (pret <= 0 || pret > MAX_FILE_LINE_LEN) { - ret = OB_IO_ERROR; - LOG_WARN("fail to write manifest", KR(ret), K(pret), K(errno)); + } else if (OB_FAIL(print_file_line_(fp, "# THIS FILE IS AUTOMATICALLY GENERATED BY OBSERVER. PLEASE DO NOT MODIFY IT MANUALLY!!!\n"))) { + LOG_WARN("fail to print file line", KR(ret)); } else if (OB_FAIL(write_head_(fp, head))) { LOG_WARN("fail to write head", KR(ret), K(head)); } else if (OB_FAIL(write_device_config_(fp, configs))) { @@ -308,66 +319,77 @@ int ObDeviceManifest::dump2file( return ret; } +// The format of head section is like: +// [head] +// version=1 +// cluster_id=0 +// head_checksum=790215355 +// device_checksum=2094936029 +// device_num=2 +// modify_timestamp_us=1725551141912558 +// last_op_id=3 +// last_sub_op_id=4 int ObDeviceManifest::write_head_(FILE *fp, const HeadSection &head) const { int ret = OB_SUCCESS; - int pret = 0; - if (FALSE_IT(pret = fprintf(fp, "%s\n", HEAD_SECTION))) { // [head] - } else if (pret <= 0 || pret > MAX_FILE_LINE_LEN) { - ret = OB_IO_ERROR; - LOG_WARN("fail to write manifest", KR(ret), K(pret), K(errno)); - } else if (FALSE_IT(pret = fprintf(fp, "%s%ld\n", VERSION_KEY, head.version_))) { // version= - } else if (pret <= 0 || pret > MAX_FILE_LINE_LEN) { - ret = OB_IO_ERROR; - LOG_WARN("fail to write manifest", KR(ret), K(pret), K(errno)); - } else if (FALSE_IT(pret = fprintf(fp, "%s%ld\n", CLUSTER_ID_KEY, head.cluster_id_))) { // cluster_id= - } else if (pret <= 0 || pret > MAX_FILE_LINE_LEN) { - ret = OB_IO_ERROR; - LOG_WARN("fail to write manifest", KR(ret), K(pret), K(errno)); - } else if (FALSE_IT(pret = fprintf(fp, "%s%ld\n", HEAD_CHECKSUM_KEY, head.head_checksum_))) { // head_checksum= - } else if (pret <= 0 || pret > MAX_FILE_LINE_LEN) { - ret = OB_IO_ERROR; - LOG_WARN("fail to write manifest", KR(ret), K(pret), K(errno)); - } else if (FALSE_IT(pret = fprintf(fp, "%s%ld\n", DEVICE_CHECKSUM_KEY, head.device_checksum_))) { // device_checksum= - } else if (pret <= 0 || pret > MAX_FILE_LINE_LEN) { - ret = OB_IO_ERROR; - LOG_WARN("fail to write manifest", KR(ret), K(pret), K(errno)); - } else if (FALSE_IT(pret = fprintf(fp, "%s%ld\n", DEVICE_NUM_KEY, head.device_num_))) { // device_num= - } else if (pret <= 0 || pret > MAX_FILE_LINE_LEN) { - ret = OB_IO_ERROR; - LOG_WARN("fail to write manifest", KR(ret), K(pret), K(errno)); - } else if (FALSE_IT(pret = fprintf(fp, "%s%ld\n", MODIFY_TIMESTAMP_KEY, head.modify_timestamp_us_))) { // modify_timestamp_us= - } else if (pret <= 0 || pret > MAX_FILE_LINE_LEN) { - ret = OB_IO_ERROR; - LOG_WARN("fail to write manifest", KR(ret), K(pret), K(errno)); - } else if (FALSE_IT(pret = fprintf(fp, "%s%ld\n", LAST_OP_ID_KEY, head.last_op_id_))) { // last_op_id= - } else if (pret <= 0 || pret > MAX_FILE_LINE_LEN) { - ret = OB_IO_ERROR; - LOG_WARN("fail to write manifest", KR(ret), K(pret), K(errno)); - } else if (FALSE_IT(pret = fprintf(fp, "%s%ld\n", LAST_SUB_OP_ID_KEY, head.last_sub_op_id_))) { // last_sub_op_id= - } else if (pret <= 0 || pret > MAX_FILE_LINE_LEN) { - ret = OB_IO_ERROR; - LOG_WARN("fail to write manifest", KR(ret), K(pret), K(errno)); - } else if (FALSE_IT(pret = fprintf(fp, "%s\n", DEVICE_SECTION))) { // [device] - } else if (pret <= 0 || pret > MAX_FILE_LINE_LEN) { - ret = OB_IO_ERROR; - LOG_WARN("fail to write manifest", KR(ret), K(pret), K(errno)); + if (OB_FAIL(print_file_line_(fp, "%s\n", HEAD_SECTION))) { // [head] + LOG_WARN("fail to print file line", KR(ret)); + } else if (OB_FAIL(print_file_line_(fp, "%s%ld\n", VERSION_KEY, head.version_))) { // version= + LOG_WARN("fail to print file line", KR(ret)); + } else if (OB_FAIL(print_file_line_(fp, "%s%ld\n", CLUSTER_ID_KEY, head.cluster_id_))) { // cluster_id= + LOG_WARN("fail to print file line", KR(ret)); + } else if (OB_FAIL(print_file_line_(fp, "%s%ld\n", HEAD_CHECKSUM_KEY, head.head_checksum_))) { // head_checksum= + LOG_WARN("fail to print file line", KR(ret)); + } else if (OB_FAIL(print_file_line_(fp, "%s%ld\n", DEVICE_CHECKSUM_KEY, head.device_checksum_))) { // device_checksum= + LOG_WARN("fail to print file line", KR(ret)); + } else if (OB_FAIL(print_file_line_(fp, "%s%ld\n", DEVICE_NUM_KEY, head.device_num_))) { // device_num= + LOG_WARN("fail to print file line", KR(ret)); + } else if (OB_FAIL(print_file_line_(fp, "%s%ld\n", MODIFY_TIMESTAMP_KEY, head.modify_timestamp_us_))) { // modify_timestamp_us= + LOG_WARN("fail to print file line", KR(ret)); + } else if (OB_FAIL(print_file_line_(fp, "%s%ld\n", LAST_OP_ID_KEY, head.last_op_id_))) { // last_op_id= + LOG_WARN("fail to print file line", KR(ret)); + } else if (OB_FAIL(print_file_line_(fp, "%s%ld\n", LAST_SUB_OP_ID_KEY, head.last_sub_op_id_))) { // last_sub_op_id= + LOG_WARN("fail to print file line", KR(ret)); } - LOG_INFO("finish to write head", KR(ret)); + LOG_INFO("finish to write manifest head", KR(ret)); return ret; } -// The format of each line in device section is like: -// Example 1: -// used_for=...&path=...&endpoint=... -// &access_mode=access_by_id&access_id=...&access_key=...&encrypt_info=...&extension=... -// &old_access_mode=access_by_id&old_access_id=...&old_access_key=...&old_encrypt_info=... -// &old_extension=...&state=...&state_info=...&create_timestamp=...&last_check_timestamp=... -// &op_id=...&sub_op_id=...&storage_id=...&iops=...&bandwidth=... -// Example 2: -// used_for=...&path=...&endpoint=...&access_mode=access_by_ram_url&ram_url=xxx&state=... -// &state_info=...&create_timestamp=...&last_check_timestamp=...&op_id=...&sub_op_id=... -// &storage_id=...&iops=...&bandwidth=... +// The format of device section is like: +// [device begin] +// used_for=DATA +// path=oss://bucket-name/root_dir +// endpoint=host=cn-xxx.com +// access_info=access_id=ccccc&encrypt_key=ddddd +// extension=appid=456&checksum_type=md5 +// old_access_info=access_id=aaa&encrypt_key=bbb +// old_extension=appid=123&checksum_type=crc32 +// state=ADDED +// state_info=is_connective=true +// create_timestamp=1679579590156 +// last_check_timestamp=1679579590256 +// op_id=3 +// sub_op_id=4 +// storage_id=2 +// max_iops=10000 +// max_bandwidth=1024000000 +// [device end] +// [device begin] +// used_for=LOG +// path=oss://bucket-name/root_dir +// endpoint=host=cn-xxx.com +// access_info=access_id=aaa&encrypt_key=bbb +// extension=appid=123&checksum_type=crc32 +// state=ADDED +// state_info=is_connective=true +// create_timestamp=1679579590156 +// last_check_timestamp=1679579590256 +// op_id=1 +// sub_op_id=2 +// storage_id=1 +// max_iops=10000 +// max_bandwidth=1024000000 +// [device end] // Note: not all the fields are needed. int ObDeviceManifest::write_device_config_( FILE *fp, @@ -383,87 +405,53 @@ int ObDeviceManifest::write_device_config_( if (!config.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("device config is invalid", KR(ret), K(config)); - } else if (OB_FAIL(databuff_printf(buf, MAX_FILE_LINE_LEN, "%s%s&%s%s&%s%s&%s", - ObDeviceConfigParser::USED_FOR, config.used_for_, - ObDeviceConfigParser::PATH, config.path_, - ObDeviceConfigParser::ENDPOINT, config.endpoint_, - config.access_info_))) { - LOG_WARN("fail to construct device config line buf", KR(ret)); + } else if (OB_FAIL(print_file_line_(fp, "%s\n", DEVICE_BEGIN_SECTION))) { // [device begin] + LOG_WARN("fail to print file line", KR(ret)); + } else if (OB_FAIL(print_file_line_(fp, "%s%s\n", ObDeviceConfigParser::USED_FOR, config.used_for_))) { + LOG_WARN("fail to print file line", KR(ret)); + } else if (OB_FAIL(print_file_line_(fp, "%s%s\n", ObDeviceConfigParser::PATH, config.path_))) { + LOG_WARN("fail to print file line", KR(ret)); + } else if (OB_FAIL(print_file_line_(fp, "%s%s\n", ObDeviceConfigParser::ENDPOINT, config.endpoint_))) { + LOG_WARN("fail to print file line", KR(ret)); + } else if (OB_FAIL(print_file_line_(fp, "%s%s\n", ObDeviceConfigParser::ACCESS_INFO, config.access_info_))) { + LOG_WARN("fail to print file line", KR(ret)); + } else if ((0 < STRLEN(config.encrypt_info_)) && + OB_FAIL(print_file_line_(fp, "%s%s\n", ObDeviceConfigParser::ENCRYPT_INFO, config.encrypt_info_))) { + LOG_WARN("fail to print file line", KR(ret)); + } else if ((0 < STRLEN(config.extension_)) && + OB_FAIL(print_file_line_(fp, "%s%s\n", ObDeviceConfigParser::EXTENSION, config.extension_))) { + LOG_WARN("fail to print file line", KR(ret)); + } else if ((0 < STRLEN(config.old_access_info_)) && + OB_FAIL(print_file_line_(fp, "%s%s\n", ObDeviceConfigParser::OLD_ACCESS_INFO, config.old_access_info_))) { + LOG_WARN("fail to print file line", KR(ret)); + } else if ((0 < STRLEN(config.old_encrypt_info_)) && + OB_FAIL(print_file_line_(fp, "%s%s\n", ObDeviceConfigParser::OLD_ENCRYPT_INFO, config.old_encrypt_info_))) { + LOG_WARN("fail to print file line", KR(ret)); + } else if ((0 < STRLEN(config.old_extension_)) && + OB_FAIL(print_file_line_(fp, "%s%s\n", ObDeviceConfigParser::OLD_EXTENSION, config.old_extension_))) { + LOG_WARN("fail to print file line", KR(ret)); + } else if (OB_FAIL(print_file_line_(fp, "%s%s\n", ObDeviceConfigParser::STATE, config.state_))) { + LOG_WARN("fail to print file line", KR(ret)); + } else if (OB_FAIL(print_file_line_(fp, "%s%s\n", ObDeviceConfigParser::STATE_INFO, config.state_info_))) { + LOG_WARN("fail to print file line", KR(ret)); + } else if (OB_FAIL(print_file_line_(fp, "%s%ld\n", ObDeviceConfigParser::CREATE_TIMESTAMP, config.create_timestamp_))) { + LOG_WARN("fail to print file line", KR(ret)); + } else if (OB_FAIL(print_file_line_(fp, "%s%ld\n", ObDeviceConfigParser::LAST_CHECK_TIMESTAMP, config.last_check_timestamp_))) { + LOG_WARN("fail to print file line", KR(ret)); + } else if (OB_FAIL(print_file_line_(fp, "%s%lu\n", ObDeviceConfigParser::OP_ID, config.op_id_))) { + LOG_WARN("fail to print file line", KR(ret)); + } else if (OB_FAIL(print_file_line_(fp, "%s%lu\n", ObDeviceConfigParser::SUB_OP_ID, config.sub_op_id_))) { + LOG_WARN("fail to print file line", KR(ret)); + } else if (OB_FAIL(print_file_line_(fp, "%s%lu\n", ObDeviceConfigParser::STORAGE_ID, config.storage_id_))) { + LOG_WARN("fail to print file line", KR(ret)); + } else if (OB_FAIL(print_file_line_(fp, "%s%ld\n", ObDeviceConfigParser::MAX_IOPS, config.max_iops_))) { + LOG_WARN("fail to print file line", KR(ret)); + } else if (OB_FAIL(print_file_line_(fp, "%s%ld\n", ObDeviceConfigParser::MAX_BANDWIDTH, config.max_bandwidth_))) { + LOG_WARN("fail to print file line", KR(ret)); + } else if (OB_FAIL(print_file_line_(fp, "%s\n", DEVICE_END_SECTION))) { // [device end] + LOG_WARN("fail to print file line", KR(ret)); } - if (OB_SUCC(ret) && 0 != STRLEN(config.encrypt_info_)) { - if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf), - "&%s%s", ObDeviceConfigParser::ENCRYPT_INFO, config.encrypt_info_))) { - LOG_WARN("fail to construct device config line buf", KR(ret), K(config)); - } - } - if (OB_SUCC(ret) && 0 != STRLEN(config.extension_)) { - if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf), - "&%s%s", ObDeviceConfigParser::EXTENSION, config.extension_))) { - LOG_WARN("fail to construct device config line buf", KR(ret), K(config)); - } - } - if (OB_SUCC(ret) && 0 != STRLEN(config.old_access_info_)) { - if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf), - "&%s", config.old_access_info_))) { - LOG_WARN("fail to construct device config line buf", KR(ret), K(config)); - } - } - if (OB_SUCC(ret) && 0 != STRLEN(config.old_encrypt_info_)) { - if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf), - "&%s%s", ObDeviceConfigParser::OLD_ENCRYPT_INFO, config.old_encrypt_info_))) { - LOG_WARN("fail to construct device config line buf", KR(ret), K(config)); - } - } - if (OB_SUCC(ret) && 0 != STRLEN(config.old_extension_)) { - if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf), - "&%s%s", ObDeviceConfigParser::OLD_EXTENSION, config.old_extension_))) { - LOG_WARN("fail to construct device config line buf", KR(ret), K(config)); - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf), - "&%s%s", ObDeviceConfigParser::STATE, config.state_))) { - LOG_WARN("fail to construct device config line buf", KR(ret), K(config)); - } - } - if (OB_SUCC(ret) && 0 != STRLEN(config.state_info_)) { - if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf), - "&%s%s", ObDeviceConfigParser::STATE_INFO, config.state_info_))) { - LOG_WARN("fail to construct device config line buf", KR(ret), K(config)); - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf), - "&%s%ld", ObDeviceConfigParser::CREATE_TIMESTAMP, config.create_timestamp_))) { - LOG_WARN("fail to construct device config line buf", KR(ret), K(config)); - } else if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf), - "&%s%ld", ObDeviceConfigParser::LAST_CHECK_TIMESTAMP, config.last_check_timestamp_))) { - LOG_WARN("fail to construct device config line buf", KR(ret), K(config)); - } else if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf), - "&%s%lu", ObDeviceConfigParser::OP_ID, config.op_id_))) { - LOG_WARN("fail to construct device config line buf", KR(ret), K(config)); - } else if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf), - "&%s%lu", ObDeviceConfigParser::SUB_OP_ID, config.sub_op_id_))) { - LOG_WARN("fail to construct device config line buf", KR(ret), K(config)); - } else if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf), - "&%s%lu", ObDeviceConfigParser::STORAGE_ID, config.storage_id_))) { - LOG_WARN("fail to construct device config line buf", KR(ret), K(config)); - } else if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf), - "&%s%ld", ObDeviceConfigParser::MAX_IOPS, config.max_iops_))) { - LOG_WARN("fail to construct device config line buf", KR(ret), K(config)); - } else if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf), - "&%s%ld", ObDeviceConfigParser::MAX_BANDWIDTH, config.max_bandwidth_))) { - LOG_WARN("fail to construct device config line buf", KR(ret), K(config)); - } - } - if (OB_SUCC(ret)) { - pret = fprintf(fp, "%s\n", buf); - if (pret <= 0 || pret > MAX_FILE_LINE_LEN) { - ret = OB_IO_ERROR; - LOG_WARN("fail to write manifest", KR(ret), K(pret), K(errno), K(buf)); - } - } - LOG_INFO("finish to write manifest device line", KR(ret), K(buf)); + LOG_INFO("finish to write manifest device", KR(ret), K(buf)); } } LOG_INFO("finish to write device config", KR(ret)); @@ -564,5 +552,24 @@ int ObDeviceManifest::validate_config_checksum_( return ret; } +int ObDeviceManifest::print_file_line_(FILE *fp, const char *fmt, ...) const +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(fp) || OB_ISNULL(fmt)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", KR(ret), KP(fp), KP(fmt)); + } else { + va_list args; + va_start(args, fmt); + int pret = vfprintf(fp, fmt, args); + va_end(args); + if (OB_UNLIKELY((pret <= 0) || (pret >= MAX_FILE_LINE_LEN))) { + ret = OB_IO_ERROR; + LOG_WARN("fail to vfprintf", KR(ret), K(pret), K(errno), KERRMSG); + } + } + return ret; +} + } // namespace share } // namespace oceanbase diff --git a/src/share/object_storage/ob_device_manifest.h b/src/share/object_storage/ob_device_manifest.h index 924971db4..88fd90ab0 100644 --- a/src/share/object_storage/ob_device_manifest.h +++ b/src/share/object_storage/ob_device_manifest.h @@ -106,12 +106,14 @@ public: private: static const char MANIFEST_FILE_NAME[]; static const char HEAD_SECTION[]; - static const char DEVICE_SECTION[]; + static const char DEVICE_BEGIN_SECTION[]; + static const char DEVICE_END_SECTION[]; enum SectionType { SECTION_TYPE_INVLAID, SECTION_TYPE_HEAD, - SECTION_TYPE_DEVICE + SECTION_TYPE_DEVICE_BEGIN, + SECTION_TYPE_DEVICE_END }; static const int64_t MANIFEST_VERSION; @@ -130,7 +132,7 @@ private: int parse_file_(FILE *fp, common::ObIArray &config_arr, HeadSection &head); int parse_section_type_(const char *buf, SectionType &type); int parse_head_section_(const char *buf, HeadSection &head); - int parse_device_section_(char *buf, ObDeviceConfig &device_config); + int parse_device_section_(const char *buf, ObDeviceConfig &device_config); int write_head_(FILE *fp, const HeadSection &head) const; // sort device configs according to used_for_, path_ and endpoint_ int sort_device_configs_(common::ObIArray &config_arr) const; @@ -141,6 +143,7 @@ private: HeadSection &head) const; int validate_config_checksum_(const common::ObIArray &config_arr, const HeadSection &head) const; + int print_file_line_(FILE *fp, const char *fmt, ...) const; private: bool is_inited_; diff --git a/src/share/object_storage/ob_device_manifest_task.cpp b/src/share/object_storage/ob_device_manifest_task.cpp index 4dd1807f3..e477f4f5f 100644 --- a/src/share/object_storage/ob_device_manifest_task.cpp +++ b/src/share/object_storage/ob_device_manifest_task.cpp @@ -497,29 +497,30 @@ int ObDeviceManifestTask::update_device_config( LOG_WARN("fail to get device config", KR(ret), K(device_config_key)); } else if (FALSE_IT(device_config.op_id_ = storage_op_info.op_id_)) { } else if (FALSE_IT(device_config.sub_op_id_ = storage_op_info.sub_op_id_)) { + // Note: must MEMSET zero before STRCPY, because calc checksum depends on sizeof() + } else if (FALSE_IT(MEMSET(device_config.state_, 0, sizeof(device_config.state_)))) { } else if (FALSE_IT(STRCPY(device_config.state_, ObZoneStorageState::get_str(op_type)))) { } else if (ObZoneStorageState::STATE::CHANGING == op_type) { - // reset old_access_info - memset(device_config.old_access_info_, 0, sizeof(device_config.old_access_info_)); - // replace old_access_info with access_info - if (OB_FAIL(ObDeviceConfigParser::convert_access_info_to_old( - device_config.access_info_, device_config.old_access_info_))) { - LOG_WARN("fail to convert access info to old", KR(ret), "access_info", - common::ObSzString(device_config.access_info_)); - } else { - STRCPY(device_config.old_encrypt_info_, device_config.encrypt_info_); - STRCPY(device_config.old_extension_, device_config.extension_); - STRCPY(device_config.access_info_, zone_storage_info.dest_attr_.authorization_); - STRCPY(device_config.extension_, zone_storage_info.dest_attr_.extension_); - device_config.max_iops_ = zone_storage_info.max_iops_; - device_config.max_bandwidth_ = zone_storage_info.max_bandwidth_; - device_config.last_check_timestamp_ = common::ObTimeUtility::fast_current_time(); - char tmp_state_info[OB_MAX_STORAGE_STATE_INFO_LENGTH] = { 0 }; - MEMCPY(tmp_state_info, STORAGE_IS_CONNECTIVE_KEY, STRLEN(STORAGE_IS_CONNECTIVE_KEY)); - MEMCPY(tmp_state_info + STRLEN(tmp_state_info), is_connective ? "true" : "false", - is_connective ? STRLEN("true") : STRLEN("false")); - STRCPY(device_config.state_info_, tmp_state_info); - } + // Note: must MEMSET zero before STRCPY, because calc checksum depends on sizeof() + MEMSET(device_config.old_access_info_, 0, sizeof(device_config.old_access_info_)); + MEMSET(device_config.old_encrypt_info_, 0, sizeof(device_config.old_encrypt_info_)); + MEMSET(device_config.old_extension_, 0, sizeof(device_config.old_extension_)); + STRCPY(device_config.old_access_info_, device_config.access_info_); + STRCPY(device_config.old_encrypt_info_, device_config.encrypt_info_); + STRCPY(device_config.old_extension_, device_config.extension_); + MEMSET(device_config.access_info_, 0, sizeof(device_config.access_info_)); + MEMSET(device_config.extension_, 0, sizeof(device_config.extension_)); + STRCPY(device_config.access_info_, zone_storage_info.dest_attr_.authorization_); + STRCPY(device_config.extension_, zone_storage_info.dest_attr_.extension_); + device_config.max_iops_ = zone_storage_info.max_iops_; + device_config.max_bandwidth_ = zone_storage_info.max_bandwidth_; + device_config.last_check_timestamp_ = common::ObTimeUtility::fast_current_time(); + char tmp_state_info[OB_MAX_STORAGE_STATE_INFO_LENGTH] = { 0 }; + MEMCPY(tmp_state_info, STORAGE_IS_CONNECTIVE_KEY, STRLEN(STORAGE_IS_CONNECTIVE_KEY)); + MEMCPY(tmp_state_info + STRLEN(tmp_state_info), is_connective ? "true" : "false", + is_connective ? STRLEN("true") : STRLEN("false")); + MEMSET(device_config.state_info_, 0, sizeof(device_config.state_info_)); + STRCPY(device_config.state_info_, tmp_state_info); } if (FAILEDx(ObDeviceConfigMgr::get_instance().update_device_config(device_config))) { LOG_WARN("fail to update device config", KR(ret), K(device_config)); diff --git a/unittest/share/device/test_device_config.cpp b/unittest/share/device/test_device_config.cpp index 025bc74c1..b6828f64d 100644 --- a/unittest/share/device/test_device_config.cpp +++ b/unittest/share/device/test_device_config.cpp @@ -31,112 +31,100 @@ class TestDeviceConfig: public ::testing::Test public: TestDeviceConfig() {} virtual ~TestDeviceConfig() {} - virtual void SetUp() - { - char buf_log_id[] = "used_for=LOG&path=oss://antsys-oceanbasebackup/backup_clog_1&endpoint=" - "host=cn-hangzhou-alipay-b.oss-cdn.aliyun-inc.com&access_mode=access_by_id&access_id=a&ac" - "cess_key=b&encrypt_info=encrypt_key=b&extension=null&old_access_mode=access_by_id&old_access" - "_id=c&old_access_key=d&old_encrypt_info=encrypt_key=b&old_extension=null&state=ADDING&state" - "_info=is_connective=true&create_timestamp=1679579590156&last_check_timestamp=1679579590256" - "&op_id=1&sub_op_id=1&storage_id=1&max_iops=0&max_bandwidth=0"; - char buf_log_ram_url[] = "used_for=LOG&path=oss://antsys-oceanbasebackup/backup_clog_2&endpoint" - "=host=cn-hangzhou-alipay-b.oss-cdn.aliyun-inc.com&access_mode=access_by_ram_url&ram_url=xxxx" - "&state=ADDING&state_info=is_connective=true&create_timestamp=1679579590356&last_check_timest" - "amp=1679579590456&op_id=2&sub_op_id=1&storage_id=2&max_iops=0&max_bandwidth=0"; - char buf_data[] = "used_for=DATA&path=oss://antsys-oceanbasebackup/backup_sstable&endpoint=" - "host=cn-hangzhou-alipay-b.oss-cdn.aliyun-inc.com&access_mode=access_by_id&access_id=a&ac" - "cess_key=b&encrypt_info=encrypt_key=b&extension=null&old_access_mode=access_by_id&old_access" - "_id=c&old_access_key=d&old_encrypt_info=encrypt_key=b&old_extension=null&state=ADDING&state" - "_info=is_connective=false&create_timestamp=1679579590556&last_check_timestamp=1679579590656" - "&op_id=3&sub_op_id=1&storage_id=3&max_iops=0&max_bandwidth=0"; - memcpy(buf_log_id_, buf_log_id, sizeof(buf_log_id)); - memcpy(buf_log_ram_url_, buf_log_ram_url, sizeof(buf_log_ram_url)); - memcpy(buf_data_, buf_data, sizeof(buf_data)); - } + virtual void SetUp() {} virtual void TearDown() {} static void SetUpTestCase() {} static void TearDownTestCase() {} private: DISALLOW_COPY_AND_ASSIGN(TestDeviceConfig); -protected: - char buf_log_id_[16384]; - char buf_log_ram_url_[16384]; - char buf_data_[16384]; }; -TEST_F(TestDeviceConfig, DISABLED_test_device_config_parser) +TEST_F(TestDeviceConfig, parser_and_dump_load_and_get) { + // 1. test device config parser ObDeviceConfig config; - ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_one_device_config(buf_log_id_, config)); + const char *used_for_str = "used_for=LOG"; + const char *path_str = "path=oss://bucket-name/root_dir"; + const char *endpoint_str = "endpoint=host=cn-xxx.com"; + const char *access_info_str = "access_info=access_id=aaa&encrypt_key=bbb"; + const char *extension_str = "extension=appid=123&checksum_type=crc32"; + const char *state_str = "state=ADDED"; + const char *state_info_str = "state_info=is_connective=true"; + const char *create_timestamp_str = "create_timestamp=1679579590156"; + const char *last_check_timestamp_str = "last_check_timestamp=1679579590256"; + const char *op_id_str = "op_id=1"; + const char *sub_op_id_str = "sub_op_id=2"; + const char *storage_id_str = "storage_id=1"; + const char *max_iops_str = "max_iops=10000"; + const char *max_bandwidth_str = "max_bandwidth=1024000000"; + ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(used_for_str, config)); + ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(path_str, config)); + ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(endpoint_str, config)); + ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(access_info_str, config)); + ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(extension_str, config)); + ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(state_str, config)); + ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(state_info_str, config)); + ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(create_timestamp_str, config)); + ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(last_check_timestamp_str, config)); + ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(op_id_str, config)); + ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(sub_op_id_str, config)); + ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(storage_id_str, config)); + ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(max_iops_str, config)); + ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(max_bandwidth_str, config)); ASSERT_EQ(0, STRCMP(config.used_for_, "LOG")); - ASSERT_EQ(0, STRCMP(config.path_, "oss://antsys-oceanbasebackup/backup_clog_1")); - ASSERT_EQ(0, STRCMP(config.endpoint_, "host=cn-hangzhou-alipay-b.oss-cdn.aliyun-inc.com")); - ASSERT_EQ(0, STRCMP(config.access_info_, "access_mode=access_by_id&access_id=a&access_key=b")); - ASSERT_EQ(0, STRCMP(config.encrypt_info_, "encrypt_key=b")); - ASSERT_EQ(0, STRCMP(config.extension_, "null")); - ASSERT_EQ(0, STRCMP(config.old_access_info_, "old_access_mode=access_by_id&old_access_id=c&old_access_key=d")); - ASSERT_EQ(0, STRCMP(config.old_encrypt_info_, "encrypt_key=b")); - ASSERT_EQ(0, STRCMP(config.old_extension_, "null")); - ASSERT_EQ(0, STRCMP(config.state_, "ADDING")); + ASSERT_EQ(0, STRCMP(config.path_, "oss://bucket-name/root_dir")); + ASSERT_EQ(0, STRCMP(config.endpoint_, "host=cn-xxx.com")); + ASSERT_EQ(0, STRCMP(config.access_info_, "access_id=aaa&encrypt_key=bbb")); + ASSERT_EQ(0, STRCMP(config.extension_, "appid=123&checksum_type=crc32")); + ASSERT_EQ(0, STRCMP(config.state_, "ADDED")); ASSERT_EQ(0, STRCMP(config.state_info_, "is_connective=true")); ASSERT_EQ(1679579590156, config.create_timestamp_); ASSERT_EQ(1679579590256, config.last_check_timestamp_); ASSERT_EQ(1, config.op_id_); - ASSERT_EQ(1, config.sub_op_id_); + ASSERT_EQ(2, config.sub_op_id_); ASSERT_EQ(1, config.storage_id_); - ASSERT_EQ(0, config.max_iops_); - ASSERT_EQ(0, config.max_bandwidth_); + ASSERT_EQ(10000, config.max_iops_); + ASSERT_EQ(1024000000, config.max_bandwidth_); - ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_one_device_config(buf_log_ram_url_, config)); - ASSERT_EQ(0, STRCMP(config.used_for_, "LOG")); - ASSERT_EQ(0, STRCMP(config.path_, "oss://antsys-oceanbasebackup/backup_clog_2")); - ASSERT_EQ(0, STRCMP(config.endpoint_, "host=cn-hangzhou-alipay-b.oss-cdn.aliyun-inc.com")); - ASSERT_EQ(0, STRCMP(config.access_info_, "access_mode=access_by_ram_url&ram_url=xxxx")); - ASSERT_EQ(0, STRCMP(config.state_, "ADDING")); - ASSERT_EQ(0, STRCMP(config.state_info_, "is_connective=true")); - ASSERT_EQ(1679579590356, config.create_timestamp_); - ASSERT_EQ(1679579590456, config.last_check_timestamp_); - ASSERT_EQ(2, config.op_id_); - ASSERT_EQ(1, config.sub_op_id_); - ASSERT_EQ(2, config.storage_id_); - ASSERT_EQ(0, config.max_iops_); - ASSERT_EQ(0, config.max_bandwidth_); -} -TEST_F(TestDeviceConfig, DISABLED_test_device_manifest) -{ + // 2. test device manifest dump2file and load ObDeviceManifest device_manifest; ObArray dump_configs; ObArray load_configs; - ObDeviceConfig config; - ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_one_device_config(buf_log_id_, config)); ASSERT_EQ(OB_SUCCESS, dump_configs.push_back(config)); - ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_one_device_config(buf_log_ram_url_, config)); - ASSERT_EQ(OB_SUCCESS, dump_configs.push_back(config)); - ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_one_device_config(buf_data_, config)); - ASSERT_EQ(OB_SUCCESS, dump_configs.push_back(config)); - + ObDeviceConfig config2 = config; + MEMSET(config2.used_for_, 0, sizeof(config2.used_for_)); + STRCPY(config2.used_for_, "DATA"); + MEMSET(config2.old_access_info_, 0, sizeof(config2.old_access_info_)); + STRCPY(config2.old_access_info_, "access_id=aaa&encrypt_key=bbb"); + MEMSET(config2.access_info_, 0, sizeof(config2.access_info_)); + STRCPY(config2.access_info_, "access_id=ccccc&encrypt_key=ddddd"); + MEMSET(config2.old_extension_, 0, sizeof(config2.old_extension_)); + STRCPY(config2.old_extension_, "appid=123&checksum_type=crc32"); + MEMSET(config2.extension_, 0, sizeof(config2.extension_)); + STRCPY(config2.extension_, "appid=456&checksum_type=md5"); + config2.op_id_ = 3; + config2.sub_op_id_ = 4; + config2.storage_id_ = 2; + ASSERT_EQ(OB_SUCCESS, dump_configs.push_back(config2)); ObDeviceManifest::HeadSection dump_head; - dump_head.version_ = 1; - dump_head.device_num_ = 3; + dump_head.device_num_ = 2; dump_head.modify_timestamp_us_ = ObTimeUtil::fast_current_time(); dump_head.last_op_id_ = 3; - dump_head.last_sub_op_id_ = 1; + dump_head.last_sub_op_id_ = 4; ObDeviceManifest::HeadSection load_head; const char *data_dir = "./TestDeviceConfigDir"; ASSERT_EQ(OB_SUCCESS, FileDirectoryUtils::create_directory(data_dir)); - ASSERT_EQ(OB_SUCCESS, device_manifest.init(data_dir)); ASSERT_EQ(OB_SUCCESS, device_manifest.dump2file(dump_configs, dump_head)); ASSERT_EQ(OB_SUCCESS, device_manifest.load(load_configs, load_head)); // sort by used_for, path and endpoint ObArray sorted_dump_configs; - ASSERT_EQ(OB_SUCCESS, sorted_dump_configs.push_back(dump_configs[2])); - ASSERT_EQ(OB_SUCCESS, sorted_dump_configs.push_back(dump_configs[0])); ASSERT_EQ(OB_SUCCESS, sorted_dump_configs.push_back(dump_configs[1])); - ASSERT_EQ(3, load_configs.count()); + ASSERT_EQ(OB_SUCCESS, sorted_dump_configs.push_back(dump_configs[0])); + ASSERT_EQ(2, load_configs.count()); for (int64_t i = 0; i < load_configs.count(); ++i) { ASSERT_EQ(0, STRCMP(sorted_dump_configs[i].used_for_, load_configs[i].used_for_)); ASSERT_EQ(0, STRCMP(sorted_dump_configs[i].path_, load_configs[i].path_)); @@ -165,59 +153,58 @@ TEST_F(TestDeviceConfig, DISABLED_test_device_manifest) ASSERT_EQ(dump_head.last_op_id_, load_head.last_op_id_); ASSERT_EQ(dump_head.last_sub_op_id_, load_head.last_sub_op_id_); - // ASSERT_EQ(OB_SUCCESS, FileDirectoryUtils::delete_directory_rec(data_dir)); -} - -TEST_F(TestDeviceConfig, DISABLED_test_device_config_mgr) -{ - char data_dir[MAX_PATH_SIZE] = "./TestDeviceConfigDir"; - // char shared_storage_info[share::OB_MAX_BACKUP_DEST_LENGTH] = "file://./test_bukcet/"; - // ASSERT_EQ(OB_SUCCESS, FileDirectoryUtils::create_directory(data_dir)); + // 3. test device config mgr ObDeviceConfigMgr &config_mgr = ObDeviceConfigMgr::get_instance(); ASSERT_EQ(OB_SUCCESS, config_mgr.init(data_dir)); ASSERT_EQ(OB_SUCCESS, config_mgr.load_configs()); - // test get_all_device_configs - ObArray configs; - ObDeviceConfig config; - ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_one_device_config(buf_data_, config)); - ASSERT_EQ(OB_SUCCESS, configs.push_back(config)); - ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_one_device_config(buf_log_id_, config)); - ASSERT_EQ(OB_SUCCESS, configs.push_back(config)); - ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_one_device_config(buf_log_ram_url_, config)); - ASSERT_EQ(OB_SUCCESS, configs.push_back(config)); ObArray all_configs; ASSERT_EQ(OB_SUCCESS, config_mgr.get_all_device_configs(all_configs)); - ASSERT_EQ(3, all_configs.count()); - - // test get_device_configs - char used_for_data[] = "DATA"; - ObArray used_for_configs; - ASSERT_EQ(OB_SUCCESS, config_mgr.get_device_configs(used_for_data, used_for_configs)); - ASSERT_EQ(1, used_for_configs.count()); - ASSERT_EQ(0, STRCMP(configs[0].used_for_, used_for_configs[0].used_for_)); - ASSERT_EQ(0, STRCMP(configs[0].path_, used_for_configs[0].path_)); - ASSERT_EQ(0, STRCMP(configs[0].endpoint_, used_for_configs[0].endpoint_)); - ASSERT_EQ(0, STRCMP(configs[0].access_info_, used_for_configs[0].access_info_)); - ASSERT_EQ(0, STRCMP(configs[0].encrypt_info_, used_for_configs[0].encrypt_info_)); - ASSERT_EQ(0, STRCMP(configs[0].extension_, used_for_configs[0].extension_)); - ASSERT_EQ(0, STRCMP(configs[0].old_access_info_, used_for_configs[0].old_access_info_)); - ASSERT_EQ(0, STRCMP(configs[0].old_encrypt_info_, used_for_configs[0].old_encrypt_info_)); - ASSERT_EQ(0, STRCMP(configs[0].old_extension_, used_for_configs[0].old_extension_)); - ASSERT_EQ(0, STRCMP(configs[0].state_, used_for_configs[0].state_)); - ASSERT_EQ(0, STRCMP(configs[0].state_info_, used_for_configs[0].state_info_)); - ASSERT_EQ(configs[0].create_timestamp_, used_for_configs[0].create_timestamp_); - ASSERT_EQ(configs[0].last_check_timestamp_, used_for_configs[0].last_check_timestamp_); - ASSERT_EQ(configs[0].op_id_, used_for_configs[0].op_id_); - ASSERT_EQ(configs[0].sub_op_id_, used_for_configs[0].sub_op_id_); - ASSERT_EQ(configs[0].storage_id_, used_for_configs[0].storage_id_); - ASSERT_EQ(configs[0].max_iops_, used_for_configs[0].max_iops_); - ASSERT_EQ(configs[0].max_bandwidth_, used_for_configs[0].max_bandwidth_); - char used_for_log[] = "LOG"; - // used_for_configs.reuse(); - ASSERT_EQ(OB_SUCCESS, config_mgr.get_device_configs(used_for_log, used_for_configs)); - ASSERT_EQ(2, used_for_configs.count()); + ASSERT_EQ(2, all_configs.count()); + // test get_device_config + ObDeviceConfig data_device_config; + ASSERT_EQ(OB_SUCCESS, config_mgr.get_device_config(ObStorageUsedType::TYPE::USED_TYPE_DATA, + data_device_config)); + ASSERT_EQ(0, STRCMP(config2.used_for_, data_device_config.used_for_)); + ASSERT_EQ(0, STRCMP(config2.path_, data_device_config.path_)); + ASSERT_EQ(0, STRCMP(config2.endpoint_, data_device_config.endpoint_)); + ASSERT_EQ(0, STRCMP(config2.access_info_, data_device_config.access_info_)); + ASSERT_EQ(0, STRCMP(config2.encrypt_info_, data_device_config.encrypt_info_)); + ASSERT_EQ(0, STRCMP(config2.extension_, data_device_config.extension_)); + ASSERT_EQ(0, STRCMP(config2.old_access_info_, data_device_config.old_access_info_)); + ASSERT_EQ(0, STRCMP(config2.old_encrypt_info_, data_device_config.old_encrypt_info_)); + ASSERT_EQ(0, STRCMP(config2.old_extension_, data_device_config.old_extension_)); + ASSERT_EQ(0, STRCMP(config2.state_, data_device_config.state_)); + ASSERT_EQ(0, STRCMP(config2.state_info_, data_device_config.state_info_)); + ASSERT_EQ(config2.create_timestamp_, data_device_config.create_timestamp_); + ASSERT_EQ(config2.last_check_timestamp_, data_device_config.last_check_timestamp_); + ASSERT_EQ(config2.op_id_, data_device_config.op_id_); + ASSERT_EQ(config2.sub_op_id_, data_device_config.sub_op_id_); + ASSERT_EQ(config2.storage_id_, data_device_config.storage_id_); + ASSERT_EQ(config2.max_iops_, data_device_config.max_iops_); + ASSERT_EQ(config2.max_bandwidth_, data_device_config.max_bandwidth_); + ObDeviceConfig log_device_config; + ASSERT_EQ(OB_SUCCESS, config_mgr.get_device_config(ObStorageUsedType::TYPE::USED_TYPE_LOG, + log_device_config)); + ASSERT_EQ(0, STRCMP(config.used_for_, log_device_config.used_for_)); + ASSERT_EQ(0, STRCMP(config.path_, log_device_config.path_)); + ASSERT_EQ(0, STRCMP(config.endpoint_, log_device_config.endpoint_)); + ASSERT_EQ(0, STRCMP(config.access_info_, log_device_config.access_info_)); + ASSERT_EQ(0, STRCMP(config.encrypt_info_, log_device_config.encrypt_info_)); + ASSERT_EQ(0, STRCMP(config.extension_, log_device_config.extension_)); + ASSERT_EQ(0, STRCMP(config.old_access_info_, log_device_config.old_access_info_)); + ASSERT_EQ(0, STRCMP(config.old_encrypt_info_, log_device_config.old_encrypt_info_)); + ASSERT_EQ(0, STRCMP(config.old_extension_, log_device_config.old_extension_)); + ASSERT_EQ(0, STRCMP(config.state_, log_device_config.state_)); + ASSERT_EQ(0, STRCMP(config.state_info_, log_device_config.state_info_)); + ASSERT_EQ(config.create_timestamp_, log_device_config.create_timestamp_); + ASSERT_EQ(config.last_check_timestamp_, log_device_config.last_check_timestamp_); + ASSERT_EQ(config.op_id_, log_device_config.op_id_); + ASSERT_EQ(config.sub_op_id_, log_device_config.sub_op_id_); + ASSERT_EQ(config.storage_id_, log_device_config.storage_id_); + ASSERT_EQ(config.max_iops_, log_device_config.max_iops_); + ASSERT_EQ(config.max_bandwidth_, log_device_config.max_bandwidth_); // test get_last_op_id and get_last_sub_op_id uint64_t last_op_id = UINT64_MAX; @@ -225,17 +212,20 @@ TEST_F(TestDeviceConfig, DISABLED_test_device_config_mgr) ASSERT_EQ(OB_SUCCESS, config_mgr.get_last_op_id(last_op_id)); ASSERT_EQ(OB_SUCCESS, config_mgr.get_last_sub_op_id(last_sub_op_id)); ASSERT_EQ(3, last_op_id); - ASSERT_EQ(1, last_sub_op_id); + ASSERT_EQ(4, last_sub_op_id); // test is_op_done - bool is_done_3_1 = false; - config_mgr.is_op_done(3, 1, is_done_3_1); - ASSERT_TRUE(is_done_3_1); - bool is_done_1_1 = false; - config_mgr.is_op_done(1, 1, is_done_1_1); - ASSERT_TRUE(is_done_1_1); + bool is_done_3_4 = false; + config_mgr.is_op_done(3, 4, is_done_3_4); + ASSERT_TRUE(is_done_3_4); + bool is_done_1_2 = false; + config_mgr.is_op_done(1, 2, is_done_1_2); + ASSERT_TRUE(is_done_1_2); + bool is_done_3_5 = false; + config_mgr.is_op_done(3, 5, is_done_3_5); + ASSERT_FALSE(is_done_3_5); - // ASSERT_EQ(OB_SUCCESS, FileDirectoryUtils::delete_directory_rec(data_dir)); + ASSERT_EQ(OB_SUCCESS, FileDirectoryUtils::delete_directory_rec(data_dir)); } } // namespace unittest @@ -245,7 +235,7 @@ int main(int argc, char **argv) { oceanbase::common::FileDirectoryUtils::delete_directory_rec("./TestDeviceConfigDir"); system("rm -f test_device_config.log"); - OB_LOGGER.set_file_name("test_device_config.log"); + OB_LOGGER.set_file_name("test_device_config.log", true); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS();