fix shared storage manifest about extension with multiple fields
This commit is contained in:
obdev
ob-robot
parent
f062d160be
commit
ca51217d50
@ -90,7 +90,7 @@ int ObDeviceConfigMgr::load_configs()
|
||||
ret = OB_NOT_INIT;
|
||||
LOG_WARN("ObDeviceConfigMgr not init", KR(ret));
|
||||
} else if (OB_FAIL(device_manifest_.load(device_configs, head_))) {
|
||||
if (OB_FILE_NOT_EXIST == ret) { // first start of observer
|
||||
if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret) { // first start of observer
|
||||
LOG_INFO("device manifest file does not exist", KR(ret));
|
||||
ret = OB_SUCCESS;
|
||||
} else {
|
||||
|
||||
@ -23,17 +23,10 @@ using namespace common;
|
||||
const char ObDeviceConfigParser::USED_FOR[] = "used_for=";
|
||||
const char ObDeviceConfigParser::PATH[] = "path=";
|
||||
const char ObDeviceConfigParser::ENDPOINT[] = "endpoint=";
|
||||
const char ObDeviceConfigParser::ACCESS_MODE[] = "access_mode=";
|
||||
const char ObDeviceConfigParser::ACCESS_ID[] = "access_id=";
|
||||
const char ObDeviceConfigParser::ACCESS_KEY[] = "access_key=";
|
||||
const char ObDeviceConfigParser::ENCRYPT_KEY[] = "encrypt_key=";
|
||||
const char ObDeviceConfigParser::ACCESS_INFO[] = "access_info=";
|
||||
const char ObDeviceConfigParser::ENCRYPT_INFO[] = "encrypt_info=";
|
||||
const char ObDeviceConfigParser::EXTENSION[] = "extension=";
|
||||
const char ObDeviceConfigParser::OLD_ACCESS_MODE[] = "old_access_mode=";
|
||||
const char ObDeviceConfigParser::OLD_ACCESS_ID[] = "old_access_id=";
|
||||
|
||||
const char ObDeviceConfigParser::OLD_ACCESS_KEY[] = "old_access_key=";
|
||||
const char ObDeviceConfigParser::OLD_ENCRYPT_KEY[] = "old_encrypt_key=";
|
||||
const char ObDeviceConfigParser::OLD_ACCESS_INFO[] = "old_access_info=";
|
||||
const char ObDeviceConfigParser::OLD_ENCRYPT_INFO[] = "old_encrypt_info=";
|
||||
const char ObDeviceConfigParser::OLD_EXTENSION[] = "old_extension=";
|
||||
const char ObDeviceConfigParser::RAM_URL[] = "ram_url=";
|
||||
@ -50,94 +43,86 @@ const char ObDeviceConfigParser::MAX_BANDWIDTH[] = "max_bandwidth=";
|
||||
const char ObDeviceConfigParser::INVALID_OP_ID[] = "-1"; // because op_id in __all_zone_storage is int(20), but in DeviceConfig is uint64_t,
|
||||
// op_id is init UINT64_MAX, so when ObDeviceManifest::write_head_ to manifest op_id is change as -1
|
||||
|
||||
int ObDeviceConfigParser::parse_one_device_config(char *opt_str, ObDeviceConfig &config)
|
||||
int ObDeviceConfigParser::parse_device_config_field(const char *buf, ObDeviceConfig &device_config)
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
char *token = opt_str;
|
||||
char *saved_ptr = NULL;
|
||||
config.reset();
|
||||
|
||||
for (char *str = token; OB_SUCC(ret); str = NULL) {
|
||||
token = ::strtok_r(str, "&", &saved_ptr);
|
||||
if (OB_ISNULL(token)) {
|
||||
break;
|
||||
} else {
|
||||
// e.g., used_for=clog
|
||||
if (OB_SUCC(parse_common_field_(token, config.used_for_, FieldType::TYPE_CHAR_ARRAY, USED_FOR, OB_MAX_STORAGE_USED_FOR_LENGTH))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse used_for", KR(ret), K(token));
|
||||
// e.g., path=oss://xxx/yyy
|
||||
} else if (OB_SUCC(parse_common_field_(token, config.path_, FieldType::TYPE_CHAR_ARRAY, PATH, OB_MAX_BACKUP_DEST_LENGTH))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse path", KR(ret), K(token));
|
||||
// e.g., endpoint=xxx
|
||||
} else if (OB_SUCC(parse_common_field_(token, config.endpoint_, FieldType::TYPE_CHAR_ARRAY, ENDPOINT, OB_MAX_BACKUP_ENDPOINT_LENGTH))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse endpoint", KR(ret), K(token));
|
||||
// e.g., access_mode=access_by_id or access_id=xxx or ram_url=xxx
|
||||
} else if (OB_SUCC(parse_access_info_(token, config.access_info_))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse access_info", KR(ret), K(token));
|
||||
// e.g., encrypt_info=xxx
|
||||
} else if (OB_SUCC(parse_encrypt_info_(token, config.encrypt_info_))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse encrypt_info", KR(ret), K(token));
|
||||
// e.g., extension=xxx
|
||||
} else if (OB_SUCC(parse_extension_(token, config.extension_))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse extension", KR(ret), K(token));
|
||||
// e.g., old_access_mode=access_by_id or old_access_id=xxx
|
||||
} else if (OB_SUCC(parse_access_info_(token, config.old_access_info_, true/*is_old*/))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse access_info", KR(ret), K(token));
|
||||
// e.g., old_encrypt_info=xxx
|
||||
} else if (OB_SUCC(parse_encrypt_info_(token, config.old_encrypt_info_, true/*is_old*/))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse encrypt_info", KR(ret), K(token));
|
||||
// e.g., old_extension=xxx
|
||||
} else if (OB_SUCC(parse_extension_(token, config.old_extension_, true/*is_old*/))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse extension", KR(ret), K(token));
|
||||
// e.g., state=ADDING
|
||||
} else if (OB_SUCC(parse_common_field_(token, config.state_, FieldType::TYPE_CHAR_ARRAY, STATE, OB_MAX_STORAGE_STATE_LENGTH))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse state", KR(ret), K(token));
|
||||
// e.g., state_info=xxx
|
||||
} else if (OB_SUCC(parse_common_field_(token, config.state_info_, FieldType::TYPE_CHAR_ARRAY, STATE_INFO, OB_MAX_STORAGE_STATE_INFO_LENGTH))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse state_info", KR(ret), K(token));
|
||||
// e.g., create_timestamp=1679579590457
|
||||
} else if (OB_SUCC(parse_common_field_(token, &(config.create_timestamp_), FieldType::TYPE_INT, CREATE_TIMESTAMP))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse create_timestamp_", KR(ret), K(token));
|
||||
// e.g., last_check_timestamp=1679579590457
|
||||
} else if (OB_SUCC(parse_common_field_(token, &(config.last_check_timestamp_), FieldType::TYPE_INT, LAST_CHECK_TIMESTAMP))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse last_check_timestamp", KR(ret), K(token));
|
||||
// e.g., op_id=2
|
||||
} else if (OB_SUCC(parse_common_field_(token, &(config.op_id_), FieldType::TYPE_UINT, OP_ID))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse op_id", KR(ret), K(token));
|
||||
// e.g., sub_op_id=1
|
||||
} else if (OB_SUCC(parse_common_field_(token, &(config.sub_op_id_), FieldType::TYPE_UINT, SUB_OP_ID))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse sub_op_id", KR(ret), K(token));
|
||||
// e.g., storage_id=10
|
||||
} else if (OB_SUCC(parse_common_field_(token, &(config.storage_id_), FieldType::TYPE_UINT, STORAGE_ID))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse storage_id", KR(ret), K(token));
|
||||
// e.g., iops=10000
|
||||
} else if (OB_SUCC(parse_common_field_(token, &(config.max_iops_), FieldType::TYPE_INT, MAX_IOPS))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse iops", KR(ret), K(token));
|
||||
// e.g., bandwidth=1000000000
|
||||
} else if (OB_SUCC(parse_common_field_(token, &(config.max_bandwidth_), FieldType::TYPE_INT, MAX_BANDWIDTH))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse bandwidth", KR(ret), K(token));
|
||||
} else {
|
||||
FLOG_WARN("all item not match, unknown token", KR(ret), K(token));
|
||||
}
|
||||
}
|
||||
if (OB_ISNULL(buf)) {
|
||||
ret = OB_INVALID_ARGUMENT;
|
||||
LOG_WARN("buf is nullptr", KR(ret));
|
||||
// e.g., used_for=clog
|
||||
} else if (OB_SUCC(parse_common_field_(buf, device_config.used_for_, FieldType::TYPE_CHAR_ARRAY, USED_FOR, OB_MAX_STORAGE_USED_FOR_LENGTH))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse used_for", KR(ret), K(buf));
|
||||
// e.g., path=oss://xxx/yyy
|
||||
} else if (OB_SUCC(parse_common_field_(buf, device_config.path_, FieldType::TYPE_CHAR_ARRAY, PATH, OB_MAX_BACKUP_DEST_LENGTH))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse path", KR(ret), K(buf));
|
||||
// e.g., endpoint=xxx
|
||||
} else if (OB_SUCC(parse_common_field_(buf, device_config.endpoint_, FieldType::TYPE_CHAR_ARRAY, ENDPOINT, OB_MAX_BACKUP_ENDPOINT_LENGTH))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse endpoint", KR(ret), K(buf));
|
||||
// e.g., access_info=access_id=xxx&encrypt_key=xxx
|
||||
} else if (OB_SUCC(parse_access_info_(buf, device_config.access_info_))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse access_info", KR(ret), K(buf));
|
||||
// e.g., encrypt_info=xxx
|
||||
} else if (OB_SUCC(parse_encrypt_info_(buf, device_config.encrypt_info_))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse encrypt_info", KR(ret), K(buf));
|
||||
// e.g., extension=xxx
|
||||
} else if (OB_SUCC(parse_extension_(buf, device_config.extension_))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse extension", KR(ret), K(buf));
|
||||
// e.g., old_access_info=access_id=xxx&encrypt_key=xxx
|
||||
} else if (OB_SUCC(parse_access_info_(buf, device_config.old_access_info_, true/*is_old*/))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse access_info", KR(ret), K(buf));
|
||||
// e.g., old_encrypt_info=xxx
|
||||
} else if (OB_SUCC(parse_encrypt_info_(buf, device_config.old_encrypt_info_, true/*is_old*/))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse encrypt_info", KR(ret), K(buf));
|
||||
// e.g., old_extension=xxx
|
||||
} else if (OB_SUCC(parse_extension_(buf, device_config.old_extension_, true/*is_old*/))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse extension", KR(ret), K(buf));
|
||||
// e.g., state=ADDING
|
||||
} else if (OB_SUCC(parse_common_field_(buf, device_config.state_, FieldType::TYPE_CHAR_ARRAY, STATE, OB_MAX_STORAGE_STATE_LENGTH))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse state", KR(ret), K(buf));
|
||||
// e.g., state_info=xxx
|
||||
} else if (OB_SUCC(parse_common_field_(buf, device_config.state_info_, FieldType::TYPE_CHAR_ARRAY, STATE_INFO, OB_MAX_STORAGE_STATE_INFO_LENGTH))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse state_info", KR(ret), K(buf));
|
||||
// e.g., create_timestamp=1679579590457
|
||||
} else if (OB_SUCC(parse_common_field_(buf, &(device_config.create_timestamp_), FieldType::TYPE_INT, CREATE_TIMESTAMP))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse create_timestamp_", KR(ret), K(buf));
|
||||
// e.g., last_check_timestamp=1679579590457
|
||||
} else if (OB_SUCC(parse_common_field_(buf, &(device_config.last_check_timestamp_), FieldType::TYPE_INT, LAST_CHECK_TIMESTAMP))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse last_check_timestamp", KR(ret), K(buf));
|
||||
// e.g., op_id=2
|
||||
} else if (OB_SUCC(parse_common_field_(buf, &(device_config.op_id_), FieldType::TYPE_UINT, OP_ID))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse op_id", KR(ret), K(buf));
|
||||
// e.g., sub_op_id=1
|
||||
} else if (OB_SUCC(parse_common_field_(buf, &(device_config.sub_op_id_), FieldType::TYPE_UINT, SUB_OP_ID))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse sub_op_id", KR(ret), K(buf));
|
||||
// e.g., storage_id=10
|
||||
} else if (OB_SUCC(parse_common_field_(buf, &(device_config.storage_id_), FieldType::TYPE_UINT, STORAGE_ID))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse storage_id", KR(ret), K(buf));
|
||||
// e.g., iops=10000
|
||||
} else if (OB_SUCC(parse_common_field_(buf, &(device_config.max_iops_), FieldType::TYPE_INT, MAX_IOPS))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse iops", KR(ret), K(buf));
|
||||
// e.g., bandwidth=1000000000
|
||||
} else if (OB_SUCC(parse_common_field_(buf, &(device_config.max_bandwidth_), FieldType::TYPE_INT, MAX_BANDWIDTH))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse bandwidth", KR(ret), K(buf));
|
||||
} else {
|
||||
LOG_WARN("all item not match", KR(ret), K(buf));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
@ -178,43 +163,19 @@ int ObDeviceConfigParser::parse_access_info_(
|
||||
const bool is_old)
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
#ifndef OB_BUILD_TDE_SECURITY
|
||||
const char *access_info_keys[] = {ACCESS_MODE, ACCESS_ID, ACCESS_KEY, RAM_URL};
|
||||
const char *old_access_info_keys[] = {OLD_ACCESS_MODE, OLD_ACCESS_ID, OLD_ACCESS_KEY, OLD_RAM_URL};
|
||||
#else
|
||||
const char *access_info_keys[] = {ACCESS_MODE, ACCESS_ID, ENCRYPT_KEY, RAM_URL};
|
||||
const char *old_access_info_keys[] = {OLD_ACCESS_MODE, OLD_ACCESS_ID, OLD_ENCRYPT_KEY, OLD_RAM_URL};
|
||||
#endif
|
||||
STATIC_ASSERT(ARRAYSIZEOF(access_info_keys) == ARRAYSIZEOF(old_access_info_keys), "key count mismatch");
|
||||
const int64_t key_cnt = ARRAYSIZEOF(access_info_keys);
|
||||
const char *key_name = nullptr;
|
||||
bool is_item_match = false;
|
||||
|
||||
if (OB_ISNULL(token) || OB_ISNULL(access_info)) {
|
||||
if (OB_ISNULL(token)) {
|
||||
ret = OB_INVALID_ARGUMENT;
|
||||
LOG_WARN("invalid argument", KR(ret), KP(token), KP(access_info));
|
||||
}
|
||||
|
||||
for (int64_t i = 0; OB_SUCC(ret) && (i < key_cnt); ++i) {
|
||||
key_name = is_old ? old_access_info_keys[i] : access_info_keys[i];
|
||||
if (0 == STRNCMP(key_name, token, STRLEN(key_name))) {
|
||||
const int64_t access_info_len = STRLEN(access_info);
|
||||
const char *value_str = token + STRLEN(key_name);
|
||||
if (0 == STRLEN(value_str)) {
|
||||
ret = OB_INVALID_ARGUMENT;
|
||||
LOG_WARN("value is empty", KR(ret), K(key_name), K(token));
|
||||
} else if (0 == access_info_len) { // first field of access_info
|
||||
STRCPY(access_info, token);
|
||||
} else { // not first field of access_info, need to concat '&'
|
||||
access_info[access_info_len] = '&';
|
||||
STRCPY(access_info + access_info_len + 1, token);
|
||||
}
|
||||
is_item_match = true;
|
||||
break;
|
||||
LOG_WARN("invalid argument", KR(ret), KP(token));
|
||||
} else if (is_old) { // old_access_info=xxx
|
||||
if (OB_FAIL(parse_config_type_char_array(OLD_ACCESS_INFO, token, OB_MAX_BACKUP_AUTHORIZATION_LENGTH,
|
||||
access_info)) && (OB_ITEM_NOT_MATCH != ret)) {
|
||||
LOG_WARN("fail to parse_config_type_char_array", KR(ret), K(OLD_ENCRYPT_INFO), K(token));
|
||||
}
|
||||
} else { // access_info=xxx
|
||||
if (OB_FAIL(parse_config_type_char_array(ACCESS_INFO, token, OB_MAX_BACKUP_AUTHORIZATION_LENGTH,
|
||||
access_info)) && (OB_ITEM_NOT_MATCH != ret)) {
|
||||
LOG_WARN("fail to parse_config_type_char_array", KR(ret), K(ENCRYPT_INFO), K(token));
|
||||
}
|
||||
}
|
||||
if (OB_SUCC(ret) && !is_item_match) {
|
||||
ret = OB_ITEM_NOT_MATCH;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
@ -282,10 +243,6 @@ int ObDeviceConfigParser::parse_config_type_char_array(
|
||||
ret = OB_INVALID_ARGUMENT;
|
||||
LOG_WARN("value str is too long", KR(ret), K(key_name), "value_len",
|
||||
STRLEN(value_str), K(max_value_len));
|
||||
// when restart server, because manifest's head.shared_storage_info_ is '\0', so head.shared_storage_info_ must be overwritten
|
||||
// TODO:当沧溟新的bootstrap共享存储集群的代码合入了后，这里删除掉 @xiaotao.ht
|
||||
} else if (STRLEN(value_str) == 0) {
|
||||
MEMSET(value, 0, max_value_len);
|
||||
} else {
|
||||
STRCPY(value, value_str);
|
||||
}
|
||||
@ -367,60 +324,5 @@ int ObDeviceConfigParser::parse_config_type_uint(
|
||||
return ret;
|
||||
}
|
||||
|
||||
// 1. convert from "access_mode=access_by_id&access_id=xxx" to
|
||||
// "old_access_mode=access_by_id&old_access_id=xxx"
|
||||
// 2. convert from "access_mode=access_by_ram_url&ram_url=xxx" to
|
||||
// "old_access_mode=access_by_ram_url&old_ram_url=xxx"
|
||||
int ObDeviceConfigParser::convert_access_info_to_old(char *access_info, char *old_access_info)
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
#ifndef OB_BUILD_TDE_SECURITY
|
||||
const char *access_info_keys[] = {ACCESS_MODE, ACCESS_ID, ACCESS_KEY, RAM_URL};
|
||||
#else
|
||||
const char *access_info_keys[] = {ACCESS_MODE, ACCESS_ID, ENCRYPT_KEY, ACCESS_KEY, RAM_URL};
|
||||
#endif
|
||||
const int64_t key_cnt = ARRAYSIZEOF(access_info_keys);
|
||||
const char *key_name = nullptr;
|
||||
char *token = access_info;
|
||||
char *saved_ptr = NULL;
|
||||
if (OB_ISNULL(access_info) || OB_ISNULL(old_access_info)) {
|
||||
ret = OB_INVALID_ARGUMENT;
|
||||
LOG_WARN("invalid argument", KR(ret), KP(access_info), KP(old_access_info));
|
||||
} else {
|
||||
for (char *str = token; OB_SUCC(ret); str = NULL) {
|
||||
token = ::strtok_r(str, "&", &saved_ptr);
|
||||
if (OB_ISNULL(token)) {
|
||||
break;
|
||||
} else {
|
||||
bool is_item_match = false;
|
||||
for (int64_t i = 0; OB_SUCC(ret) && (i < key_cnt); ++i) {
|
||||
key_name = access_info_keys[i];
|
||||
if (0 == STRNCMP(key_name, token, STRLEN(key_name))) {
|
||||
const int64_t old_access_info_len = STRLEN(old_access_info);
|
||||
const char *value_str = token + STRLEN(key_name);
|
||||
if (0 == STRLEN(value_str)) {
|
||||
ret = OB_INVALID_ARGUMENT;
|
||||
LOG_WARN("value is empty", KR(ret), K(key_name), K(token));
|
||||
} else if (0 == old_access_info_len) { // first field of old_access_info
|
||||
STRCPY(old_access_info, "old_");
|
||||
STRCPY(old_access_info + STRLEN("old_"), token);
|
||||
} else { // not first field of old_access_info, need to concat '&'
|
||||
old_access_info[old_access_info_len] = '&';
|
||||
STRCPY(old_access_info + old_access_info_len + 1, "old_");
|
||||
STRCPY(old_access_info + old_access_info_len + 1 + STRLEN("old_"), token);
|
||||
}
|
||||
is_item_match = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (OB_SUCC(ret) && !is_item_match) {
|
||||
ret = OB_ITEM_NOT_MATCH;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
} // namespace share
|
||||
} // namespace oceanbase
|
||||
|
||||
@ -25,16 +25,10 @@ public:
|
||||
static const char USED_FOR[];
|
||||
static const char PATH[];
|
||||
static const char ENDPOINT[];
|
||||
static const char ACCESS_MODE[];
|
||||
static const char ACCESS_ID[];
|
||||
static const char ACCESS_KEY[];
|
||||
static const char ENCRYPT_KEY[];
|
||||
static const char ACCESS_INFO[];
|
||||
static const char ENCRYPT_INFO[];
|
||||
static const char EXTENSION[];
|
||||
static const char OLD_ACCESS_MODE[];
|
||||
static const char OLD_ACCESS_ID[];
|
||||
static const char OLD_ACCESS_KEY[];
|
||||
static const char OLD_ENCRYPT_KEY[];
|
||||
static const char OLD_ACCESS_INFO[];
|
||||
static const char OLD_ENCRYPT_INFO[];
|
||||
static const char OLD_EXTENSION[];
|
||||
static const char RAM_URL[];
|
||||
@ -51,14 +45,13 @@ public:
|
||||
static const char INVALID_OP_ID[];
|
||||
|
||||
public:
|
||||
static int parse_one_device_config(char *opt_str, ObDeviceConfig &config);
|
||||
static int parse_device_config_field(const char *buf, ObDeviceConfig &device_config);
|
||||
static int parse_config_type_int(const char *key_name, const char *token, int64_t &value);
|
||||
static int parse_config_type_uint(const char *key_name, const char *token, uint64_t &value);
|
||||
static int parse_config_type_char_array(const char *key_name,
|
||||
const char *token,
|
||||
const int64_t max_value_len,
|
||||
char *value);
|
||||
static int convert_access_info_to_old(char *access_info, char *old_access_info);
|
||||
|
||||
private:
|
||||
enum class FieldType
|
||||
|
||||
@ -30,7 +30,8 @@ const int64_t ObDeviceManifest::MANIFEST_VERSION = 1;
|
||||
const int64_t ObDeviceManifest::MAX_FILE_LINE_LEN = 16384; // 16K
|
||||
const char ObDeviceManifest::MANIFEST_FILE_NAME[] = "manifest";
|
||||
const char ObDeviceManifest::HEAD_SECTION[] = "[head]";
|
||||
const char ObDeviceManifest::DEVICE_SECTION[] = "[device]";
|
||||
const char ObDeviceManifest::DEVICE_BEGIN_SECTION[] = "[device begin]";
|
||||
const char ObDeviceManifest::DEVICE_END_SECTION[] = "[device end]";
|
||||
const char ObDeviceManifest::COMMENT_SYMBOL = '#';
|
||||
const char ObDeviceManifest::VERSION_KEY[] = "version=";
|
||||
const char ObDeviceManifest::CLUSTER_ID_KEY[] = "cluster_id=";
|
||||
@ -71,17 +72,15 @@ int ObDeviceManifest::load(ObIArray<ObDeviceConfig> &config_arr, HeadSection &he
|
||||
if (IS_NOT_INIT) {
|
||||
ret = OB_NOT_INIT;
|
||||
LOG_WARN("ObDeviceManifest not init", KR(ret));
|
||||
} else if (OB_ISNULL(data_dir_)) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("data dir is nullptr", KR(ret));
|
||||
} else if (OB_FAIL(databuff_printf(manifest_path, OB_MAX_FILE_NAME_LENGTH, "%s/%s",
|
||||
data_dir_, MANIFEST_FILE_NAME))) {
|
||||
LOG_WARN("construct manifest path fail", KR(ret));
|
||||
} else if (OB_ISNULL(fp = fopen(manifest_path, "r"))) {
|
||||
if (ENOENT == errno) {
|
||||
ret = OB_FILE_NOT_EXIST;
|
||||
LOG_INFO("device manifest file does not exist", KR(ret), K(manifest_path));
|
||||
} else {
|
||||
ret = OB_IO_ERROR;
|
||||
LOG_WARN("cannot open file", KR(ret), K(manifest_path), K(errno));
|
||||
}
|
||||
ret = ObIODeviceLocalFileOp::convert_sys_errno();
|
||||
LOG_WARN("fail to open device manifest file", KR(ret), K(errno), KERRMSG);
|
||||
} else if (OB_FAIL(parse_file_(fp, config_arr, head))) {
|
||||
LOG_WARN("fail to parse file", KR(ret), K(manifest_path));
|
||||
} else if (OB_FAIL(validate_config_checksum_(config_arr, head))) {
|
||||
@ -91,7 +90,10 @@ int ObDeviceManifest::load(ObIArray<ObDeviceConfig> &config_arr, HeadSection &he
|
||||
LOG_WARN("invalid device manifest", K(cluster_id_in_GCONF), "manifest_cluster_id", head.cluster_id_);
|
||||
}
|
||||
if (OB_NOT_NULL(fp)) {
|
||||
fclose(fp);
|
||||
if (OB_UNLIKELY(0 != fclose(fp))) {
|
||||
ret = ObIODeviceLocalFileOp::convert_sys_errno();
|
||||
LOG_ERROR("fail to close file", KR(ret), K(errno), KERRMSG);
|
||||
}
|
||||
}
|
||||
LOG_INFO("finish to load device manifest", KR(ret), K(config_arr), K(head));
|
||||
return ret;
|
||||
@ -101,23 +103,35 @@ int ObDeviceManifest::parse_file_(FILE *fp, ObIArray<ObDeviceConfig> &config_arr
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
SectionType curr_section_type = SECTION_TYPE_INVLAID;
|
||||
SMART_VAR(char[MAX_FILE_LINE_LEN + 2], line_buf) {
|
||||
SMART_VARS_2((char[MAX_FILE_LINE_LEN + 2], line_buf), (ObDeviceConfig, device_config)) {
|
||||
while (OB_SUCC(ret)) {
|
||||
if (0 != feof(fp)) {
|
||||
break; // end of file
|
||||
} else if (0 != ferror(fp)) {
|
||||
} else if (OB_UNLIKELY(0 != ferror(fp))) {
|
||||
ret = OB_IO_ERROR;
|
||||
LOG_WARN("fail to read manifest file", KR(ret), K(errno));
|
||||
} else if (OB_ISNULL(fgets(line_buf, sizeof(line_buf), fp))) {
|
||||
// do nothing
|
||||
} else if (STRLEN(line_buf) > MAX_FILE_LINE_LEN) {
|
||||
} else if (OB_UNLIKELY(STRLEN(line_buf) > MAX_FILE_LINE_LEN)) {
|
||||
ret = OB_SIZE_OVERFLOW;
|
||||
LOG_WARN("file line len is too long", KR(ret), K(MAX_FILE_LINE_LEN));
|
||||
} else if (FALSE_IT(line_buf[STRLEN(line_buf) - 1] = '\0')) { // remove the '\n'
|
||||
} else if (0 == STRLEN(line_buf) || (COMMENT_SYMBOL == line_buf[0])) {
|
||||
// skip blank line or comment line
|
||||
} else if (OB_SUCC(parse_section_type_(line_buf, curr_section_type))) {
|
||||
// this line is a section name
|
||||
// this line is a section name.
|
||||
// SECTION_TYPE_DEVICE_END means finishing parsing one device config, push back the
|
||||
// parsed device config and reset device_config to parse the next device config.
|
||||
if (SECTION_TYPE_DEVICE_END == curr_section_type) {
|
||||
if (OB_UNLIKELY(!device_config.is_valid())) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("invalid device config", KR(ret), K(device_config));
|
||||
} else if (OB_FAIL(config_arr.push_back(device_config))) {
|
||||
LOG_WARN("fail to push back", KR(ret), K(device_config));
|
||||
} else {
|
||||
device_config.reset();
|
||||
}
|
||||
}
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse section type", KR(ret), K(line_buf));
|
||||
} else if (SECTION_TYPE_HEAD == curr_section_type) {
|
||||
@ -125,23 +139,17 @@ int ObDeviceManifest::parse_file_(FILE *fp, ObIArray<ObDeviceConfig> &config_arr
|
||||
if (OB_FAIL(parse_head_section_(line_buf, head))) {
|
||||
LOG_WARN("fail to parse head section", KR(ret), K(line_buf), K(head));
|
||||
}
|
||||
} else if (SECTION_TYPE_DEVICE == curr_section_type) {
|
||||
} else if (SECTION_TYPE_DEVICE_BEGIN == curr_section_type) {
|
||||
ret = OB_SUCCESS; // ignore ret, and then parse device section
|
||||
SMART_VAR(ObDeviceConfig, device_config) {
|
||||
if (OB_FAIL(parse_device_section_(line_buf, device_config))) {
|
||||
LOG_WARN("fail to parse device section", KR(ret), K(line_buf));
|
||||
} else if (OB_FAIL(config_arr.push_back(device_config))) {
|
||||
LOG_WARN("fail to push back", KR(ret), K(device_config));
|
||||
} else {
|
||||
LOG_INFO("succ to parse device section", K(device_config));
|
||||
}
|
||||
if (OB_FAIL(parse_device_section_(line_buf, device_config))) {
|
||||
LOG_WARN("fail to parse device section", KR(ret), K(line_buf));
|
||||
}
|
||||
} else {
|
||||
LOG_WARN("unknow manifest section", KR(ret), K(curr_section_type), K(line_buf));
|
||||
}
|
||||
}
|
||||
}
|
||||
if (OB_SUCC(ret) && (config_arr.count() != head.device_num_)) {
|
||||
if (OB_SUCC(ret) && OB_UNLIKELY((config_arr.count() != head.device_num_))) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("device num does not match between head and device sections", KR(ret), K(head), K(config_arr));
|
||||
}
|
||||
@ -154,8 +162,10 @@ int ObDeviceManifest::parse_section_type_(const char *buf, SectionType &type)
|
||||
int ret = OB_SUCCESS;
|
||||
if (0 == STRCMP(buf, HEAD_SECTION)) {
|
||||
type = SECTION_TYPE_HEAD;
|
||||
} else if (0 == STRCMP(buf, DEVICE_SECTION)) {
|
||||
type = SECTION_TYPE_DEVICE;
|
||||
} else if (0 == STRCMP(buf, DEVICE_BEGIN_SECTION)) {
|
||||
type = SECTION_TYPE_DEVICE_BEGIN;
|
||||
} else if (0 == STRCMP(buf, DEVICE_END_SECTION)) {
|
||||
type = SECTION_TYPE_DEVICE_END;
|
||||
} else {
|
||||
ret = OB_ITEM_NOT_MATCH;
|
||||
}
|
||||
@ -165,7 +175,10 @@ int ObDeviceManifest::parse_section_type_(const char *buf, SectionType &type)
|
||||
int ObDeviceManifest::parse_head_section_(const char *buf, HeadSection &head)
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
if (OB_SUCC(ObDeviceConfigParser::parse_config_type_int(VERSION_KEY, buf, head.version_))) {
|
||||
if (OB_ISNULL(buf)) {
|
||||
ret = OB_INVALID_ARGUMENT;
|
||||
LOG_WARN("buf is nullptr", KR(ret));
|
||||
} else if (OB_SUCC(ObDeviceConfigParser::parse_config_type_int(VERSION_KEY, buf, head.version_))) {
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse_config_type_int", KR(ret), K(VERSION_KEY), K(buf));
|
||||
} else if (OB_SUCC(ObDeviceConfigParser::parse_config_type_int(CLUSTER_ID_KEY, buf,
|
||||
@ -202,20 +215,18 @@ int ObDeviceManifest::parse_head_section_(const char *buf, HeadSection &head)
|
||||
return ret;
|
||||
}
|
||||
|
||||
int ObDeviceManifest::parse_device_section_(char *buf, ObDeviceConfig &device_config)
|
||||
int ObDeviceManifest::parse_device_section_(const char *buf, ObDeviceConfig &device_config)
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
if (OB_SUCC(ObDeviceConfigParser::parse_one_device_config(buf, device_config))) {
|
||||
if (!device_config.is_valid()) {
|
||||
ret = OB_INVALID_ARGUMENT;
|
||||
LOG_WARN("invalid non-initial device config", KR(ret), K(device_config));
|
||||
if (OB_ISNULL(buf)) {
|
||||
ret = OB_INVALID_ARGUMENT;
|
||||
LOG_WARN("buf is nullptr", KR(ret));
|
||||
} else if (OB_FAIL(ObDeviceConfigParser::parse_device_config_field(buf, device_config))) {
|
||||
if (OB_ITEM_NOT_MATCH == ret) {
|
||||
LOG_WARN("unknow manifest device section line", KR(ret), K(buf));
|
||||
} else {
|
||||
LOG_INFO("succ to parse one device config", K(device_config));
|
||||
LOG_WARN("fail to parse device config field", KR(ret), K(buf));
|
||||
}
|
||||
} else if (OB_ITEM_NOT_MATCH != ret) {
|
||||
LOG_WARN("fail to parse_one_device_config", KR(ret), K(buf));
|
||||
} else {
|
||||
LOG_WARN("unknow manifest device section line", KR(ret), K(buf));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
@ -256,6 +267,9 @@ int ObDeviceManifest::dump2file(
|
||||
allocator.alloc(OB_MAX_FILE_NAME_LENGTH)))) {
|
||||
ret = OB_ALLOCATE_MEMORY_FAILED;
|
||||
LOG_WARN("fail to malloc memory for his_manifest_path", KR(ret), K(OB_MAX_FILE_NAME_LENGTH));
|
||||
} else if (OB_ISNULL(data_dir_)) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("data dir is nullptr", KR(ret));
|
||||
} else if (OB_FAIL(databuff_printf(manifest_path, OB_MAX_FILE_NAME_LENGTH, "%s/%s",
|
||||
data_dir_, MANIFEST_FILE_NAME))) {
|
||||
LOG_WARN("fail to construct manifest path", KR(ret));
|
||||
@ -272,11 +286,8 @@ int ObDeviceManifest::dump2file(
|
||||
} else if (OB_ISNULL(fp = ::fdopen(fd, "w"))) {
|
||||
ret = ObIODeviceLocalFileOp::convert_sys_errno();
|
||||
LOG_WARN("fail to fdopen", KR(ret), K(fd), K(errno), KERRMSG);
|
||||
} else if (FALSE_IT(pret =
|
||||
fprintf(fp, "# THIS FILE IS AUTOMATICALLY GENERATED BY OBSERVER. PLEASE DO NOT MODIFY IT MANUALLY!!!\n"))) {
|
||||
} else if (pret <= 0 || pret > MAX_FILE_LINE_LEN) {
|
||||
ret = OB_IO_ERROR;
|
||||
LOG_WARN("fail to write manifest", KR(ret), K(pret), K(errno));
|
||||
} else if (OB_FAIL(print_file_line_(fp, "# THIS FILE IS AUTOMATICALLY GENERATED BY OBSERVER. PLEASE DO NOT MODIFY IT MANUALLY!!!\n"))) {
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if (OB_FAIL(write_head_(fp, head))) {
|
||||
LOG_WARN("fail to write head", KR(ret), K(head));
|
||||
} else if (OB_FAIL(write_device_config_(fp, configs))) {
|
||||
@ -308,66 +319,77 @@ int ObDeviceManifest::dump2file(
|
||||
return ret;
|
||||
}
|
||||
|
||||
// The format of head section is like:
|
||||
// [head]
|
||||
// version=1
|
||||
// cluster_id=0
|
||||
// head_checksum=790215355
|
||||
// device_checksum=2094936029
|
||||
// device_num=2
|
||||
// modify_timestamp_us=1725551141912558
|
||||
// last_op_id=3
|
||||
// last_sub_op_id=4
|
||||
int ObDeviceManifest::write_head_(FILE *fp, const HeadSection &head) const
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
int pret = 0;
|
||||
if (FALSE_IT(pret = fprintf(fp, "%s\n", HEAD_SECTION))) { // [head]
|
||||
} else if (pret <= 0 || pret > MAX_FILE_LINE_LEN) {
|
||||
ret = OB_IO_ERROR;
|
||||
LOG_WARN("fail to write manifest", KR(ret), K(pret), K(errno));
|
||||
} else if (FALSE_IT(pret = fprintf(fp, "%s%ld\n", VERSION_KEY, head.version_))) { // version=
|
||||
} else if (pret <= 0 || pret > MAX_FILE_LINE_LEN) {
|
||||
ret = OB_IO_ERROR;
|
||||
LOG_WARN("fail to write manifest", KR(ret), K(pret), K(errno));
|
||||
} else if (FALSE_IT(pret = fprintf(fp, "%s%ld\n", CLUSTER_ID_KEY, head.cluster_id_))) { // cluster_id=
|
||||
} else if (pret <= 0 || pret > MAX_FILE_LINE_LEN) {
|
||||
ret = OB_IO_ERROR;
|
||||
LOG_WARN("fail to write manifest", KR(ret), K(pret), K(errno));
|
||||
} else if (FALSE_IT(pret = fprintf(fp, "%s%ld\n", HEAD_CHECKSUM_KEY, head.head_checksum_))) { // head_checksum=
|
||||
} else if (pret <= 0 || pret > MAX_FILE_LINE_LEN) {
|
||||
ret = OB_IO_ERROR;
|
||||
LOG_WARN("fail to write manifest", KR(ret), K(pret), K(errno));
|
||||
} else if (FALSE_IT(pret = fprintf(fp, "%s%ld\n", DEVICE_CHECKSUM_KEY, head.device_checksum_))) { // device_checksum=
|
||||
} else if (pret <= 0 || pret > MAX_FILE_LINE_LEN) {
|
||||
ret = OB_IO_ERROR;
|
||||
LOG_WARN("fail to write manifest", KR(ret), K(pret), K(errno));
|
||||
} else if (FALSE_IT(pret = fprintf(fp, "%s%ld\n", DEVICE_NUM_KEY, head.device_num_))) { // device_num=
|
||||
} else if (pret <= 0 || pret > MAX_FILE_LINE_LEN) {
|
||||
ret = OB_IO_ERROR;
|
||||
LOG_WARN("fail to write manifest", KR(ret), K(pret), K(errno));
|
||||
} else if (FALSE_IT(pret = fprintf(fp, "%s%ld\n", MODIFY_TIMESTAMP_KEY, head.modify_timestamp_us_))) { // modify_timestamp_us=
|
||||
} else if (pret <= 0 || pret > MAX_FILE_LINE_LEN) {
|
||||
ret = OB_IO_ERROR;
|
||||
LOG_WARN("fail to write manifest", KR(ret), K(pret), K(errno));
|
||||
} else if (FALSE_IT(pret = fprintf(fp, "%s%ld\n", LAST_OP_ID_KEY, head.last_op_id_))) { // last_op_id=
|
||||
} else if (pret <= 0 || pret > MAX_FILE_LINE_LEN) {
|
||||
ret = OB_IO_ERROR;
|
||||
LOG_WARN("fail to write manifest", KR(ret), K(pret), K(errno));
|
||||
} else if (FALSE_IT(pret = fprintf(fp, "%s%ld\n", LAST_SUB_OP_ID_KEY, head.last_sub_op_id_))) { // last_sub_op_id=
|
||||
} else if (pret <= 0 || pret > MAX_FILE_LINE_LEN) {
|
||||
ret = OB_IO_ERROR;
|
||||
LOG_WARN("fail to write manifest", KR(ret), K(pret), K(errno));
|
||||
} else if (FALSE_IT(pret = fprintf(fp, "%s\n", DEVICE_SECTION))) { // [device]
|
||||
} else if (pret <= 0 || pret > MAX_FILE_LINE_LEN) {
|
||||
ret = OB_IO_ERROR;
|
||||
LOG_WARN("fail to write manifest", KR(ret), K(pret), K(errno));
|
||||
if (OB_FAIL(print_file_line_(fp, "%s\n", HEAD_SECTION))) { // [head]
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if (OB_FAIL(print_file_line_(fp, "%s%ld\n", VERSION_KEY, head.version_))) { // version=
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if (OB_FAIL(print_file_line_(fp, "%s%ld\n", CLUSTER_ID_KEY, head.cluster_id_))) { // cluster_id=
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if (OB_FAIL(print_file_line_(fp, "%s%ld\n", HEAD_CHECKSUM_KEY, head.head_checksum_))) { // head_checksum=
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if (OB_FAIL(print_file_line_(fp, "%s%ld\n", DEVICE_CHECKSUM_KEY, head.device_checksum_))) { // device_checksum=
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if (OB_FAIL(print_file_line_(fp, "%s%ld\n", DEVICE_NUM_KEY, head.device_num_))) { // device_num=
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if (OB_FAIL(print_file_line_(fp, "%s%ld\n", MODIFY_TIMESTAMP_KEY, head.modify_timestamp_us_))) { // modify_timestamp_us=
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if (OB_FAIL(print_file_line_(fp, "%s%ld\n", LAST_OP_ID_KEY, head.last_op_id_))) { // last_op_id=
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if (OB_FAIL(print_file_line_(fp, "%s%ld\n", LAST_SUB_OP_ID_KEY, head.last_sub_op_id_))) { // last_sub_op_id=
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
}
|
||||
LOG_INFO("finish to write head", KR(ret));
|
||||
LOG_INFO("finish to write manifest head", KR(ret));
|
||||
return ret;
|
||||
}
|
||||
|
||||
// The format of each line in device section is like:
|
||||
// Example 1:
|
||||
// used_for=...&path=...&endpoint=...
|
||||
// &access_mode=access_by_id&access_id=...&access_key=...&encrypt_info=...&extension=...
|
||||
// &old_access_mode=access_by_id&old_access_id=...&old_access_key=...&old_encrypt_info=...
|
||||
// &old_extension=...&state=...&state_info=...&create_timestamp=...&last_check_timestamp=...
|
||||
// &op_id=...&sub_op_id=...&storage_id=...&iops=...&bandwidth=...
|
||||
// Example 2:
|
||||
// used_for=...&path=...&endpoint=...&access_mode=access_by_ram_url&ram_url=xxx&state=...
|
||||
// &state_info=...&create_timestamp=...&last_check_timestamp=...&op_id=...&sub_op_id=...
|
||||
// &storage_id=...&iops=...&bandwidth=...
|
||||
// The format of device section is like:
|
||||
// [device begin]
|
||||
// used_for=DATA
|
||||
// path=oss://bucket-name/root_dir
|
||||
// endpoint=host=cn-xxx.com
|
||||
// access_info=access_id=ccccc&encrypt_key=ddddd
|
||||
// extension=appid=456&checksum_type=md5
|
||||
// old_access_info=access_id=aaa&encrypt_key=bbb
|
||||
// old_extension=appid=123&checksum_type=crc32
|
||||
// state=ADDED
|
||||
// state_info=is_connective=true
|
||||
// create_timestamp=1679579590156
|
||||
// last_check_timestamp=1679579590256
|
||||
// op_id=3
|
||||
// sub_op_id=4
|
||||
// storage_id=2
|
||||
// max_iops=10000
|
||||
// max_bandwidth=1024000000
|
||||
// [device end]
|
||||
// [device begin]
|
||||
// used_for=LOG
|
||||
// path=oss://bucket-name/root_dir
|
||||
// endpoint=host=cn-xxx.com
|
||||
// access_info=access_id=aaa&encrypt_key=bbb
|
||||
// extension=appid=123&checksum_type=crc32
|
||||
// state=ADDED
|
||||
// state_info=is_connective=true
|
||||
// create_timestamp=1679579590156
|
||||
// last_check_timestamp=1679579590256
|
||||
// op_id=1
|
||||
// sub_op_id=2
|
||||
// storage_id=1
|
||||
// max_iops=10000
|
||||
// max_bandwidth=1024000000
|
||||
// [device end]
|
||||
// Note: not all the fields are needed.
|
||||
int ObDeviceManifest::write_device_config_(
|
||||
FILE *fp,
|
||||
@ -383,87 +405,53 @@ int ObDeviceManifest::write_device_config_(
|
||||
if (!config.is_valid()) {
|
||||
ret = OB_INVALID_ARGUMENT;
|
||||
LOG_WARN("device config is invalid", KR(ret), K(config));
|
||||
} else if (OB_FAIL(databuff_printf(buf, MAX_FILE_LINE_LEN, "%s%s&%s%s&%s%s&%s",
|
||||
ObDeviceConfigParser::USED_FOR, config.used_for_,
|
||||
ObDeviceConfigParser::PATH, config.path_,
|
||||
ObDeviceConfigParser::ENDPOINT, config.endpoint_,
|
||||
config.access_info_))) {
|
||||
LOG_WARN("fail to construct device config line buf", KR(ret));
|
||||
} else if (OB_FAIL(print_file_line_(fp, "%s\n", DEVICE_BEGIN_SECTION))) { // [device begin]
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if (OB_FAIL(print_file_line_(fp, "%s%s\n", ObDeviceConfigParser::USED_FOR, config.used_for_))) {
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if (OB_FAIL(print_file_line_(fp, "%s%s\n", ObDeviceConfigParser::PATH, config.path_))) {
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if (OB_FAIL(print_file_line_(fp, "%s%s\n", ObDeviceConfigParser::ENDPOINT, config.endpoint_))) {
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if (OB_FAIL(print_file_line_(fp, "%s%s\n", ObDeviceConfigParser::ACCESS_INFO, config.access_info_))) {
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if ((0 < STRLEN(config.encrypt_info_)) &&
|
||||
OB_FAIL(print_file_line_(fp, "%s%s\n", ObDeviceConfigParser::ENCRYPT_INFO, config.encrypt_info_))) {
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if ((0 < STRLEN(config.extension_)) &&
|
||||
OB_FAIL(print_file_line_(fp, "%s%s\n", ObDeviceConfigParser::EXTENSION, config.extension_))) {
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if ((0 < STRLEN(config.old_access_info_)) &&
|
||||
OB_FAIL(print_file_line_(fp, "%s%s\n", ObDeviceConfigParser::OLD_ACCESS_INFO, config.old_access_info_))) {
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if ((0 < STRLEN(config.old_encrypt_info_)) &&
|
||||
OB_FAIL(print_file_line_(fp, "%s%s\n", ObDeviceConfigParser::OLD_ENCRYPT_INFO, config.old_encrypt_info_))) {
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if ((0 < STRLEN(config.old_extension_)) &&
|
||||
OB_FAIL(print_file_line_(fp, "%s%s\n", ObDeviceConfigParser::OLD_EXTENSION, config.old_extension_))) {
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if (OB_FAIL(print_file_line_(fp, "%s%s\n", ObDeviceConfigParser::STATE, config.state_))) {
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if (OB_FAIL(print_file_line_(fp, "%s%s\n", ObDeviceConfigParser::STATE_INFO, config.state_info_))) {
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if (OB_FAIL(print_file_line_(fp, "%s%ld\n", ObDeviceConfigParser::CREATE_TIMESTAMP, config.create_timestamp_))) {
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if (OB_FAIL(print_file_line_(fp, "%s%ld\n", ObDeviceConfigParser::LAST_CHECK_TIMESTAMP, config.last_check_timestamp_))) {
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if (OB_FAIL(print_file_line_(fp, "%s%lu\n", ObDeviceConfigParser::OP_ID, config.op_id_))) {
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if (OB_FAIL(print_file_line_(fp, "%s%lu\n", ObDeviceConfigParser::SUB_OP_ID, config.sub_op_id_))) {
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if (OB_FAIL(print_file_line_(fp, "%s%lu\n", ObDeviceConfigParser::STORAGE_ID, config.storage_id_))) {
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if (OB_FAIL(print_file_line_(fp, "%s%ld\n", ObDeviceConfigParser::MAX_IOPS, config.max_iops_))) {
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if (OB_FAIL(print_file_line_(fp, "%s%ld\n", ObDeviceConfigParser::MAX_BANDWIDTH, config.max_bandwidth_))) {
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
} else if (OB_FAIL(print_file_line_(fp, "%s\n", DEVICE_END_SECTION))) { // [device end]
|
||||
LOG_WARN("fail to print file line", KR(ret));
|
||||
}
|
||||
if (OB_SUCC(ret) && 0 != STRLEN(config.encrypt_info_)) {
|
||||
if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf),
|
||||
"&%s%s", ObDeviceConfigParser::ENCRYPT_INFO, config.encrypt_info_))) {
|
||||
LOG_WARN("fail to construct device config line buf", KR(ret), K(config));
|
||||
}
|
||||
}
|
||||
if (OB_SUCC(ret) && 0 != STRLEN(config.extension_)) {
|
||||
if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf),
|
||||
"&%s%s", ObDeviceConfigParser::EXTENSION, config.extension_))) {
|
||||
LOG_WARN("fail to construct device config line buf", KR(ret), K(config));
|
||||
}
|
||||
}
|
||||
if (OB_SUCC(ret) && 0 != STRLEN(config.old_access_info_)) {
|
||||
if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf),
|
||||
"&%s", config.old_access_info_))) {
|
||||
LOG_WARN("fail to construct device config line buf", KR(ret), K(config));
|
||||
}
|
||||
}
|
||||
if (OB_SUCC(ret) && 0 != STRLEN(config.old_encrypt_info_)) {
|
||||
if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf),
|
||||
"&%s%s", ObDeviceConfigParser::OLD_ENCRYPT_INFO, config.old_encrypt_info_))) {
|
||||
LOG_WARN("fail to construct device config line buf", KR(ret), K(config));
|
||||
}
|
||||
}
|
||||
if (OB_SUCC(ret) && 0 != STRLEN(config.old_extension_)) {
|
||||
if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf),
|
||||
"&%s%s", ObDeviceConfigParser::OLD_EXTENSION, config.old_extension_))) {
|
||||
LOG_WARN("fail to construct device config line buf", KR(ret), K(config));
|
||||
}
|
||||
}
|
||||
if (OB_SUCC(ret)) {
|
||||
if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf),
|
||||
"&%s%s", ObDeviceConfigParser::STATE, config.state_))) {
|
||||
LOG_WARN("fail to construct device config line buf", KR(ret), K(config));
|
||||
}
|
||||
}
|
||||
if (OB_SUCC(ret) && 0 != STRLEN(config.state_info_)) {
|
||||
if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf),
|
||||
"&%s%s", ObDeviceConfigParser::STATE_INFO, config.state_info_))) {
|
||||
LOG_WARN("fail to construct device config line buf", KR(ret), K(config));
|
||||
}
|
||||
}
|
||||
if (OB_SUCC(ret)) {
|
||||
if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf),
|
||||
"&%s%ld", ObDeviceConfigParser::CREATE_TIMESTAMP, config.create_timestamp_))) {
|
||||
LOG_WARN("fail to construct device config line buf", KR(ret), K(config));
|
||||
} else if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf),
|
||||
"&%s%ld", ObDeviceConfigParser::LAST_CHECK_TIMESTAMP, config.last_check_timestamp_))) {
|
||||
LOG_WARN("fail to construct device config line buf", KR(ret), K(config));
|
||||
} else if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf),
|
||||
"&%s%lu", ObDeviceConfigParser::OP_ID, config.op_id_))) {
|
||||
LOG_WARN("fail to construct device config line buf", KR(ret), K(config));
|
||||
} else if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf),
|
||||
"&%s%lu", ObDeviceConfigParser::SUB_OP_ID, config.sub_op_id_))) {
|
||||
LOG_WARN("fail to construct device config line buf", KR(ret), K(config));
|
||||
} else if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf),
|
||||
"&%s%lu", ObDeviceConfigParser::STORAGE_ID, config.storage_id_))) {
|
||||
LOG_WARN("fail to construct device config line buf", KR(ret), K(config));
|
||||
} else if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf),
|
||||
"&%s%ld", ObDeviceConfigParser::MAX_IOPS, config.max_iops_))) {
|
||||
LOG_WARN("fail to construct device config line buf", KR(ret), K(config));
|
||||
} else if (OB_FAIL(databuff_printf(buf + STRLEN(buf), MAX_FILE_LINE_LEN - STRLEN(buf),
|
||||
"&%s%ld", ObDeviceConfigParser::MAX_BANDWIDTH, config.max_bandwidth_))) {
|
||||
LOG_WARN("fail to construct device config line buf", KR(ret), K(config));
|
||||
}
|
||||
}
|
||||
if (OB_SUCC(ret)) {
|
||||
pret = fprintf(fp, "%s\n", buf);
|
||||
if (pret <= 0 || pret > MAX_FILE_LINE_LEN) {
|
||||
ret = OB_IO_ERROR;
|
||||
LOG_WARN("fail to write manifest", KR(ret), K(pret), K(errno), K(buf));
|
||||
}
|
||||
}
|
||||
LOG_INFO("finish to write manifest device line", KR(ret), K(buf));
|
||||
LOG_INFO("finish to write manifest device", KR(ret), K(buf));
|
||||
}
|
||||
}
|
||||
LOG_INFO("finish to write device config", KR(ret));
|
||||
@ -564,5 +552,24 @@ int ObDeviceManifest::validate_config_checksum_(
|
||||
return ret;
|
||||
}
|
||||
|
||||
int ObDeviceManifest::print_file_line_(FILE *fp, const char *fmt, ...) const
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
if (OB_ISNULL(fp) || OB_ISNULL(fmt)) {
|
||||
ret = OB_INVALID_ARGUMENT;
|
||||
LOG_WARN("invalid arguments", KR(ret), KP(fp), KP(fmt));
|
||||
} else {
|
||||
va_list args;
|
||||
va_start(args, fmt);
|
||||
int pret = vfprintf(fp, fmt, args);
|
||||
va_end(args);
|
||||
if (OB_UNLIKELY((pret <= 0) || (pret >= MAX_FILE_LINE_LEN))) {
|
||||
ret = OB_IO_ERROR;
|
||||
LOG_WARN("fail to vfprintf", KR(ret), K(pret), K(errno), KERRMSG);
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
} // namespace share
|
||||
} // namespace oceanbase
|
||||
|
||||
@ -106,12 +106,14 @@ public:
|
||||
private:
|
||||
static const char MANIFEST_FILE_NAME[];
|
||||
static const char HEAD_SECTION[];
|
||||
static const char DEVICE_SECTION[];
|
||||
static const char DEVICE_BEGIN_SECTION[];
|
||||
static const char DEVICE_END_SECTION[];
|
||||
enum SectionType
|
||||
{
|
||||
SECTION_TYPE_INVLAID,
|
||||
SECTION_TYPE_HEAD,
|
||||
SECTION_TYPE_DEVICE
|
||||
SECTION_TYPE_DEVICE_BEGIN,
|
||||
SECTION_TYPE_DEVICE_END
|
||||
};
|
||||
|
||||
static const int64_t MANIFEST_VERSION;
|
||||
@ -130,7 +132,7 @@ private:
|
||||
int parse_file_(FILE *fp, common::ObIArray<ObDeviceConfig> &config_arr, HeadSection &head);
|
||||
int parse_section_type_(const char *buf, SectionType &type);
|
||||
int parse_head_section_(const char *buf, HeadSection &head);
|
||||
int parse_device_section_(char *buf, ObDeviceConfig &device_config);
|
||||
int parse_device_section_(const char *buf, ObDeviceConfig &device_config);
|
||||
int write_head_(FILE *fp, const HeadSection &head) const;
|
||||
// sort device configs according to used_for_, path_ and endpoint_
|
||||
int sort_device_configs_(common::ObIArray<ObDeviceConfig> &config_arr) const;
|
||||
@ -141,6 +143,7 @@ private:
|
||||
HeadSection &head) const;
|
||||
int validate_config_checksum_(const common::ObIArray<ObDeviceConfig> &config_arr,
|
||||
const HeadSection &head) const;
|
||||
int print_file_line_(FILE *fp, const char *fmt, ...) const;
|
||||
|
||||
private:
|
||||
bool is_inited_;
|
||||
|
||||
@ -497,29 +497,30 @@ int ObDeviceManifestTask::update_device_config(
|
||||
LOG_WARN("fail to get device config", KR(ret), K(device_config_key));
|
||||
} else if (FALSE_IT(device_config.op_id_ = storage_op_info.op_id_)) {
|
||||
} else if (FALSE_IT(device_config.sub_op_id_ = storage_op_info.sub_op_id_)) {
|
||||
// Note: must MEMSET zero before STRCPY, because calc checksum depends on sizeof()
|
||||
} else if (FALSE_IT(MEMSET(device_config.state_, 0, sizeof(device_config.state_)))) {
|
||||
} else if (FALSE_IT(STRCPY(device_config.state_, ObZoneStorageState::get_str(op_type)))) {
|
||||
} else if (ObZoneStorageState::STATE::CHANGING == op_type) {
|
||||
// reset old_access_info
|
||||
memset(device_config.old_access_info_, 0, sizeof(device_config.old_access_info_));
|
||||
// replace old_access_info with access_info
|
||||
if (OB_FAIL(ObDeviceConfigParser::convert_access_info_to_old(
|
||||
device_config.access_info_, device_config.old_access_info_))) {
|
||||
LOG_WARN("fail to convert access info to old", KR(ret), "access_info",
|
||||
common::ObSzString(device_config.access_info_));
|
||||
} else {
|
||||
STRCPY(device_config.old_encrypt_info_, device_config.encrypt_info_);
|
||||
STRCPY(device_config.old_extension_, device_config.extension_);
|
||||
STRCPY(device_config.access_info_, zone_storage_info.dest_attr_.authorization_);
|
||||
STRCPY(device_config.extension_, zone_storage_info.dest_attr_.extension_);
|
||||
device_config.max_iops_ = zone_storage_info.max_iops_;
|
||||
device_config.max_bandwidth_ = zone_storage_info.max_bandwidth_;
|
||||
device_config.last_check_timestamp_ = common::ObTimeUtility::fast_current_time();
|
||||
char tmp_state_info[OB_MAX_STORAGE_STATE_INFO_LENGTH] = { 0 };
|
||||
MEMCPY(tmp_state_info, STORAGE_IS_CONNECTIVE_KEY, STRLEN(STORAGE_IS_CONNECTIVE_KEY));
|
||||
MEMCPY(tmp_state_info + STRLEN(tmp_state_info), is_connective ? "true" : "false",
|
||||
is_connective ? STRLEN("true") : STRLEN("false"));
|
||||
STRCPY(device_config.state_info_, tmp_state_info);
|
||||
}
|
||||
// Note: must MEMSET zero before STRCPY, because calc checksum depends on sizeof()
|
||||
MEMSET(device_config.old_access_info_, 0, sizeof(device_config.old_access_info_));
|
||||
MEMSET(device_config.old_encrypt_info_, 0, sizeof(device_config.old_encrypt_info_));
|
||||
MEMSET(device_config.old_extension_, 0, sizeof(device_config.old_extension_));
|
||||
STRCPY(device_config.old_access_info_, device_config.access_info_);
|
||||
STRCPY(device_config.old_encrypt_info_, device_config.encrypt_info_);
|
||||
STRCPY(device_config.old_extension_, device_config.extension_);
|
||||
MEMSET(device_config.access_info_, 0, sizeof(device_config.access_info_));
|
||||
MEMSET(device_config.extension_, 0, sizeof(device_config.extension_));
|
||||
STRCPY(device_config.access_info_, zone_storage_info.dest_attr_.authorization_);
|
||||
STRCPY(device_config.extension_, zone_storage_info.dest_attr_.extension_);
|
||||
device_config.max_iops_ = zone_storage_info.max_iops_;
|
||||
device_config.max_bandwidth_ = zone_storage_info.max_bandwidth_;
|
||||
device_config.last_check_timestamp_ = common::ObTimeUtility::fast_current_time();
|
||||
char tmp_state_info[OB_MAX_STORAGE_STATE_INFO_LENGTH] = { 0 };
|
||||
MEMCPY(tmp_state_info, STORAGE_IS_CONNECTIVE_KEY, STRLEN(STORAGE_IS_CONNECTIVE_KEY));
|
||||
MEMCPY(tmp_state_info + STRLEN(tmp_state_info), is_connective ? "true" : "false",
|
||||
is_connective ? STRLEN("true") : STRLEN("false"));
|
||||
MEMSET(device_config.state_info_, 0, sizeof(device_config.state_info_));
|
||||
STRCPY(device_config.state_info_, tmp_state_info);
|
||||
}
|
||||
if (FAILEDx(ObDeviceConfigMgr::get_instance().update_device_config(device_config))) {
|
||||
LOG_WARN("fail to update device config", KR(ret), K(device_config));
|
||||
|
||||
@ -31,112 +31,100 @@ class TestDeviceConfig: public ::testing::Test
|
||||
public:
|
||||
TestDeviceConfig() {}
|
||||
virtual ~TestDeviceConfig() {}
|
||||
virtual void SetUp()
|
||||
{
|
||||
char buf_log_id[] = "used_for=LOG&path=oss://antsys-oceanbasebackup/backup_clog_1&endpoint="
|
||||
"host=cn-hangzhou-alipay-b.oss-cdn.aliyun-inc.com&access_mode=access_by_id&access_id=a&ac"
|
||||
"cess_key=b&encrypt_info=encrypt_key=b&extension=null&old_access_mode=access_by_id&old_access"
|
||||
"_id=c&old_access_key=d&old_encrypt_info=encrypt_key=b&old_extension=null&state=ADDING&state"
|
||||
"_info=is_connective=true&create_timestamp=1679579590156&last_check_timestamp=1679579590256"
|
||||
"&op_id=1&sub_op_id=1&storage_id=1&max_iops=0&max_bandwidth=0";
|
||||
char buf_log_ram_url[] = "used_for=LOG&path=oss://antsys-oceanbasebackup/backup_clog_2&endpoint"
|
||||
"=host=cn-hangzhou-alipay-b.oss-cdn.aliyun-inc.com&access_mode=access_by_ram_url&ram_url=xxxx"
|
||||
"&state=ADDING&state_info=is_connective=true&create_timestamp=1679579590356&last_check_timest"
|
||||
"amp=1679579590456&op_id=2&sub_op_id=1&storage_id=2&max_iops=0&max_bandwidth=0";
|
||||
char buf_data[] = "used_for=DATA&path=oss://antsys-oceanbasebackup/backup_sstable&endpoint="
|
||||
"host=cn-hangzhou-alipay-b.oss-cdn.aliyun-inc.com&access_mode=access_by_id&access_id=a&ac"
|
||||
"cess_key=b&encrypt_info=encrypt_key=b&extension=null&old_access_mode=access_by_id&old_access"
|
||||
"_id=c&old_access_key=d&old_encrypt_info=encrypt_key=b&old_extension=null&state=ADDING&state"
|
||||
"_info=is_connective=false&create_timestamp=1679579590556&last_check_timestamp=1679579590656"
|
||||
"&op_id=3&sub_op_id=1&storage_id=3&max_iops=0&max_bandwidth=0";
|
||||
memcpy(buf_log_id_, buf_log_id, sizeof(buf_log_id));
|
||||
memcpy(buf_log_ram_url_, buf_log_ram_url, sizeof(buf_log_ram_url));
|
||||
memcpy(buf_data_, buf_data, sizeof(buf_data));
|
||||
}
|
||||
virtual void SetUp() {}
|
||||
virtual void TearDown() {}
|
||||
static void SetUpTestCase() {}
|
||||
static void TearDownTestCase() {}
|
||||
private:
|
||||
DISALLOW_COPY_AND_ASSIGN(TestDeviceConfig);
|
||||
protected:
|
||||
char buf_log_id_[16384];
|
||||
char buf_log_ram_url_[16384];
|
||||
char buf_data_[16384];
|
||||
};
|
||||
|
||||
TEST_F(TestDeviceConfig, DISABLED_test_device_config_parser)
|
||||
TEST_F(TestDeviceConfig, parser_and_dump_load_and_get)
|
||||
{
|
||||
// 1. test device config parser
|
||||
ObDeviceConfig config;
|
||||
ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_one_device_config(buf_log_id_, config));
|
||||
const char *used_for_str = "used_for=LOG";
|
||||
const char *path_str = "path=oss://bucket-name/root_dir";
|
||||
const char *endpoint_str = "endpoint=host=cn-xxx.com";
|
||||
const char *access_info_str = "access_info=access_id=aaa&encrypt_key=bbb";
|
||||
const char *extension_str = "extension=appid=123&checksum_type=crc32";
|
||||
const char *state_str = "state=ADDED";
|
||||
const char *state_info_str = "state_info=is_connective=true";
|
||||
const char *create_timestamp_str = "create_timestamp=1679579590156";
|
||||
const char *last_check_timestamp_str = "last_check_timestamp=1679579590256";
|
||||
const char *op_id_str = "op_id=1";
|
||||
const char *sub_op_id_str = "sub_op_id=2";
|
||||
const char *storage_id_str = "storage_id=1";
|
||||
const char *max_iops_str = "max_iops=10000";
|
||||
const char *max_bandwidth_str = "max_bandwidth=1024000000";
|
||||
ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(used_for_str, config));
|
||||
ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(path_str, config));
|
||||
ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(endpoint_str, config));
|
||||
ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(access_info_str, config));
|
||||
ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(extension_str, config));
|
||||
ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(state_str, config));
|
||||
ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(state_info_str, config));
|
||||
ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(create_timestamp_str, config));
|
||||
ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(last_check_timestamp_str, config));
|
||||
ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(op_id_str, config));
|
||||
ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(sub_op_id_str, config));
|
||||
ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(storage_id_str, config));
|
||||
ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(max_iops_str, config));
|
||||
ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_device_config_field(max_bandwidth_str, config));
|
||||
ASSERT_EQ(0, STRCMP(config.used_for_, "LOG"));
|
||||
ASSERT_EQ(0, STRCMP(config.path_, "oss://antsys-oceanbasebackup/backup_clog_1"));
|
||||
ASSERT_EQ(0, STRCMP(config.endpoint_, "host=cn-hangzhou-alipay-b.oss-cdn.aliyun-inc.com"));
|
||||
ASSERT_EQ(0, STRCMP(config.access_info_, "access_mode=access_by_id&access_id=a&access_key=b"));
|
||||
ASSERT_EQ(0, STRCMP(config.encrypt_info_, "encrypt_key=b"));
|
||||
ASSERT_EQ(0, STRCMP(config.extension_, "null"));
|
||||
ASSERT_EQ(0, STRCMP(config.old_access_info_, "old_access_mode=access_by_id&old_access_id=c&old_access_key=d"));
|
||||
ASSERT_EQ(0, STRCMP(config.old_encrypt_info_, "encrypt_key=b"));
|
||||
ASSERT_EQ(0, STRCMP(config.old_extension_, "null"));
|
||||
ASSERT_EQ(0, STRCMP(config.state_, "ADDING"));
|
||||
ASSERT_EQ(0, STRCMP(config.path_, "oss://bucket-name/root_dir"));
|
||||
ASSERT_EQ(0, STRCMP(config.endpoint_, "host=cn-xxx.com"));
|
||||
ASSERT_EQ(0, STRCMP(config.access_info_, "access_id=aaa&encrypt_key=bbb"));
|
||||
ASSERT_EQ(0, STRCMP(config.extension_, "appid=123&checksum_type=crc32"));
|
||||
ASSERT_EQ(0, STRCMP(config.state_, "ADDED"));
|
||||
ASSERT_EQ(0, STRCMP(config.state_info_, "is_connective=true"));
|
||||
ASSERT_EQ(1679579590156, config.create_timestamp_);
|
||||
ASSERT_EQ(1679579590256, config.last_check_timestamp_);
|
||||
ASSERT_EQ(1, config.op_id_);
|
||||
ASSERT_EQ(1, config.sub_op_id_);
|
||||
ASSERT_EQ(2, config.sub_op_id_);
|
||||
ASSERT_EQ(1, config.storage_id_);
|
||||
ASSERT_EQ(0, config.max_iops_);
|
||||
ASSERT_EQ(0, config.max_bandwidth_);
|
||||
ASSERT_EQ(10000, config.max_iops_);
|
||||
ASSERT_EQ(1024000000, config.max_bandwidth_);
|
||||
|
||||
ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_one_device_config(buf_log_ram_url_, config));
|
||||
ASSERT_EQ(0, STRCMP(config.used_for_, "LOG"));
|
||||
ASSERT_EQ(0, STRCMP(config.path_, "oss://antsys-oceanbasebackup/backup_clog_2"));
|
||||
ASSERT_EQ(0, STRCMP(config.endpoint_, "host=cn-hangzhou-alipay-b.oss-cdn.aliyun-inc.com"));
|
||||
ASSERT_EQ(0, STRCMP(config.access_info_, "access_mode=access_by_ram_url&ram_url=xxxx"));
|
||||
ASSERT_EQ(0, STRCMP(config.state_, "ADDING"));
|
||||
ASSERT_EQ(0, STRCMP(config.state_info_, "is_connective=true"));
|
||||
ASSERT_EQ(1679579590356, config.create_timestamp_);
|
||||
ASSERT_EQ(1679579590456, config.last_check_timestamp_);
|
||||
ASSERT_EQ(2, config.op_id_);
|
||||
ASSERT_EQ(1, config.sub_op_id_);
|
||||
ASSERT_EQ(2, config.storage_id_);
|
||||
ASSERT_EQ(0, config.max_iops_);
|
||||
ASSERT_EQ(0, config.max_bandwidth_);
|
||||
}
|
||||
|
||||
TEST_F(TestDeviceConfig, DISABLED_test_device_manifest)
|
||||
{
|
||||
// 2. test device manifest dump2file and load
|
||||
ObDeviceManifest device_manifest;
|
||||
ObArray<ObDeviceConfig> dump_configs;
|
||||
ObArray<ObDeviceConfig> load_configs;
|
||||
ObDeviceConfig config;
|
||||
ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_one_device_config(buf_log_id_, config));
|
||||
ASSERT_EQ(OB_SUCCESS, dump_configs.push_back(config));
|
||||
ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_one_device_config(buf_log_ram_url_, config));
|
||||
ASSERT_EQ(OB_SUCCESS, dump_configs.push_back(config));
|
||||
ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_one_device_config(buf_data_, config));
|
||||
ASSERT_EQ(OB_SUCCESS, dump_configs.push_back(config));
|
||||
|
||||
ObDeviceConfig config2 = config;
|
||||
MEMSET(config2.used_for_, 0, sizeof(config2.used_for_));
|
||||
STRCPY(config2.used_for_, "DATA");
|
||||
MEMSET(config2.old_access_info_, 0, sizeof(config2.old_access_info_));
|
||||
STRCPY(config2.old_access_info_, "access_id=aaa&encrypt_key=bbb");
|
||||
MEMSET(config2.access_info_, 0, sizeof(config2.access_info_));
|
||||
STRCPY(config2.access_info_, "access_id=ccccc&encrypt_key=ddddd");
|
||||
MEMSET(config2.old_extension_, 0, sizeof(config2.old_extension_));
|
||||
STRCPY(config2.old_extension_, "appid=123&checksum_type=crc32");
|
||||
MEMSET(config2.extension_, 0, sizeof(config2.extension_));
|
||||
STRCPY(config2.extension_, "appid=456&checksum_type=md5");
|
||||
config2.op_id_ = 3;
|
||||
config2.sub_op_id_ = 4;
|
||||
config2.storage_id_ = 2;
|
||||
ASSERT_EQ(OB_SUCCESS, dump_configs.push_back(config2));
|
||||
ObDeviceManifest::HeadSection dump_head;
|
||||
dump_head.version_ = 1;
|
||||
dump_head.device_num_ = 3;
|
||||
dump_head.device_num_ = 2;
|
||||
dump_head.modify_timestamp_us_ = ObTimeUtil::fast_current_time();
|
||||
dump_head.last_op_id_ = 3;
|
||||
dump_head.last_sub_op_id_ = 1;
|
||||
dump_head.last_sub_op_id_ = 4;
|
||||
ObDeviceManifest::HeadSection load_head;
|
||||
|
||||
const char *data_dir = "./TestDeviceConfigDir";
|
||||
ASSERT_EQ(OB_SUCCESS, FileDirectoryUtils::create_directory(data_dir));
|
||||
|
||||
ASSERT_EQ(OB_SUCCESS, device_manifest.init(data_dir));
|
||||
ASSERT_EQ(OB_SUCCESS, device_manifest.dump2file(dump_configs, dump_head));
|
||||
ASSERT_EQ(OB_SUCCESS, device_manifest.load(load_configs, load_head));
|
||||
|
||||
// sort by used_for, path and endpoint
|
||||
ObArray<ObDeviceConfig> sorted_dump_configs;
|
||||
ASSERT_EQ(OB_SUCCESS, sorted_dump_configs.push_back(dump_configs[2]));
|
||||
ASSERT_EQ(OB_SUCCESS, sorted_dump_configs.push_back(dump_configs[0]));
|
||||
ASSERT_EQ(OB_SUCCESS, sorted_dump_configs.push_back(dump_configs[1]));
|
||||
ASSERT_EQ(3, load_configs.count());
|
||||
ASSERT_EQ(OB_SUCCESS, sorted_dump_configs.push_back(dump_configs[0]));
|
||||
ASSERT_EQ(2, load_configs.count());
|
||||
for (int64_t i = 0; i < load_configs.count(); ++i) {
|
||||
ASSERT_EQ(0, STRCMP(sorted_dump_configs[i].used_for_, load_configs[i].used_for_));
|
||||
ASSERT_EQ(0, STRCMP(sorted_dump_configs[i].path_, load_configs[i].path_));
|
||||
@ -165,59 +153,58 @@ TEST_F(TestDeviceConfig, DISABLED_test_device_manifest)
|
||||
ASSERT_EQ(dump_head.last_op_id_, load_head.last_op_id_);
|
||||
ASSERT_EQ(dump_head.last_sub_op_id_, load_head.last_sub_op_id_);
|
||||
|
||||
// ASSERT_EQ(OB_SUCCESS, FileDirectoryUtils::delete_directory_rec(data_dir));
|
||||
}
|
||||
|
||||
TEST_F(TestDeviceConfig, DISABLED_test_device_config_mgr)
|
||||
{
|
||||
char data_dir[MAX_PATH_SIZE] = "./TestDeviceConfigDir";
|
||||
// char shared_storage_info[share::OB_MAX_BACKUP_DEST_LENGTH] = "file://./test_bukcet/";
|
||||
// ASSERT_EQ(OB_SUCCESS, FileDirectoryUtils::create_directory(data_dir));
|
||||
|
||||
// 3. test device config mgr
|
||||
ObDeviceConfigMgr &config_mgr = ObDeviceConfigMgr::get_instance();
|
||||
ASSERT_EQ(OB_SUCCESS, config_mgr.init(data_dir));
|
||||
ASSERT_EQ(OB_SUCCESS, config_mgr.load_configs());
|
||||
|
||||
// test get_all_device_configs
|
||||
ObArray<ObDeviceConfig> configs;
|
||||
ObDeviceConfig config;
|
||||
ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_one_device_config(buf_data_, config));
|
||||
ASSERT_EQ(OB_SUCCESS, configs.push_back(config));
|
||||
ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_one_device_config(buf_log_id_, config));
|
||||
ASSERT_EQ(OB_SUCCESS, configs.push_back(config));
|
||||
ASSERT_EQ(OB_SUCCESS, ObDeviceConfigParser::parse_one_device_config(buf_log_ram_url_, config));
|
||||
ASSERT_EQ(OB_SUCCESS, configs.push_back(config));
|
||||
ObArray<ObDeviceConfig> all_configs;
|
||||
ASSERT_EQ(OB_SUCCESS, config_mgr.get_all_device_configs(all_configs));
|
||||
ASSERT_EQ(3, all_configs.count());
|
||||
|
||||
// test get_device_configs
|
||||
char used_for_data[] = "DATA";
|
||||
ObArray<ObDeviceConfig> used_for_configs;
|
||||
ASSERT_EQ(OB_SUCCESS, config_mgr.get_device_configs(used_for_data, used_for_configs));
|
||||
ASSERT_EQ(1, used_for_configs.count());
|
||||
ASSERT_EQ(0, STRCMP(configs[0].used_for_, used_for_configs[0].used_for_));
|
||||
ASSERT_EQ(0, STRCMP(configs[0].path_, used_for_configs[0].path_));
|
||||
ASSERT_EQ(0, STRCMP(configs[0].endpoint_, used_for_configs[0].endpoint_));
|
||||
ASSERT_EQ(0, STRCMP(configs[0].access_info_, used_for_configs[0].access_info_));
|
||||
ASSERT_EQ(0, STRCMP(configs[0].encrypt_info_, used_for_configs[0].encrypt_info_));
|
||||
ASSERT_EQ(0, STRCMP(configs[0].extension_, used_for_configs[0].extension_));
|
||||
ASSERT_EQ(0, STRCMP(configs[0].old_access_info_, used_for_configs[0].old_access_info_));
|
||||
ASSERT_EQ(0, STRCMP(configs[0].old_encrypt_info_, used_for_configs[0].old_encrypt_info_));
|
||||
ASSERT_EQ(0, STRCMP(configs[0].old_extension_, used_for_configs[0].old_extension_));
|
||||
ASSERT_EQ(0, STRCMP(configs[0].state_, used_for_configs[0].state_));
|
||||
ASSERT_EQ(0, STRCMP(configs[0].state_info_, used_for_configs[0].state_info_));
|
||||
ASSERT_EQ(configs[0].create_timestamp_, used_for_configs[0].create_timestamp_);
|
||||
ASSERT_EQ(configs[0].last_check_timestamp_, used_for_configs[0].last_check_timestamp_);
|
||||
ASSERT_EQ(configs[0].op_id_, used_for_configs[0].op_id_);
|
||||
ASSERT_EQ(configs[0].sub_op_id_, used_for_configs[0].sub_op_id_);
|
||||
ASSERT_EQ(configs[0].storage_id_, used_for_configs[0].storage_id_);
|
||||
ASSERT_EQ(configs[0].max_iops_, used_for_configs[0].max_iops_);
|
||||
ASSERT_EQ(configs[0].max_bandwidth_, used_for_configs[0].max_bandwidth_);
|
||||
char used_for_log[] = "LOG";
|
||||
// used_for_configs.reuse();
|
||||
ASSERT_EQ(OB_SUCCESS, config_mgr.get_device_configs(used_for_log, used_for_configs));
|
||||
ASSERT_EQ(2, used_for_configs.count());
|
||||
ASSERT_EQ(2, all_configs.count());
|
||||
// test get_device_config
|
||||
ObDeviceConfig data_device_config;
|
||||
ASSERT_EQ(OB_SUCCESS, config_mgr.get_device_config(ObStorageUsedType::TYPE::USED_TYPE_DATA,
|
||||
data_device_config));
|
||||
ASSERT_EQ(0, STRCMP(config2.used_for_, data_device_config.used_for_));
|
||||
ASSERT_EQ(0, STRCMP(config2.path_, data_device_config.path_));
|
||||
ASSERT_EQ(0, STRCMP(config2.endpoint_, data_device_config.endpoint_));
|
||||
ASSERT_EQ(0, STRCMP(config2.access_info_, data_device_config.access_info_));
|
||||
ASSERT_EQ(0, STRCMP(config2.encrypt_info_, data_device_config.encrypt_info_));
|
||||
ASSERT_EQ(0, STRCMP(config2.extension_, data_device_config.extension_));
|
||||
ASSERT_EQ(0, STRCMP(config2.old_access_info_, data_device_config.old_access_info_));
|
||||
ASSERT_EQ(0, STRCMP(config2.old_encrypt_info_, data_device_config.old_encrypt_info_));
|
||||
ASSERT_EQ(0, STRCMP(config2.old_extension_, data_device_config.old_extension_));
|
||||
ASSERT_EQ(0, STRCMP(config2.state_, data_device_config.state_));
|
||||
ASSERT_EQ(0, STRCMP(config2.state_info_, data_device_config.state_info_));
|
||||
ASSERT_EQ(config2.create_timestamp_, data_device_config.create_timestamp_);
|
||||
ASSERT_EQ(config2.last_check_timestamp_, data_device_config.last_check_timestamp_);
|
||||
ASSERT_EQ(config2.op_id_, data_device_config.op_id_);
|
||||
ASSERT_EQ(config2.sub_op_id_, data_device_config.sub_op_id_);
|
||||
ASSERT_EQ(config2.storage_id_, data_device_config.storage_id_);
|
||||
ASSERT_EQ(config2.max_iops_, data_device_config.max_iops_);
|
||||
ASSERT_EQ(config2.max_bandwidth_, data_device_config.max_bandwidth_);
|
||||
ObDeviceConfig log_device_config;
|
||||
ASSERT_EQ(OB_SUCCESS, config_mgr.get_device_config(ObStorageUsedType::TYPE::USED_TYPE_LOG,
|
||||
log_device_config));
|
||||
ASSERT_EQ(0, STRCMP(config.used_for_, log_device_config.used_for_));
|
||||
ASSERT_EQ(0, STRCMP(config.path_, log_device_config.path_));
|
||||
ASSERT_EQ(0, STRCMP(config.endpoint_, log_device_config.endpoint_));
|
||||
ASSERT_EQ(0, STRCMP(config.access_info_, log_device_config.access_info_));
|
||||
ASSERT_EQ(0, STRCMP(config.encrypt_info_, log_device_config.encrypt_info_));
|
||||
ASSERT_EQ(0, STRCMP(config.extension_, log_device_config.extension_));
|
||||
ASSERT_EQ(0, STRCMP(config.old_access_info_, log_device_config.old_access_info_));
|
||||
ASSERT_EQ(0, STRCMP(config.old_encrypt_info_, log_device_config.old_encrypt_info_));
|
||||
ASSERT_EQ(0, STRCMP(config.old_extension_, log_device_config.old_extension_));
|
||||
ASSERT_EQ(0, STRCMP(config.state_, log_device_config.state_));
|
||||
ASSERT_EQ(0, STRCMP(config.state_info_, log_device_config.state_info_));
|
||||
ASSERT_EQ(config.create_timestamp_, log_device_config.create_timestamp_);
|
||||
ASSERT_EQ(config.last_check_timestamp_, log_device_config.last_check_timestamp_);
|
||||
ASSERT_EQ(config.op_id_, log_device_config.op_id_);
|
||||
ASSERT_EQ(config.sub_op_id_, log_device_config.sub_op_id_);
|
||||
ASSERT_EQ(config.storage_id_, log_device_config.storage_id_);
|
||||
ASSERT_EQ(config.max_iops_, log_device_config.max_iops_);
|
||||
ASSERT_EQ(config.max_bandwidth_, log_device_config.max_bandwidth_);
|
||||
|
||||
// test get_last_op_id and get_last_sub_op_id
|
||||
uint64_t last_op_id = UINT64_MAX;
|
||||
@ -225,17 +212,20 @@ TEST_F(TestDeviceConfig, DISABLED_test_device_config_mgr)
|
||||
ASSERT_EQ(OB_SUCCESS, config_mgr.get_last_op_id(last_op_id));
|
||||
ASSERT_EQ(OB_SUCCESS, config_mgr.get_last_sub_op_id(last_sub_op_id));
|
||||
ASSERT_EQ(3, last_op_id);
|
||||
ASSERT_EQ(1, last_sub_op_id);
|
||||
ASSERT_EQ(4, last_sub_op_id);
|
||||
|
||||
// test is_op_done
|
||||
bool is_done_3_1 = false;
|
||||
config_mgr.is_op_done(3, 1, is_done_3_1);
|
||||
ASSERT_TRUE(is_done_3_1);
|
||||
bool is_done_1_1 = false;
|
||||
config_mgr.is_op_done(1, 1, is_done_1_1);
|
||||
ASSERT_TRUE(is_done_1_1);
|
||||
bool is_done_3_4 = false;
|
||||
config_mgr.is_op_done(3, 4, is_done_3_4);
|
||||
ASSERT_TRUE(is_done_3_4);
|
||||
bool is_done_1_2 = false;
|
||||
config_mgr.is_op_done(1, 2, is_done_1_2);
|
||||
ASSERT_TRUE(is_done_1_2);
|
||||
bool is_done_3_5 = false;
|
||||
config_mgr.is_op_done(3, 5, is_done_3_5);
|
||||
ASSERT_FALSE(is_done_3_5);
|
||||
|
||||
// ASSERT_EQ(OB_SUCCESS, FileDirectoryUtils::delete_directory_rec(data_dir));
|
||||
ASSERT_EQ(OB_SUCCESS, FileDirectoryUtils::delete_directory_rec(data_dir));
|
||||
}
|
||||
|
||||
} // namespace unittest
|
||||
@ -245,7 +235,7 @@ int main(int argc, char **argv)
|
||||
{
|
||||
oceanbase::common::FileDirectoryUtils::delete_directory_rec("./TestDeviceConfigDir");
|
||||
system("rm -f test_device_config.log");
|
||||
OB_LOGGER.set_file_name("test_device_config.log");
|
||||
OB_LOGGER.set_file_name("test_device_config.log", true);
|
||||
OB_LOGGER.set_log_level("INFO");
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
return RUN_ALL_TESTS();
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user