From f1a0605889097065152b3ffef3b452eefede47fd Mon Sep 17 00:00:00 2001 From: nautaa <870284156@qq.com> Date: Tue, 29 Oct 2024 06:44:52 +0000 Subject: [PATCH] [CP] [CP] add option to configure s3 addressing model --- .../oblib/src/lib/restore/ob_storage_info.cpp | 20 +++++++++++++++++++ deps/oblib/src/lib/restore/ob_storage_info.h | 10 ++++++++++ .../src/lib/restore/ob_storage_s3_base.cpp | 14 +++++++++++++ .../src/lib/restore/ob_storage_s3_base.h | 3 ++- .../lib/restore/test_storage_info.cpp | 20 +++++++++++++++++++ 5 files changed, 66 insertions(+), 1 deletion(-) diff --git a/deps/oblib/src/lib/restore/ob_storage_info.cpp b/deps/oblib/src/lib/restore/ob_storage_info.cpp index f1486284b..f25e1ca13 100644 --- a/deps/oblib/src/lib/restore/ob_storage_info.cpp +++ b/deps/oblib/src/lib/restore/ob_storage_info.cpp @@ -291,6 +291,12 @@ int ObObjectStorageInfo::parse_storage_info_(const char *storage_info, bool &has } else if (OB_FAIL(set_storage_info_field_(token, extension_, sizeof(extension_)))) { LOG_WARN("failed to set delete mode", K(ret), K(token)); } + } else if (0 == strncmp(ADDRESSING_MODEL, token, strlen(ADDRESSING_MODEL))) { + if (OB_FAIL(check_addressing_model_(token + strlen(ADDRESSING_MODEL)))) { + OB_LOG(WARN, "failed to check addressing model", K(ret), K(token)); + } else if (OB_FAIL(set_storage_info_field_(token, extension_, sizeof(extension_)))) { + LOG_WARN("failed to set addressing model", K(ret), K(token)); + } } else if (0 == strncmp(CHECKSUM_TYPE, token, strlen(CHECKSUM_TYPE))) { const char *checksum_type_str = token + strlen(CHECKSUM_TYPE); if (OB_FAIL(set_checksum_type_(checksum_type_str))) { @@ -322,6 +328,20 @@ int ObObjectStorageInfo::check_delete_mode_(const char *delete_mode) return ret; } +int ObObjectStorageInfo::check_addressing_model_(const char *addressing_model) const +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(addressing_model)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid args", K(ret), KP(addressing_model)); + } else if (0 != strcmp(addressing_model, ADDRESSING_MODEL_VIRTUAL_HOSTED_STYLE) + && 0 != strcmp(addressing_model, ADDRESSING_MODEL_PATH_STYLE)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "addressing model is invalid", K(ret), K(addressing_model)); + } + return ret; +} + bool is_oss_supported_checksum(const ObStorageChecksumType checksum_type) { return checksum_type == ObStorageChecksumType::OB_NO_CHECKSUM_ALGO diff --git a/deps/oblib/src/lib/restore/ob_storage_info.h b/deps/oblib/src/lib/restore/ob_storage_info.h index e57c235cb..eeda8d4f9 100644 --- a/deps/oblib/src/lib/restore/ob_storage_info.h +++ b/deps/oblib/src/lib/restore/ob_storage_info.h @@ -42,11 +42,20 @@ const char *const REGION = "s3_region="; const char *const MAX_IOPS = "max_iops="; const char *const MAX_BANDWIDTH = "max_bandwidth="; +const char *const ADDRESSING_MODEL = "addressing_model="; +const char *const ADDRESSING_MODEL_VIRTUAL_HOSTED_STYLE = "virtual_hosted_style"; +const char *const ADDRESSING_MODEL_PATH_STYLE = "path_style"; + const char *const CHECKSUM_TYPE = "checksum_type="; const char *const CHECKSUM_TYPE_NO_CHECKSUM = "no_checksum"; const char *const CHECKSUM_TYPE_MD5 = "md5"; const char *const CHECKSUM_TYPE_CRC32 = "crc32"; +enum ObStorageAddressingModel +{ + OB_VIRTUAL_HOSTED_STYLE = 0, + OB_PATH_STYLE = 1, +}; enum ObStorageChecksumType : uint8_t { OB_NO_CHECKSUM_ALGO = 0, @@ -98,6 +107,7 @@ protected: virtual int get_access_key_(char *key_buf, const int64_t key_buf_len) const; virtual int parse_storage_info_(const char *storage_info, bool &has_appid); int check_delete_mode_(const char *delete_mode); + int check_addressing_model_(const char *addressing_model) const; int set_checksum_type_(const char *checksum_type_str); int set_storage_info_field_(const char *info, char *field, const int64_t length); diff --git a/deps/oblib/src/lib/restore/ob_storage_s3_base.cpp b/deps/oblib/src/lib/restore/ob_storage_s3_base.cpp index 752585382..107d52c57 100644 --- a/deps/oblib/src/lib/restore/ob_storage_s3_base.cpp +++ b/deps/oblib/src/lib/restore/ob_storage_s3_base.cpp @@ -208,6 +208,9 @@ int ObS3Client::init_s3_client_configuration_(const ObS3Account &account, config.payloadSigningPolicy = Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never; config.endpointOverride = account.endpoint_; config.executor = nullptr; + if (account.addressing_model_ == ObStorageAddressingModel::OB_PATH_STYLE) { + config.useVirtualAddressing = false; + } // Default maxRetries is 10 std::shared_ptr retryStrategy = @@ -1022,6 +1025,7 @@ void ObS3Account::reset() MEMSET(endpoint_, 0, sizeof(endpoint_)); MEMSET(access_id_, 0, sizeof(access_id_)); MEMSET(secret_key_, 0, sizeof(secret_key_)); + addressing_model_ = ObStorageAddressingModel::OB_VIRTUAL_HOSTED_STYLE; } int64_t ObS3Account::hash() const @@ -1031,6 +1035,7 @@ int64_t ObS3Account::hash() const hash_value = murmurhash(endpoint_, static_cast(strlen(endpoint_)), hash_value); hash_value = murmurhash(access_id_, static_cast(strlen(access_id_)), hash_value); hash_value = murmurhash(secret_key_, static_cast(strlen(secret_key_)), hash_value); + hash_value = murmurhash(&addressing_model_, sizeof(addressing_model_), hash_value); return hash_value; } @@ -1084,6 +1089,15 @@ int ObS3Account::parse_from(const char *storage_info_str, const int64_t size) ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "delete mode is invalid", K(ret), KCSTRING(token)); } + } else if (0 == strncmp(ADDRESSING_MODEL, token, strlen(ADDRESSING_MODEL))) { + if (0 == strcmp(token + strlen(ADDRESSING_MODEL), ADDRESSING_MODEL_VIRTUAL_HOSTED_STYLE)) { + addressing_model_ = ObStorageAddressingModel::OB_VIRTUAL_HOSTED_STYLE; + } else if (0 == strcmp(token + strlen(ADDRESSING_MODEL), ADDRESSING_MODEL_PATH_STYLE)) { + addressing_model_ = ObStorageAddressingModel::OB_PATH_STYLE; + } else { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "addressing model is invalid", K(ret), KCSTRING(token)); + } } else { OB_LOG(DEBUG, "unknown s3 info", K(*token)); } diff --git a/deps/oblib/src/lib/restore/ob_storage_s3_base.h b/deps/oblib/src/lib/restore/ob_storage_s3_base.h index f2cbbb2af..37d00e689 100644 --- a/deps/oblib/src/lib/restore/ob_storage_s3_base.h +++ b/deps/oblib/src/lib/restore/ob_storage_s3_base.h @@ -136,7 +136,7 @@ struct ObS3Account void reset(); bool is_valid() const { return is_valid_; } int64_t hash() const; - TO_STRING_KV(K_(is_valid), K_(delete_mode), K_(region), K_(endpoint), K_(access_id)); + TO_STRING_KV(K_(is_valid), K_(delete_mode), K_(region), K_(endpoint), K_(access_id), K_(addressing_model)); int parse_from(const char *storage_info_str, const int64_t size); int set_field(const char *value, char *field, const uint32_t field_length); @@ -147,6 +147,7 @@ struct ObS3Account char endpoint_[MAX_S3_ENDPOINT_LENGTH]; char access_id_[MAX_S3_ACCESS_ID_LENGTH]; // ak char secret_key_[MAX_S3_SECRET_KEY_LENGTH]; // sk + ObStorageAddressingModel addressing_model_; }; class ObS3MemoryManager : public Aws::Utils::Memory::MemorySystemInterface diff --git a/deps/oblib/unittest/lib/restore/test_storage_info.cpp b/deps/oblib/unittest/lib/restore/test_storage_info.cpp index 59b456898..9bf76ad1b 100644 --- a/deps/oblib/unittest/lib/restore/test_storage_info.cpp +++ b/deps/oblib/unittest/lib/restore/test_storage_info.cpp @@ -187,6 +187,26 @@ TEST(ObObjectStorageInfo, s3) ASSERT_EQ(OB_INVALID_ARGUMENT, info1.set(uri, storage_info)); } +TEST(ObObjectStorageInfo, s3_compatible) +{ + const char *uri = "s3://backup_dir?host=xxx.com&access_id=111&access_key=222&s3_region=333"; + ObObjectStorageInfo info1; + + const char *storage_info = ""; + storage_info = "host=xxx.com&access_id=111&access_key=222&s3_region=333&delete_mode=delete&addressing_model="; + info1.reset(); + ASSERT_EQ(OB_INVALID_ARGUMENT, info1.set(uri, storage_info)); + storage_info = "host=xxx.com&access_id=111&access_key=222&s3_region=333&delete_mode=delete&addressing_model=path_style"; + info1.reset(); + ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info)); + storage_info = "host=xxx.com&access_id=111&access_key=222&s3_region=333&addressing_model=virtual_hosted_style"; + info1.reset(); + ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info)); + storage_info = "host=xxx.com&access_id=111&access_key=222&s3_region=333&addressing_model=xxx"; + info1.reset(); + ASSERT_EQ(OB_INVALID_ARGUMENT, info1.set(uri, storage_info)); +} + int main(int argc, char **argv) { system("rm -f test_storage_info.log*");