diff --git a/deps/oblib/src/lib/restore/ob_i_storage.h b/deps/oblib/src/lib/restore/ob_i_storage.h index b9aee015a..ccc9f7c52 100644 --- a/deps/oblib/src/lib/restore/ob_i_storage.h +++ b/deps/oblib/src/lib/restore/ob_i_storage.h @@ -58,7 +58,6 @@ int handle_listed_object(ObBaseDirEntryOperator &op, const char *obj_name, const int64_t obj_name_len, const int64_t obj_size); int handle_listed_directory(ObBaseDirEntryOperator &op, const char *dir_name, const int64_t dir_name_len); -int get_storage_prefix_from_path(const common::ObString &uri, const char *&prefix); int build_bucket_and_object_name(ObIAllocator &allocator, const ObString &uri, ObString &bucket, ObString &object); int construct_fragment_full_name(const ObString &logical_appendable_object_name, diff --git a/src/share/config/ob_config_helper.cpp b/src/share/config/ob_config_helper.cpp index d062dc081..15d758415 100644 --- a/src/share/config/ob_config_helper.cpp +++ b/src/share/config/ob_config_helper.cpp @@ -1427,5 +1427,28 @@ bool ObConfigRegexpEngineChecker::check(const ObConfigItem &t) const return valid; } +bool ObConfigS3URLEncodeTypeChecker::check(const ObConfigItem &t) const +{ + // When compliantRfc3986Encoding is set to true: + // - Adhere to RFC 3986 by supporting the encoding of reserved characters + // such as '-', '_', '.', '$', '@', etc. + // - This approach mitigates inconsistencies in server behavior when accessing + // COS using the S3 SDK. + // Otherwise, the reserved characters will not be encoded, + // following the default behavior of the S3 SDK. + bool bret = false; + common::ObString tmp_str(t.str()); + if (0 == tmp_str.case_compare("default")) { + bret = true; + Aws::Http::SetCompliantRfc3986Encoding(false); + } else if (0 == tmp_str.case_compare("compliantRfc3986Encoding")) { + bret = true; + Aws::Http::SetCompliantRfc3986Encoding(true); + } else { + bret = false; + } + return bret; +} + } // end of namepace common } // end of namespace oceanbase diff --git a/src/share/config/ob_config_helper.h b/src/share/config/ob_config_helper.h index 6e46d09f5..17cb18a71 100644 --- a/src/share/config/ob_config_helper.h +++ b/src/share/config/ob_config_helper.h @@ -922,6 +922,16 @@ private: DISALLOW_COPY_AND_ASSIGN(ObConfigRegexpEngineChecker); }; +class ObConfigS3URLEncodeTypeChecker : public ObConfigChecker +{ +public: + ObConfigS3URLEncodeTypeChecker() {} + virtual ~ObConfigS3URLEncodeTypeChecker() {} + virtual bool check(const ObConfigItem &t) const override; +private: + DISALLOW_COPY_AND_ASSIGN(ObConfigS3URLEncodeTypeChecker); +}; + typedef __ObConfigContainer ObConfigContainer; diff --git a/src/share/ob_device_manager.cpp b/src/share/ob_device_manager.cpp index c789c1551..1503f6d16 100644 --- a/src/share/ob_device_manager.cpp +++ b/src/share/ob_device_manager.cpp @@ -59,6 +59,17 @@ int ObDeviceManager::init_devices_env() OB_LOG(WARN, "fail to init cos storage", K(ret)); } else if (OB_FAIL(init_s3_env())) { OB_LOG(WARN, "fail to init s3 storage", K(ret)); + } else { + // When compliantRfc3986Encoding is set to true: + // - Adhere to RFC 3986 by supporting the encoding of reserved characters + // such as '-', '_', '.', '$', '@', etc. + // - This approach mitigates inconsistencies in server behavior when accessing + // COS using the S3 SDK. + // Otherwise, the reserved characters will not be encoded, + // following the default behavior of the S3 SDK. + const bool compliantRfc3986Encoding = + (0 == ObString(GCONF.ob_storage_s3_url_encode_type).case_compare("compliantRfc3986Encoding")); + Aws::Http::SetCompliantRfc3986Encoding(compliantRfc3986Encoding); } } diff --git a/src/share/parameter/ob_parameter_seed.ipp b/src/share/parameter/ob_parameter_seed.ipp index e67ffb7e9..1af31ce31 100644 --- a/src/share/parameter/ob_parameter_seed.ipp +++ b/src/share/parameter/ob_parameter_seed.ipp @@ -2172,3 +2172,10 @@ DEF_INT(ob_vector_memory_limit_percentage, OB_TENANT_PARAMETER, "0", "[0,100)", "Used to control the upper limit percentage of memory resources that the vector_index module can use. Range:[0, 100)", ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); + +DEF_STR_WITH_CHECKER(ob_storage_s3_url_encode_type, OB_CLUSTER_PARAMETER, "default", + common::ObConfigS3URLEncodeTypeChecker, + "Determines the URL encoding method for S3 requests." + "\"default\": Uses the S3 standard URL encoding method." + "\"compliantRfc3986Encoding\": Uses URL encoding that adheres to the RFC 3986 standard.", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result index c4dedf0ef..ac470e6f5 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result @@ -188,6 +188,7 @@ ob_query_switch_leader_retry_timeout ob_ratelimit_stat_period ob_ssl_invited_common_names ob_startup_mode +ob_storage_s3_url_encode_type ob_vector_memory_limit_percentage open_cursors optimizer_index_cost_adj