From 866cb1d50e3db1e03d474004ca9b03a4c5264dbd Mon Sep 17 00:00:00 2001 From: obdev Date: Thu, 24 Oct 2024 15:14:00 +0000 Subject: [PATCH] fix cos timeout param --- deps/oblib/src/lib/restore/CMakeLists.txt | 3 +- .../src/lib/restore/cos/ob_cos_wrapper.cpp | 6 +- deps/oblib/src/lib/restore/cos/ob_singleton.h | 154 ---------------- deps/oblib/src/lib/restore/ob_i_storage.h | 1 + .../lib/restore/ob_object_storage_base.cpp | 49 ++++++ .../src/lib/restore/ob_object_storage_base.h | 166 ++++++++++++++++++ .../src/lib/restore/ob_storage_oss_base.h | 1 - .../src/lib/restore/ob_storage_s3_base.h | 1 - 8 files changed, 220 insertions(+), 161 deletions(-) create mode 100644 deps/oblib/src/lib/restore/ob_object_storage_base.cpp create mode 100644 deps/oblib/src/lib/restore/ob_object_storage_base.h diff --git a/deps/oblib/src/lib/restore/CMakeLists.txt b/deps/oblib/src/lib/restore/CMakeLists.txt index 40691479f..20aed1a04 100644 --- a/deps/oblib/src/lib/restore/CMakeLists.txt +++ b/deps/oblib/src/lib/restore/CMakeLists.txt @@ -17,7 +17,8 @@ oblib_add_library(restore OBJECT cos/ob_cos_wrapper_handle.h cos/ob_singleton.h ob_object_device.cpp - ob_object_device.h) + ob_object_device.h + ob_object_storage_base.cpp) add_library(oss INTERFACE) target_include_directories(oss diff --git a/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.cpp b/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.cpp index 089353759..def5d811b 100644 --- a/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.cpp +++ b/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.cpp @@ -27,15 +27,13 @@ #include "cos_crc64.h" #include "ob_cos_wrapper.h" +#include "lib/restore/ob_object_storage_base.h" namespace oceanbase { namespace common { -thread_local uint64_t ObObjectStorageTenantGuard::tl_tenant_id_ = OB_SERVER_TENANT_ID; -thread_local int64_t ObObjectStorageTenantGuard::tl_timeout_us_ = OB_STORAGE_MAX_IO_TIMEOUT_US; - namespace qcloud_cos { using namespace oceanbase::common; @@ -189,7 +187,7 @@ int ob_set_retry_headers( class ObStorageCOSRetryStrategy : public ObStorageIORetryStrategyBase { public: - ObStorageCOSRetryStrategy(const int64_t timeout_us = ObObjectStorageTenantGuard::tl_tenant_id_) + ObStorageCOSRetryStrategy(const int64_t timeout_us = ObObjectStorageTenantGuard::get_timeout_us()) : ObStorageIORetryStrategyBase(timeout_us), origin_headers_(nullptr), ref_headers_(nullptr), diff --git a/deps/oblib/src/lib/restore/cos/ob_singleton.h b/deps/oblib/src/lib/restore/cos/ob_singleton.h index fa9bb677f..c746c2b4a 100644 --- a/deps/oblib/src/lib/restore/cos/ob_singleton.h +++ b/deps/oblib/src/lib/restore/cos/ob_singleton.h @@ -49,160 +49,6 @@ private: } // qcloud_cos -static int64_t OB_STORAGE_MAX_IO_TIMEOUT_US = 20LL * 1000LL * 1000LL; // 20s -// disable the retry mechanism -static constexpr int64_t DO_NOT_RETRY = 0; - -// interface class, used for COS, -// as COS cannot utilize OB's code directly, to avoid compilation errors -template -class ObStorageIORetryStrategyBase -{ -public: - using RetType = OutcomeType_; - - ObStorageIORetryStrategyBase(const int64_t timeout_us) - : start_time_us_(0), // cannot use virtual func `current_time_us` in constructor - timeout_us_(timeout_us < 0 ? OB_STORAGE_MAX_IO_TIMEOUT_US : timeout_us) // 0 means never retry - {} - // virtual ~ObStorageIORetryStrategyBase() {} - void reuse(const int64_t timeout_us) - { - start_time_us_ = current_time_us(); - timeout_us_ = (timeout_us < 0 ? OB_STORAGE_MAX_IO_TIMEOUT_US : timeout_us); - } - virtual int64_t current_time_us() const = 0; - - bool should_retry(const RetType &outcome, const int64_t attempted_retries) const - { - bool bret = should_retry_impl_(outcome, attempted_retries); - if (bret && is_timeout_(attempted_retries)) { - bret = false; - } - return bret; - } - - virtual uint32_t calc_delay_time_us( - const RetType &outcome, const int64_t attempted_retries) const - { - static const uint32_t base_delay_us = 25 * 1000; // 25ms - return base_delay_us * (1 << attempted_retries); - } - - virtual void log_error( - const RetType &outcome, const int64_t attempted_retries) const = 0; - -protected: - virtual bool is_timeout_(const int64_t attempted_retries) const = 0; - - virtual bool should_retry_impl_( - const RetType &outcome, const int64_t attempted_retries) const = 0; - -protected: - int64_t start_time_us_; - int64_t timeout_us_; -}; - -// interface class, used for OSS & S3, -template -class ObStorageIORetryStrategy : public ObStorageIORetryStrategyBase -{ -public: - using typename ObStorageIORetryStrategyBase::RetType; - using ObStorageIORetryStrategyBase::start_time_us_; - using ObStorageIORetryStrategyBase::timeout_us_; - using ObStorageIORetryStrategyBase::calc_delay_time_us; - - ObStorageIORetryStrategy(const int64_t timeout_us) - : ObStorageIORetryStrategyBase(timeout_us) - { - start_time_us_ = ObTimeUtility::current_time(); - } - virtual ~ObStorageIORetryStrategy() {} - - virtual int64_t current_time_us() const override - { - return ObTimeUtility::current_time(); - } - -protected: - virtual bool is_timeout_(const int64_t attempted_retries) const override - { - bool bret = false; - const int64_t cur_time_us = current_time_us(); - if (cur_time_us >= start_time_us_ + timeout_us_) { - OB_LOG_RET(WARN, OB_TIMEOUT, "request reach time limit", K(start_time_us_), - K(timeout_us_), K(cur_time_us), K(attempted_retries)); - bret = true; - } - return bret; - } -}; - -template -using FuncRetType = typename std::result_of::type; - -template -FuncRetType execute_until_timeout( - ObStorageIORetryStrategyBase> &retry_strategy, - FuncType func, - Args... args) -{ - int64_t retries = 0; - bool should_retry_flag = true; - // func_ret may be pointer, so use {} construct it - FuncRetType func_ret {}; - do { - func_ret = func(args...); - if (!retry_strategy.should_retry(func_ret, retries)) { - should_retry_flag = false; - } else { - // if should_retry, log the current error - retry_strategy.log_error(func_ret, retries); - uint32_t sleep_time_us = retry_strategy.calc_delay_time_us(func_ret, retries); - ::usleep(sleep_time_us); - } - retries++; - } while (should_retry_flag); - - return func_ret; -} - -class ObObjectStorageTenantGuard -{ -public: - ObObjectStorageTenantGuard(const uint64_t tenant_id, const int64_t timeout_us) - : old_tenant_id_(tl_tenant_id_), - old_timeout_us_(tl_timeout_us_) - { - tl_tenant_id_ = tenant_id; - tl_timeout_us_ = timeout_us; - } - - virtual ~ObObjectStorageTenantGuard() - { - tl_tenant_id_ = old_tenant_id_; - tl_timeout_us_ = old_timeout_us_; - } - - static uint64_t get_tenant_id() - { - return tl_tenant_id_; - } - - static int64_t get_timeout_us() - { - return tl_timeout_us_; - } - -public: - static thread_local uint64_t tl_tenant_id_; - uint64_t old_tenant_id_; - - static thread_local int64_t tl_timeout_us_; - int64_t old_timeout_us_; -}; - } // common } // oceanbase diff --git a/deps/oblib/src/lib/restore/ob_i_storage.h b/deps/oblib/src/lib/restore/ob_i_storage.h index 47317d54d..8a3dc166c 100644 --- a/deps/oblib/src/lib/restore/ob_i_storage.h +++ b/deps/oblib/src/lib/restore/ob_i_storage.h @@ -19,6 +19,7 @@ #include "lib/container/ob_se_array.h" #include "common/storage/ob_device_common.h" #include "ob_storage_info.h" +#include "ob_object_storage_base.h" namespace oceanbase { diff --git a/deps/oblib/src/lib/restore/ob_object_storage_base.cpp b/deps/oblib/src/lib/restore/ob_object_storage_base.cpp new file mode 100644 index 000000000..d481d1f8d --- /dev/null +++ b/deps/oblib/src/lib/restore/ob_object_storage_base.cpp @@ -0,0 +1,49 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "ob_object_storage_base.h" + +namespace oceanbase +{ +namespace common +{ + +thread_local uint64_t ObObjectStorageTenantGuard::tl_tenant_id_ = OB_SERVER_TENANT_ID; +thread_local int64_t ObObjectStorageTenantGuard::tl_timeout_us_ = OB_STORAGE_MAX_IO_TIMEOUT_US; + +ObObjectStorageTenantGuard::ObObjectStorageTenantGuard( + const uint64_t tenant_id, const int64_t timeout_us) + : old_tenant_id_(tl_tenant_id_), + old_timeout_us_(tl_timeout_us_) +{ + tl_tenant_id_ = tenant_id; + tl_timeout_us_ = timeout_us; +} + +ObObjectStorageTenantGuard::~ObObjectStorageTenantGuard() +{ + tl_tenant_id_ = old_tenant_id_; + tl_timeout_us_ = old_timeout_us_; +} + +uint64_t ObObjectStorageTenantGuard::get_tenant_id() +{ + return tl_tenant_id_; +} + +int64_t ObObjectStorageTenantGuard::get_timeout_us() +{ + return tl_timeout_us_; +} + +} // common +} // oceanbase \ No newline at end of file diff --git a/deps/oblib/src/lib/restore/ob_object_storage_base.h b/deps/oblib/src/lib/restore/ob_object_storage_base.h new file mode 100644 index 000000000..ea17029af --- /dev/null +++ b/deps/oblib/src/lib/restore/ob_object_storage_base.h @@ -0,0 +1,166 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef SRC_LIBRARY_SRC_LIB_RESTORE_OB_OBJECT_STORAGE_BASE_H_ +#define SRC_LIBRARY_SRC_LIB_RESTORE_OB_OBJECT_STORAGE_BASE_H_ + +#include "lib/ob_define.h" +#include "lib/string/ob_string.h" +#include "lib/utility/utility.h" + +namespace oceanbase +{ +namespace common +{ + +static int64_t OB_STORAGE_MAX_IO_TIMEOUT_US = 20LL * 1000LL * 1000LL; // 20s +// disable the retry mechanism +static constexpr int64_t DO_NOT_RETRY = 0; + +// interface class, used for COS, +// as COS cannot utilize OB's code directly, to avoid compilation errors +template +class ObStorageIORetryStrategyBase +{ +public: + using RetType = OutcomeType_; + + ObStorageIORetryStrategyBase(const int64_t timeout_us) + : start_time_us_(0), // cannot use virtual func `current_time_us` in constructor + timeout_us_(timeout_us < 0 ? OB_STORAGE_MAX_IO_TIMEOUT_US : timeout_us) // 0 means never retry + {} + + // A virtual destructor can lead to link failures, temporarily commented out + // virtual ~ObStorageIORetryStrategyBase() {} + void reuse(const int64_t timeout_us) + { + start_time_us_ = current_time_us(); + timeout_us_ = (timeout_us < 0 ? OB_STORAGE_MAX_IO_TIMEOUT_US : timeout_us); + } + virtual int64_t current_time_us() const = 0; + + bool should_retry(const RetType &outcome, const int64_t attempted_retries) const + { + bool bret = should_retry_impl_(outcome, attempted_retries); + if (bret && is_timeout_(attempted_retries)) { + bret = false; + } + return bret; + } + + virtual uint32_t calc_delay_time_us( + const RetType &outcome, const int64_t attempted_retries) const + { + static const uint32_t base_delay_us = 25 * 1000; // 25ms + return base_delay_us * (1 << attempted_retries); + } + + virtual void log_error( + const RetType &outcome, const int64_t attempted_retries) const = 0; + +protected: + virtual bool is_timeout_(const int64_t attempted_retries) const = 0; + + virtual bool should_retry_impl_( + const RetType &outcome, const int64_t attempted_retries) const = 0; + +protected: + int64_t start_time_us_; + int64_t timeout_us_; +}; + +// interface class, used for OSS & S3, +template +class ObStorageIORetryStrategy : public ObStorageIORetryStrategyBase +{ +public: + using typename ObStorageIORetryStrategyBase::RetType; + using ObStorageIORetryStrategyBase::start_time_us_; + using ObStorageIORetryStrategyBase::timeout_us_; + using ObStorageIORetryStrategyBase::calc_delay_time_us; + + ObStorageIORetryStrategy(const int64_t timeout_us) + : ObStorageIORetryStrategyBase(timeout_us) + { + start_time_us_ = ObTimeUtility::current_time(); + } + virtual ~ObStorageIORetryStrategy() {} + + virtual int64_t current_time_us() const override + { + return ObTimeUtility::current_time(); + } + +protected: + virtual bool is_timeout_(const int64_t attempted_retries) const override + { + bool bret = false; + const int64_t cur_time_us = current_time_us(); + if (cur_time_us >= start_time_us_ + timeout_us_) { + OB_LOG_RET(WARN, OB_TIMEOUT, "request reach time limit", K(start_time_us_), + K(timeout_us_), K(cur_time_us), K(attempted_retries)); + bret = true; + } + return bret; + } +}; + +template +using FuncRetType = typename std::result_of::type; + +template +FuncRetType execute_until_timeout( + ObStorageIORetryStrategyBase> &retry_strategy, + FuncType func, + Args... args) +{ + int64_t retries = 0; + bool should_retry_flag = true; + // func_ret may be pointer, so use {} construct it + FuncRetType func_ret {}; + do { + func_ret = func(args...); + if (!retry_strategy.should_retry(func_ret, retries)) { + should_retry_flag = false; + } else { + // if should_retry, log the current error + retry_strategy.log_error(func_ret, retries); + uint32_t sleep_time_us = retry_strategy.calc_delay_time_us(func_ret, retries); + ::usleep(sleep_time_us); + } + retries++; + } while (should_retry_flag); + + return func_ret; +} + +class ObObjectStorageTenantGuard +{ +public: + ObObjectStorageTenantGuard(const uint64_t tenant_id, const int64_t timeout_us); + virtual ~ObObjectStorageTenantGuard(); + + static uint64_t get_tenant_id(); + static int64_t get_timeout_us(); + +private: + static thread_local uint64_t tl_tenant_id_; + uint64_t old_tenant_id_; + + static thread_local int64_t tl_timeout_us_; + int64_t old_timeout_us_; +}; + +} // common +} // oceanbase + +#endif // SRC_LIBRARY_SRC_LIB_RESTORE_OB_OBJECT_STORAGE_BASE_H_ \ No newline at end of file diff --git a/deps/oblib/src/lib/restore/ob_storage_oss_base.h b/deps/oblib/src/lib/restore/ob_storage_oss_base.h index c881db59d..7462cc862 100644 --- a/deps/oblib/src/lib/restore/ob_storage_oss_base.h +++ b/deps/oblib/src/lib/restore/ob_storage_oss_base.h @@ -31,7 +31,6 @@ #include "lib/compress/ob_compressor_pool.h" #include "ob_i_storage.h" #include "common/storage/ob_device_common.h" -#include "cos/ob_singleton.h" #include "lib/container/ob_se_array.h" namespace oceanbase diff --git a/deps/oblib/src/lib/restore/ob_storage_s3_base.h b/deps/oblib/src/lib/restore/ob_storage_s3_base.h index 2ea126882..f2cbbb2af 100644 --- a/deps/oblib/src/lib/restore/ob_storage_s3_base.h +++ b/deps/oblib/src/lib/restore/ob_storage_s3_base.h @@ -52,7 +52,6 @@ #include #include #pragma pop_macro("private") -#include "cos/ob_singleton.h" namespace oceanbase {