[FEAT MERGE]Added object storage COS as the backup destination

deps/init/oceanbase.el7.aarch64.deps

@@ -27,6 +27,7 @@ devdeps-ncurses-static-6.2-72022100815.el7.aarch64.rpm
 devdeps-boost-1.74.0-22022110914.el7.aarch64.rpm
 devdeps-s2geometry-0.9.0-42022111116.el7.aarch64.rpm
 devdeps-icu-69.1-72022112416.el7.aarch64.rpm
+devdeps-cos-c-sdk-5.0.16-52023070517.el7.aarch64.rpm
 
 [tools]
 obdevtools-binutils-2.30-12022100413.el7.aarch64.rpm

deps/init/oceanbase.el7.x86_64.deps

@@ -29,6 +29,7 @@ devdeps-ncurses-static-6.2-72022100815.el7.x86_64.rpm
 devdeps-boost-1.74.0-22022110914.el7.x86_64.rpm
 devdeps-s2geometry-0.9.0-42022111116.el7.x86_64.rpm
 devdeps-icu-69.1-72022112416.el7.x86_64.rpm
+devdeps-cos-c-sdk-5.0.16-52023070517.el7.x86_64.rpm
 
 [tools]
 obdevtools-binutils-2.30-12022100413.el7.x86_64.rpm

deps/init/oceanbase.el8.aarch64.deps

@@ -27,6 +27,7 @@ devdeps-ncurses-static-6.2-72022100815.el8.aarch64.rpm
 devdeps-boost-1.74.0-22022110914.el8.aarch64.rpm
 devdeps-s2geometry-0.9.0-42022111116.el8.aarch64.rpm
 devdeps-icu-69.1-72022112416.el8.aarch64.rpm
+devdeps-cos-c-sdk-5.0.16-52023070517.el8.aarch64.rpm
 
 [tools]
 obdevtools-binutils-2.30-12022100413.el8.aarch64.rpm

deps/init/oceanbase.el8.x86_64.deps

@@ -28,6 +28,7 @@ devdeps-ncurses-static-6.2-72022100815.el8.x86_64.rpm
 devdeps-boost-1.74.0-22022110914.el8.x86_64.rpm
 devdeps-s2geometry-0.9.0-42022111116.el8.x86_64.rpm
 devdeps-icu-69.1-72022112416.el8.x86_64.rpm
+devdeps-cos-c-sdk-5.0.16-52023070517.el8.x86_64.rpm
 
 [tools]
 obdevtools-binutils-2.30-12022100413.el8.x86_64.rpm

deps/oblib/src/lib/restore/CMakeLists.txt

@@ -6,8 +6,15 @@ oblib_add_library(restore OBJECT
   ob_storage.h
   ob_storage_oss_base.cpp
   ob_storage_oss_base.h
+  ob_storage_cos_base.cpp
+  ob_storage_cos_base.h
   ob_storage_path.cpp
   ob_storage_path.h
+  ob_storage_info.cpp
+  ob_storage_info.h
+  cos/ob_cos_wrapper_handle.cpp
+  cos/ob_cos_wrapper_handle.h
+  cos/ob_singleton.h
   ob_object_device.cpp
   ob_object_device.h)
 
@@ -28,4 +35,6 @@ target_link_libraries(oss
   ${DEP_DIR}/lib/libmxml.a)
 
 
-target_link_libraries(restore PUBLIC oss  oblib_base)
+target_link_libraries(restore PUBLIC oss cos_sdk oblib_base)
+
+add_subdirectory(cos)

deps/oblib/src/lib/restore/cos/CMakeLists.txt

@@ -0,0 +1,35 @@
+add_library(cos_wrapper OBJECT
+  ob_singleton.h
+  ob_cos_wrapper.h
+  ob_cos_wrapper.cpp)
+
+target_include_directories(cos_wrapper
+  PUBLIC
+  ${DEP_DIR}/include/apr-1
+  ${DEP_DIR}/include
+  ${DEP_DIR}/include/cos_c_sdk
+  ${DEP_DIR}/include/mxml
+)
+target_link_libraries(cos_wrapper PUBLIC oblib_base_base)
+
+# we need to get all .o files from cos-c-sdk rpm and mixed them into one .o file
+add_custom_command(
+  OUTPUT cos_sdk_lib.o
+  COMMAND rm -rf ${CMAKE_CURRENT_BINARY_DIR}/cos_a2o && mkdir ${CMAKE_CURRENT_BINARY_DIR}/cos_a2o && cp ${DEP_DIR}/lib/libcos_c_sdk_static.a ${CMAKE_CURRENT_BINARY_DIR}/cos_a2o && cd ${CMAKE_CURRENT_BINARY_DIR}/cos_a2o && ar x libcos_c_sdk_static.a && ld -r cos_*.o -o cos_sdk_lib.o
+  DEPENDS ${DEP_DIR}/lib/libcos_c_sdk_static.a
+)
+
+# we need to join cos-c-sdk .o file with cos_wrapper .o file
+# cuz we need to set 'localize-hidden', which can help cos_wrapper to get hidden symbol of cos-c-sdk.
+add_custom_command(
+  OUTPUT cos_sdk_objs.o
+  COMMAND ld -r $<TARGET_OBJECTS:cos_wrapper> -o cos_wrapper.o
+  COMMAND ld -r cos_wrapper.o ${CMAKE_CURRENT_BINARY_DIR}/cos_a2o/cos_sdk_lib.o -o cos_sdk_objs.tmp.o
+  COMMAND objcopy --localize-hidden cos_sdk_objs.tmp.o cos_sdk_objs.o
+  DEPENDS cos_wrapper cos_sdk_lib.o
+  BYPRODUCTS cos_sdk_objs.o
+  COMMAND_EXPAND_LISTS
+)
+
+oblib_add_extra_objects(${CMAKE_CURRENT_BINARY_DIR}/cos_sdk_objs.o)
+oblib_add_library(cos_sdk cos_sdk_objs.o)

deps/oblib/src/lib/restore/cos/ob_cos_wrapper.h

@@ -0,0 +1,325 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef SRC_LIBRARY_SRC_LIB_OB_COS_WRAPPER_H_
+#define SRC_LIBRARY_SRC_LIB_OB_COS_WRAPPER_H_
+
+#include <string.h>
+#include "ob_singleton.h"
+#include <dirent.h>
+
+namespace oceanbase
+{
+namespace common
+{
+
+// Naming qcloud_cos to avoid conflicts with the function called cos.
+namespace qcloud_cos
+{
+
+#define OB_PUBLIC_API __attribute__ ((visibility ("default")))
+
+class OB_PUBLIC_API ObCosEnv : public ObSingleton<ObCosEnv>
+{
+public:
+  struct Conf
+  {
+    static const int64_t MIN_COS_SLICE_SIZE = 32000;
+    static const int64_t MAX_COS_SLICE_SIZE = 10000000;
+    int64_t slice_size = 524288; // default 512K
+  };
+
+  ObCosEnv() : ObSingleton<ObCosEnv>(), is_inited_(false) {}
+  // global init cos env resource, must and only can be called once
+  int init();
+
+  void destroy();
+
+private:
+  bool is_inited_;
+  Conf conf_;
+};
+
+static constexpr int MAX_TAGGING_STR_LEN = 16;
+
+// COS domain name structure: bucket_name-appid.endpoint
+struct OB_PUBLIC_API ObCosAccount
+{
+  // max domain length
+  static constexpr int MAX_COS_DOMAIN_LENGTH = 1536;
+  // max endpoint length
+  static constexpr int MAX_COS_ENDPOINT_LENGTH = 128;
+  // max access id length
+  static constexpr int MAX_COS_ACCESS_ID_LENGTH = 128;
+  // max access key length
+  static constexpr int MAX_COS_ACCESS_KEY_LENGTH = 128;
+  // max appid length
+  static constexpr int MAX_COS_APPID_LENGTH = 128;
+
+  // cos endpoint
+  char endpoint_[MAX_COS_ENDPOINT_LENGTH];
+  // your access id
+  char access_id_[MAX_COS_ACCESS_ID_LENGTH];
+  // your access key
+  char access_key_[MAX_COS_ACCESS_KEY_LENGTH];
+  // your appid
+  char appid_[MAX_COS_APPID_LENGTH];
+  // cos object delete mode
+  char delete_mode_[MAX_TAGGING_STR_LEN];
+
+  ObCosAccount()
+  {
+    memset(this, 0, sizeof(*this));
+  }
+
+  ~ObCosAccount() {}
+
+  void clear()
+  {
+    memset(this, 0, sizeof(*this));
+  }
+
+  // parse endpoint, access id, access key and appid from storage_info
+  // You must call parse_from first before using any field.
+  int parse_from(const char *storage_info, uint32_t size);
+
+private:
+  // make sure 'value' end with '\0'
+  int set_field(const char *value, char *field, uint32_t length);
+};
+
+
+// The string does not own the buffer.
+struct OB_PUBLIC_API CosStringBuffer
+{
+  const char *data_;
+  int32_t size_;
+
+  CosStringBuffer() : data_(NULL), size_(0) {}
+
+  CosStringBuffer(const char *ptr, int len) : data_(ptr), size_(len) {}
+
+  ~CosStringBuffer() {}
+
+  bool empty() const
+  {
+    return NULL == data_ || 0 >= size_;
+  }
+
+  bool prefix_match(const char *str) const
+  {
+    bool match = false;
+    int32_t str_len = 0;
+    if (NULL != str) {
+      str_len = static_cast<int32_t>(strlen(str));
+    }
+
+    if (data_ == str) {
+      match = (size_ >= str_len ? true : false);
+    } else if (size_ < str_len) {
+      // skip
+    } else if (0 == strncasecmp(data_, str, str_len)) {
+      match = true;
+    }
+    return match;
+  }
+};
+
+
+struct OB_PUBLIC_API CosObjectMeta
+{
+  enum CosObjectType
+  {
+    COS_OBJ_INVALID,
+    COS_OBJ_NORMAL,
+    COS_OBJ_APPENDABLE,
+  };
+
+  int64_t file_length_;
+  int64_t last_modified_ts_;
+  int type_;
+
+  CosObjectMeta()
+  {
+    reset();
+  }
+
+  ~CosObjectMeta() {}
+
+  void reset()
+  {
+    memset(this, 0, sizeof(*this));
+    file_length_ = -1;
+    type_ = CosObjectType::COS_OBJ_INVALID;
+  }
+};
+
+
+/*= Custom memory allocation functions */
+typedef void* (*OB_COS_allocFunction) (void* opaque, size_t size);
+typedef void  (*OB_COS_freeFunction) (void* opaque, void* address);
+typedef struct { OB_COS_allocFunction customAlloc; OB_COS_freeFunction customFree; void* opaque; } OB_COS_customMem;
+
+class OB_PUBLIC_API ObCosWrapper
+{
+public:
+  struct Handle {};
+
+  // alloc_f is used to allocate the handle's memory.
+  // You need to call destroy_cos_handle when handle is not use
+  // any more.
+  static int create_cos_handle(
+    OB_COS_customMem &custom_mem,
+    const struct ObCosAccount &account,
+    Handle **h);
+
+  // You can not use handle any more after destroy_cos_handle is called.
+  static void destroy_cos_handle(Handle *h);
+
+  // Put an object to cos, the new object will overwrite the old one if object exist.
+  static int put(
+    Handle *h,
+    const CosStringBuffer &bucket_name,
+    const CosStringBuffer &object_name,
+    const char *buffer,
+    const int64_t size);
+
+  // Append content to the specific object from the @offset position in cos.
+  static int append(
+    Handle *h,
+    const CosStringBuffer &bucket_name,
+    const CosStringBuffer &object_name,
+    const char *buf,
+    const int64_t buf_size,
+    const int64_t offset);
+
+  // Get object meta
+  static int head_object_meta(
+    Handle *h,
+    const CosStringBuffer &bucket_name,
+    const CosStringBuffer &object_name,
+    bool &is_exist,
+    CosObjectMeta &meta);
+
+  // Delete one object from cos.
+  static int del(
+    Handle *h,
+    const CosStringBuffer &bucket_name,
+    const CosStringBuffer &object_name);
+
+  // Tag one object from cos
+  static int tag(
+    Handle *h,
+    const CosStringBuffer &bucket_name,
+    const CosStringBuffer &object_name);
+
+
+  // Delete all objects that match the same dir_name prefix.
+  static int del_objects_in_dir(
+    Handle *h,
+    const CosStringBuffer &bucket_name,
+    const CosStringBuffer &dir_name,
+    int64_t &deleted_cnt);
+
+  // Update object last modofied time.
+  static int update_object_modified_ts(
+    Handle *h,
+    const CosStringBuffer &bucket_name,
+    const CosStringBuffer &object_name);
+
+  // Random read data of specific range from object.
+  // Buffer's memory is provided by user.
+  static int pread(
+    Handle *h,
+    const CosStringBuffer &bucket_name,
+    const CosStringBuffer &object_name,
+    int64_t offset,
+    char *buf,
+    int64_t buf_size,
+    int64_t &read_size);
+
+  // Get whole object
+  // Buffer's memory is provided by user.
+  static int get_object(
+    Handle *h,
+    const CosStringBuffer &bucket_name,
+    const CosStringBuffer &object_name,
+    char *buf,
+    int64_t buf_size,
+    int64_t &read_size);
+
+  static int is_object_tagging(
+    Handle *h,
+    const CosStringBuffer &bucket_name,
+    const CosStringBuffer &object_name,
+    bool &is_tagging);
+
+  struct CosListObjPara {
+    enum class CosListType
+    {
+      COS_LIST_INVALID,
+      COS_LIST_CB_ARG
+    };
+
+    CosListObjPara()
+      : arg_(NULL), cur_full_path_slice_name_(NULL),
+        full_path_size_(0), cur_object_size_(0),
+        next_flag_(false), type_(CosListType::COS_LIST_INVALID)
+    {
+      last_container_name_.d_name[0] = '\0';
+      last_container_name_.d_type = DT_REG;
+    }
+
+    void* arg_;
+    char* cur_full_path_slice_name_;
+    struct dirent last_container_name_;
+    int64_t full_path_size_;
+    int64_t cur_object_size_;
+    bool next_flag_;
+    CosListType type_;
+  };
+
+  typedef int (*handleObjectNameFunc)(CosListObjPara&);
+  typedef int (*handleDirectoryFunc)(void*, const CosListObjPara::CosListType, const char*, int64_t);
+  // List objects in the same directory, include all objects
+  // in the inner sub directories.
+  // dir_name must be end with "/\0".
+  static int list_objects(
+    Handle *h,
+    const CosStringBuffer &bucket_name,
+    const CosStringBuffer &dir_name,
+    const CosStringBuffer &next_marker,
+    handleObjectNameFunc handle_object_name_f,
+    void *arg);
+
+  static int list_directories(
+    Handle *h,
+    const CosStringBuffer &bucket_name,
+    const CosStringBuffer &dir_name,
+    const CosStringBuffer &next_marker,
+    const CosStringBuffer &delimiter,
+    handleDirectoryFunc handle_directory_name_f,
+    void *arg);
+
+  static int is_empty_directory(
+    Handle *h,
+    const CosStringBuffer &bucket_name,
+    const CosStringBuffer &dir_name,
+    bool &is_empty_dir);
+};
+
+
+#undef OB_PUBLIC_API
+}
+}
+}
+#endif

deps/oblib/src/lib/restore/cos/ob_cos_wrapper_handle.cpp

@@ -0,0 +1,235 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include "lib/utility/ob_print_utils.h"
+#include "ob_cos_wrapper_handle.h"
+
+namespace oceanbase
+{
+namespace common
+{
+using namespace oceanbase::common;
+
+/**
+ * ------------------------------ObCosMemAllocator---------------------
+ */
+ObCosMemAllocator::ObCosMemAllocator()
+  : allocator_(ObModIds::BACKUP, OB_MALLOC_NORMAL_BLOCK_SIZE)
+{
+}
+
+ObCosMemAllocator::~ObCosMemAllocator()
+{
+}
+
+void* ObCosMemAllocator::alloc(size_t size)
+{
+  return allocator_.alloc(size);
+}
+
+void ObCosMemAllocator::free(void *addr)
+{
+  allocator_.free(addr);
+}
+
+void ObCosMemAllocator::reuse()
+{
+  allocator_.reuse();
+}
+
+
+// Memory function used for cos context
+static void *ob_cos_malloc(void *opaque, size_t size)
+{
+  void *buf = NULL;
+  if (NULL != opaque) {
+    ObCosMemAllocator *allocator = reinterpret_cast<ObCosMemAllocator*> (opaque);
+    buf = allocator->alloc(size);
+  }
+  return buf;
+}
+
+static void ob_cos_free(void *opaque, void *address)
+{
+  if (NULL != opaque) {
+    ObCosMemAllocator *allocator = reinterpret_cast<ObCosMemAllocator*> (opaque);
+    allocator->free(address);
+  }
+}
+
+/*--------------------------------ObCosWrapperHandle-----------------------------------*/
+ObCosWrapperHandle::ObCosWrapperHandle()
+  : is_inited_(false), handle_(nullptr), cos_account_(), allocator_(),
+    delete_mode_(ObIStorageUtil::DELETE)
+{}
+
+int ObCosWrapperHandle::init(const ObObjectStorageInfo *storage_info)
+{
+  int ret = OB_SUCCESS;
+  char storage_info_str[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 };
+
+  if (IS_INIT) {
+    ret = OB_INIT_TWICE;
+    OB_LOG(WARN, "init twice", K(ret));
+  } else if (OB_ISNULL(storage_info)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "storage info is null", K(ret));
+  } else if (OB_FAIL(storage_info->get_storage_info_str(storage_info_str, sizeof(storage_info_str)))) {
+    OB_LOG(WARN, "fail to get cos storage info str", K(ret), K(storage_info));
+  } else if (OB_FAIL(cos_account_.parse_from(storage_info_str, strlen(storage_info_str)))) {
+    OB_LOG(WARN, "fail to parse cos account from storage info str", K(ret));
+  } else if (strlen(cos_account_.delete_mode_) > 0 && OB_FAIL(set_delete_mode(cos_account_.delete_mode_))) {
+    OB_LOG(WARN, "fail to set cos delete mode", K(cos_account_.delete_mode_), K(ret));
+  } {
+    is_inited_ = true;
+  }
+  return ret;
+}
+
+void ObCosWrapperHandle::reset()
+{
+  destroy_cos_handle();
+  is_inited_ = false;
+  delete_mode_ = ObIStorageUtil::DELETE;
+}
+
+int ObCosWrapperHandle::create_cos_handle()
+{
+  int ret = OB_SUCCESS;
+  qcloud_cos::OB_COS_customMem cos_mem = {ob_cos_malloc, ob_cos_free, &allocator_};
+  if (OB_NOT_NULL(handle_)) {
+    ret = OB_ERR_UNEXPECTED;
+    OB_LOG(WARN, "handle is not null", K(ret));
+  } else if (OB_FAIL(qcloud_cos::ObCosWrapper::create_cos_handle(cos_mem, cos_account_, &handle_))) {
+    OB_LOG(WARN, "failed to create cos handle", K(ret));
+  } else if (OB_ISNULL(handle_)) {
+    ret = OB_COS_ERROR;
+    OB_LOG(WARN, "create handle succeed, but returned handle is null", K(ret));
+  }
+
+  return ret;
+}
+
+void ObCosWrapperHandle::destroy_cos_handle()
+{
+  if (nullptr != handle_) {
+    qcloud_cos::ObCosWrapper::destroy_cos_handle(handle_);
+    handle_ = nullptr;
+  }
+}
+
+int ObCosWrapperHandle::build_bucket_and_object_name(const ObString &uri)
+{
+  int ret = OB_SUCCESS;
+  ObString::obstr_size_t bucket_start = 0;
+  ObString::obstr_size_t bucket_end = 0;
+  ObString::obstr_size_t object_start = 0;
+  char* bucket_name_buff = NULL;
+  char* object_name_buff = NULL;
+
+  if (!uri.prefix_match(OB_COS_PREFIX)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "uri is invalid ", KCSTRING(OB_COS_PREFIX), K(uri), K(ret));
+  } else {
+    bucket_start = static_cast<ObString::obstr_size_t>(strlen(OB_COS_PREFIX));
+    for (int64_t i = bucket_start; OB_SUCC(ret) && i < uri.length() - 2; i++) {
+      if ('/' == *(uri.ptr() + i) && '/' == *(uri.ptr() + i + 1)) {
+        ret = OB_INVALID_ARGUMENT;
+        OB_LOG(WARN, "uri has two // ", K(uri), K(ret), K(i));
+        break;
+      }
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_ISNULL(bucket_name_buff = static_cast<char *>(allocator_.alloc(OB_MAX_URI_LENGTH)))) {
+      ret = OB_ALLOCATE_MEMORY_FAILED;
+      OB_LOG(WARN, "failed to alloc bucket name buff", K(ret), K(uri));
+    } else if (OB_ISNULL(object_name_buff = static_cast<char *>(allocator_.alloc(OB_MAX_URI_LENGTH)))) {
+      ret = OB_ALLOCATE_MEMORY_FAILED;
+      OB_LOG(WARN, "failed to alloc object name buff", K(ret), K(uri));
+    } else {
+      //make sure this last always 0
+      bucket_name_buff[0] = '\0';
+      object_name_buff[0] = '\0';
+    }
+  }
+
+  // parse bucket name
+  if (OB_SUCC(ret)) {
+    if (bucket_start >= uri.length()) {
+      ret = OB_INVALID_ARGUMENT;
+      OB_LOG(WARN, "bucket name is NULL", K(uri), K(ret));
+    } else {
+      for (bucket_end = bucket_start; OB_SUCC(ret) && bucket_end < uri.length(); ++bucket_end) {
+        if ('/' == *(uri.ptr() + bucket_end)) {
+          ObString::obstr_size_t bucket_length = bucket_end - bucket_start;
+          //must end with '\0'
+          if (OB_FAIL(databuff_printf(bucket_name_buff, OB_MAX_URI_LENGTH, "%.*s", bucket_length, uri.ptr() + bucket_start))) {
+            OB_LOG(WARN, "fail to deep copy bucket", K(uri), K(bucket_start), K(bucket_length), K(ret));
+          } else {
+            bucket_name_.assign_ptr(bucket_name_buff, strlen(bucket_name_buff) + 1);// must include '\0'
+          }
+          break;
+        }
+      }
+
+      if (OB_SUCC(ret) && bucket_end == uri.length()) {
+        ret = OB_INVALID_ARGUMENT;
+        OB_LOG(WARN, "bucket name is invalid", K(uri), K(ret));
+      }
+    }
+  }
+
+  // parse the object name
+  if (OB_SUCC(ret)) {
+    if (bucket_end + 1 >= uri.length()) {
+      ret = OB_INVALID_ARGUMENT;
+      OB_LOG(WARN, "object name is NULL", K(uri), K(ret));
+    } else {
+      object_start = bucket_end + 1;
+      ObString::obstr_size_t object_length = uri.length() - object_start;
+      //must end with '\0'
+      if (OB_FAIL(databuff_printf(object_name_buff, OB_MAX_URI_LENGTH, "%.*s", object_length, uri.ptr() + object_start))) {
+        OB_LOG(WARN, "fail to deep copy object", K(uri), K(object_start), K(object_length), K(ret));
+      } else {
+        object_name_.assign_ptr(object_name_buff, strlen(object_name_buff) + 1);//must include '\0'
+      }
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    OB_LOG(DEBUG, "get bucket object name", K(uri), K_(bucket_name), K_(object_name));
+  }
+
+  return ret;
+}
+
+int ObCosWrapperHandle::set_delete_mode(const char *parameter)
+{
+  int ret = OB_SUCCESS;
+  if (NULL == parameter) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid args", K(ret), KP(parameter));
+  } else if (0 == strcmp(parameter, "delete")) {
+    delete_mode_ = ObIStorageUtil::DELETE;
+  } else if (0 == strcmp(parameter, "tagging")) {
+    delete_mode_ = ObIStorageUtil::TAGGING;
+  } else {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "delete mode is invalid", K(ret), K(parameter));
+  }
+  return ret;
+}
+
+} // common
+} // oceanbase

deps/oblib/src/lib/restore/cos/ob_cos_wrapper_handle.h

@@ -0,0 +1,77 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef SRC_LIBRARY_SRC_LIB_RESTORE_OB_COS_WRAPPER_HANDLE_H_
+#define SRC_LIBRARY_SRC_LIB_RESTORE_OB_COS_WRAPPER_HANDLE_H_
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include "lib/string/ob_string.h"
+#include "lib/allocator/page_arena.h"
+#include "lib/restore/ob_storage_info.h"
+#include "ob_cos_wrapper.h"
+#include "lib/restore/ob_i_storage.h"
+
+namespace oceanbase
+{
+namespace common
+{
+
+// Allocator for creating cos handle
+class ObCosMemAllocator
+{
+public:
+  ObCosMemAllocator();
+  virtual ~ObCosMemAllocator();
+  void *alloc(size_t size);
+  void free(void *addr);
+  void reuse();
+private:
+  ObArenaAllocator allocator_;
+};
+
+class ObCosWrapperHandle
+{
+public:
+  ObCosWrapperHandle();
+  virtual ~ObCosWrapperHandle() {};
+
+  int init(const common::ObObjectStorageInfo *storage_info);
+  void reset();
+  int create_cos_handle();
+  void destroy_cos_handle();
+  qcloud_cos::ObCosWrapper::Handle *get_ptr() { return handle_; }
+
+  int build_bucket_and_object_name(const ObString &uri);
+  const ObString& get_bucket_name() const { return bucket_name_; }
+  const ObString& get_object_name() const { return object_name_; }
+
+  bool is_valid() const { return is_inited_ && handle_ != nullptr; }
+  bool is_inited() const { return is_inited_; }
+
+  int set_delete_mode(const char *parameter);
+  int64_t get_delete_mode() const { return delete_mode_; }
+
+private:
+  bool is_inited_;
+  qcloud_cos::ObCosWrapper::Handle *handle_;
+  qcloud_cos::ObCosAccount cos_account_;
+  ObCosMemAllocator allocator_;
+  ObString bucket_name_;
+  ObString object_name_;
+  int64_t delete_mode_;
+};
+
+
+} // common
+} // oceanbase
+#endif

deps/oblib/src/lib/restore/cos/ob_singleton.h

@@ -0,0 +1,50 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef SRC_LIBRARY_SRC_LIB_OB_SINGLETON_H_
+#define SRC_LIBRARY_SRC_LIB_OB_SINGLETON_H_
+
+namespace oceanbase
+{
+namespace common
+{
+namespace qcloud_cos
+{
+
+// A singleton base class offering an easy way to create singleton.
+template <typename T>
+class __attribute__ ((visibility ("default"))) ObSingleton
+{
+public:
+  // not thread safe
+  static T& get_instance()
+  {
+    static T instance;
+    return instance;
+  }
+
+  virtual ~ObSingleton() {}
+
+protected:
+  ObSingleton() {}
+
+private:
+  ObSingleton(const ObSingleton &);
+  ObSingleton& operator=(const ObSingleton&);
+};
+
+
+}
+}
+}
+
+#endif

deps/oblib/src/lib/restore/ob_i_storage.h

@@ -17,6 +17,7 @@
 #include "lib/string/ob_string.h"
 #include "lib/container/ob_array.h"
 #include "common/storage/ob_device_common.h"
+#include "ob_storage_info.h"
 
 namespace oceanbase
 {
@@ -39,7 +40,7 @@ public:
     TAGGING = 2,
     MAX
   };
-  virtual int open(void* base_info) = 0;
+  virtual int open(common::ObObjectStorageInfo *storage_info) = 0;
   virtual void close() = 0;
   virtual int is_exist(const common::ObString &uri, bool &exist) = 0;
   virtual int get_file_length(const common::ObString &uri, int64_t &file_length) = 0;
@@ -48,7 +49,6 @@ public:
   virtual int mkdir(const common::ObString &uri) = 0;
   virtual int list_files(const common::ObString &dir_path, common::ObBaseDirEntryOperator &op) = 0;
   virtual int del_dir(const common::ObString &uri) = 0;
-  virtual int check_backup_dest_lifecycle(const common::ObString &path, bool &is_set_lifecycle) = 0;
   virtual int list_directories(const common::ObString &dir_path, common::ObBaseDirEntryOperator &op) = 0;
   virtual int is_tagging(const common::ObString &uri, bool &is_tagging) = 0;
 };
@@ -56,7 +56,7 @@ public:
 class ObIStorageReader
 {
 public:
-  virtual int open(const common::ObString &uri, void* handle) = 0;
+  virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info) = 0;
   virtual int pread(char *buf,const int64_t buf_size, int64_t offset, int64_t &read_size) = 0;
   virtual int close() = 0;
   virtual int64_t get_length() const = 0;
@@ -66,7 +66,7 @@ public:
 class ObIStorageWriter
 {
 public:
-  virtual int open(const common::ObString &uri, void* handle) = 0;
+  virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info) = 0;
   virtual int write(const char *buf,const int64_t size) = 0;
   virtual int pwrite(const char *buf, const int64_t size, const int64_t offset) = 0;
   virtual int close() = 0;

deps/oblib/src/lib/restore/ob_object_device.cpp

@@ -19,7 +19,8 @@ namespace common
 
 const char *OB_STORAGE_ACCESS_TYPES_STR[] = {"reader", "overwriter", "appender", "random_write"};
 
-ObObjectDevice::ObObjectDevice() : oss_account_(), base_info_(NULL), is_started_(false), lock_(common::ObLatchIds::OBJECT_DEVICE_LOCK)
+ObObjectDevice::ObObjectDevice()
+  : storage_info_(), is_started_(false), lock_(common::ObLatchIds::OBJECT_DEVICE_LOCK)
 {
   auto attr = SET_USE_500("ObjectDevice");
   reader_ctx_pool_.set_attr(attr);
@@ -51,7 +52,7 @@ void ObObjectDevice::destroy()
 ObObjectDevice::~ObObjectDevice()
 {
   destroy();
-  OB_LOG(INFO, "destory the device!", KCSTRING(storage_info_));
+  OB_LOG(INFO, "destory the device!", KCSTRING(storage_info_str_));
 }
 
 /*the app logical use call ObBackupIoAdapter::get_and_init_device*/
@@ -70,29 +71,19 @@ int ObObjectDevice::start(const ObIODOpts &opts)
     ret = OB_INVALID_ARGUMENT;
     OB_LOG(WARN, "fail to start device, args wrong !", KCSTRING(opts.opts_[0].key_), K(ret));
   } else {
-    ObString storage_info(opts.opts_[0].value_.value_str);
-    if (OB_STORAGE_OSS == device_type_) {
-      if (OB_FAIL(oss_account_.parse_oss_arg(storage_info))) {
-        OB_LOG(WARN, "fail to init oss base", K(storage_info) ,K(ret));
-      }
-      base_info_ = (void*)&oss_account_;
-    } else if (OB_STORAGE_FILE == device_type_) {
-      //do nothing
-      base_info_ = NULL;
-    } else {
-      ret = OB_INVALID_ARGUMENT;
-      OB_LOG(WARN, "Invalid object device type!", K(ret), K(device_type_), KCSTRING(opts.opts_[0].value_.value_str));
+    if (OB_FAIL(storage_info_.set(device_type_, opts.opts_[0].value_.value_str))) {
+      OB_LOG(WARN, "failed to build storage_info");
     }
 
     if (OB_SUCCESS != ret) {
       //mem resource will be free with device destroy
-    } else if (OB_FAIL(util_.open(base_info_, device_type_))) {
+    } else if (OB_FAIL(util_.open(&storage_info_))) {
       OB_LOG(WARN, "fail to open the util!", K(ret), KCSTRING(opts.opts_[0].value_.value_str));
     } else if (OB_FAIL(fd_mng_.init())) {
       OB_LOG(WARN, "fail to init fd manager!", K(ret));
     } else {
       is_started_ = true;
-      if (OB_FAIL(databuff_printf(storage_info_, OB_MAX_URI_LENGTH, "%s", opts.opts_[0].value_.value_str))) {
+      if (OB_FAIL(databuff_printf(storage_info_str_, OB_MAX_URI_LENGTH, "%s", opts.opts_[0].value_.value_str))) {
         OB_LOG(WARN, "fail to copy str to storage info", K(ret));
       }
     }
@@ -161,7 +152,7 @@ int ObObjectDevice::open_for_reader(const char *pathname, void*& ctx)
     ret = OB_ALLOCATE_MEMORY_FAILED;
     OB_LOG(WARN, "fail to alloc mem for object device reader! ", K(ret));
   } else {
-    if (OB_FAIL(reader->open(pathname, base_info_))) {
+    if (OB_FAIL(reader->open(pathname, &storage_info_))) {
       OB_LOG(WARN, "fail to open for read!", K(ret));
     } else {
       ctx = (void*)reader;
@@ -184,7 +175,7 @@ int ObObjectDevice::open_for_overwriter(const char *pathname, void*& ctx)
     ret = OB_ALLOCATE_MEMORY_FAILED;
     OB_LOG(WARN, "fail to alloc mem for object device reader! ", K(ret));
   } else {
-    if (OB_FAIL(overwriter->open(pathname, base_info_))) {
+    if (OB_FAIL(overwriter->open(pathname, &storage_info_))) {
       OB_LOG(WARN, "fail to open for overwrite!", K(ret));
     } else {
       ctx = (void*)overwriter;
@@ -202,16 +193,10 @@ int ObObjectDevice::open_for_appender(const char *pathname, ObIODOpts *opts, voi
   int ret = OB_SUCCESS;
   ObStorageAppender *appender = NULL;
   ObOptValue opt_value;
-  const char* append_strategy = NULL;
-  int64_t append_version= -1;
   const char* open_mode = NULL;
 
-  get_opt_value(opts, "AppendStrategy", append_strategy);
-  get_opt_value(opts, "AppendVersion", append_version);
   get_opt_value(opts, "OpenMode", open_mode);
-  ObStorageAppender::AppenderParam param;
   StorageOpenMode mode = StorageOpenMode::CREATE_OPEN_LOCK;
-  ObAppendStrategy strategy = ObAppendStrategy::OB_APPEND_USE_OVERRITE;
 
   appender = appender_ctx_pool_.alloc();
   if (OB_ISNULL(appender)) {
@@ -232,29 +217,9 @@ int ObObjectDevice::open_for_appender(const char *pathname, ObIODOpts *opts, voi
     OB_LOG(WARN, "Invalid open mode!", KCSTRING(open_mode), K(ret));
   }
 
-  if (NULL == append_strategy || 0 == STRCMP(append_strategy, "OB_APPEND_USE_OVERRITE")) {
-    //just keep the default value
-  } else if (0 == STRCMP(append_strategy, "OB_APPEND_USE_SLICE_PUT")) {
-    strategy = ObAppendStrategy::OB_APPEND_USE_SLICE_PUT;
-  } else {
-    ret = OB_INVALID_ARGUMENT;
-    OB_LOG(WARN, "Invalid append strategy!", KCSTRING(append_strategy), K(ret));
-  }
-  param.strategy_ = strategy;
-  param.version_param_.version_ = append_version;
-  if (-1 != append_version) {
-    param.version_param_.open_object_version_ = true;
-    if (param.version_param_.version_ <= 0) {
-      ret = OB_INVALID_ARGUMENT;
-      OB_LOG(WARN, "Invalid strategy version!", K(param.version_param_.version_), K(ret));
-    }
-  } else {
-    param.version_param_.open_object_version_ = false;
-  }
-
   if (OB_SUCCESS == ret) {
     appender->set_open_mode(mode);
-    if (OB_FAIL(appender->open(pathname, base_info_, param))){
+    if (OB_FAIL(appender->open(pathname, &storage_info_))){
       OB_LOG(WARN, "fail to open the appender!", K(ret));
     } else {
       ctx = appender;
@@ -558,43 +523,11 @@ int ObObjectDevice::pwrite(const ObIOFd &fd, const int64_t offset, const int64_t
   return ret;
 }
 
-int ObObjectDevice::ob_get_bucket_lifecycle(ObIODOpts &opts)
-{
-  int ret = OB_SUCCESS;
-  const char* path = NULL;
-  bool is_set_lifecycle = false;
-  if (opts.opt_cnt_ != 2) {
-    ret = OB_INVALID_ARGUMENT;
-    OB_LOG(WARN, "ObGetBucketLifecycle need 3 element at least!", K(opts.opt_cnt_), K(ret));
-  } else if (0 != STRCMP(opts.opts_[1].key_, "path") || NULL == opts.opts_[1].value_.value_str) {
-    ret = OB_INVALID_ARGUMENT;
-    OB_LOG(WARN, "ObGetBucketLifecycle, the first para is wrong!", KCSTRING(opts.opts_[1].key_), K(ret));
-  } else {
-    path = opts.opts_[1].value_.value_str;
-    if (OB_FAIL(util_.check_backup_dest_lifecycle(ObString(path), is_set_lifecycle))) {
-      OB_LOG(WARN, "ObGetBucketLifecycle, the second para is wrong!", K(ret));
-    } else {
-      opts.opts_[0].set("get_bucket_lifecycle", is_set_lifecycle);
-    }
-  }
-  return ret;
-}
-
 //the first opt is config name & the return value
 int ObObjectDevice::get_config(ObIODOpts &opts)
 {
   int ret = OB_SUCCESS;
-  if (opts.opt_cnt_ < 1) {
-    ret = OB_INVALID_ARGUMENT;
-    OB_LOG(WARN, "opts need 1 element at least!", K(opts.opt_cnt_), K(ret));
-  } else if (0 == STRCMP(opts.opts_[0].key_, "get_bucket_lifecycle")) {
-    if (OB_FAIL(ob_get_bucket_lifecycle(opts))) {
-      OB_LOG(WARN, "fail to get bucket life cycle!", K(ret));
-    }
-  } else {
-    ret = OB_NOT_SUPPORTED;
-    OB_LOG(WARN, "the para is wrong, this interface only support get_bucket_lifecycle now!", K(device_type_), K(ret));
-  }
+  UNUSED(opts);
   return ret;
 }
 

deps/oblib/src/lib/restore/ob_object_device.h

@@ -82,20 +82,18 @@ private:
   int open_for_overwriter(const char *pathname, void*& ctx);
   int open_for_appender(const char *pathname, ObIODOpts *opts, void*& ctx);
   int release_res(void* ctx, const ObIOFd &fd, ObStorageAccessType access_type);
-  int ob_get_bucket_lifecycle(ObIODOpts &opts);
 private:
   //maybe fd mng can be device level
   common::ObFdSimulator    fd_mng_;
-  ObOssAccount oss_account_;
   
   ObStorageUtil            util_;
   /*obj ctx pool: use to create fd ctx(reader/writer)*/
   common::ObPooledAllocator<ObStorageReader, ObMalloc, ObSpinLock> reader_ctx_pool_;
   common::ObPooledAllocator<ObStorageAppender, ObMalloc, ObSpinLock> appender_ctx_pool_;
   common::ObPooledAllocator<ObStorageWriter, ObMalloc, ObSpinLock> overwriter_ctx_pool_;
-  void* base_info_;
+  common::ObObjectStorageInfo storage_info_;
   bool is_started_;
-  char storage_info_[OB_MAX_URI_LENGTH];
+  char storage_info_str_[OB_MAX_URI_LENGTH];
   common::ObSpinLock lock_;
 
 private:

deps/oblib/src/lib/restore/ob_storage.cpp

@@ -56,6 +56,7 @@ int validate_uri_type(const common::ObString &uri)
 {
   int ret = OB_SUCCESS;
   if (!uri.prefix_match(OB_OSS_PREFIX) &&
+      !uri.prefix_match(OB_COS_PREFIX) &&
       !uri.prefix_match(OB_FILE_PREFIX)) {
     ret = OB_INVALID_BACKUP_DEST;
     STORAGE_LOG(ERROR, "invalid backup uri", K(ret), K(uri));
@@ -70,6 +71,8 @@ int get_storage_type_from_path(const common::ObString &uri, ObStorageType &type)
 
   if (uri.prefix_match(OB_OSS_PREFIX)) {
     type = OB_STORAGE_OSS;
+  } else if (uri.prefix_match(OB_COS_PREFIX)) {
+    type = OB_STORAGE_COS;
   } else if (uri.prefix_match(OB_FILE_PREFIX)) {
     type = OB_STORAGE_FILE;
   } else {
@@ -91,7 +94,7 @@ const char *get_storage_type_str(const ObStorageType &type)
 
 bool is_io_error(const int result)
 {
-  return OB_IO_ERROR == result || OB_OSS_ERROR == result;
+  return OB_IO_ERROR == result || OB_OSS_ERROR == result || OB_COS_ERROR == result;
 }
 
 int get_storage_type_from_name(const char *type_str, ObStorageType &type)
@@ -187,21 +190,30 @@ ObExternalIOCounterGuard::~ObExternalIOCounterGuard()
 ObStorageUtil::ObStorageUtil()
   : file_util_(),
     oss_util_(),
+    cos_util_(),
     util_(NULL),
-    obj_base_info_(NULL),
+    storage_info_(NULL),
     init_state(false)
 {
 }
 
 /*this fun just like the get_util*/
-int ObStorageUtil::open(void* obj_base, int device_type)
+int ObStorageUtil::open(common::ObObjectStorageInfo *storage_info)
 {
   int ret = OB_SUCCESS;
+  ObStorageType device_type = OB_STORAGE_MAX_TYPE;
+
   if (is_init()) {
     ret = OB_INIT_TWICE;
-    STORAGE_LOG(WARN, "double init the storage util", K(ret), K(device_type));
+    STORAGE_LOG(WARN, "double init the storage util", K(ret));
+  } else if (OB_ISNULL(storage_info) || OB_UNLIKELY(!storage_info->is_valid())) {
+    ret = OB_INVALID_ARGUMENT;
+    STORAGE_LOG(WARN, "invalid argument", K(ret), KP(storage_info));
+  } else if (OB_FALSE_IT(device_type = storage_info->get_type())) {
   } else if (OB_STORAGE_OSS == device_type) {
     util_ = &oss_util_;
+  } else if (OB_STORAGE_COS == device_type) {
+    util_ = &cos_util_;
   } else if (OB_STORAGE_FILE == device_type) {
     util_ = &file_util_;
   } else {
@@ -210,11 +222,11 @@ int ObStorageUtil::open(void* obj_base, int device_type)
   }
 
   if (OB_SUCC(ret) && NULL != util_) {
-    if (OB_FAIL(util_->open(obj_base))) {
+    if (OB_FAIL(util_->open(storage_info))) {
       STORAGE_LOG(WARN, "failed to open util", K(ret), K(device_type));
       util_ = NULL;
     } else {
-      obj_base_info_ = obj_base;
+      storage_info_ = storage_info;
       init_state = true;
     }
   }
@@ -226,7 +238,7 @@ void ObStorageUtil::close()
   if (NULL != util_) {
     util_->close();
     util_ = NULL;
-    obj_base_info_ = NULL;
+    storage_info_ = NULL;
     init_state = false;
   }
 }
@@ -462,37 +474,6 @@ int ObStorageUtil::del_dir(const common::ObString &uri)
   return ret;
 }
 
-int ObStorageUtil::check_backup_dest_lifecycle(
-    const common::ObString &path,
-    bool &is_set_lifecycle)
-{
-  int ret = OB_NOT_SUPPORTED;
-  UNUSED(path);
-  UNUSED(is_set_lifecycle);
-  // Forbidden check backup dest lifecycle, because oss has bug which make observer core
-  /*
-  const int64_t start_ts = ObTimeUtility::current_time();
-  ObIStorageUtil *util = NULL;
-  is_set_lifecycle = false;
-
-  if (OB_FAIL(ret)) {
-  } else if (!is_init()) {
-    ret = OB_NOT_INIT;
-    STORAGE_LOG(WARN, "util is not inited", K(ret), K(path));
-  } else if (path.empty()) {
-    ret = OB_INVALID_ARGUMENT;
-    STORAGE_LOG(WARN, "delete tmp files get invalid argument", K(ret), K(path));
-  } else if (ObStorageGlobalIns::get_instance().is_io_prohibited()) {
-    ret = OB_BACKUP_IO_PROHIBITED;
-    STORAGE_LOG(WARN, "current observer backup io is prohibited", K(ret), K(path));
-  } else if (OB_FAIL(util_->check_backup_dest_lifecycle(path, is_set_lifecycle))) {
-    STORAGE_LOG(WARN, "failed to delete tmp files", K(ret), K(path));
-  }
-  print_access_storage_log("check_bucket_lifecycle", path, start_ts, 0);
-  */
-  return ret;
-}
-
 int ObStorageUtil::list_directories(const common::ObString &uri, common::ObBaseDirEntryOperator &op)
 {
   int ret = OB_SUCCESS;
@@ -540,6 +521,7 @@ ObStorageReader::ObStorageReader()
     reader_(NULL),
     file_reader_(),
     oss_reader_(),
+    cos_reader_(),
     start_ts_(0)
 {
   uri_[0] = '\0';
@@ -552,7 +534,7 @@ ObStorageReader::~ObStorageReader()
   }
 }
 
-int ObStorageReader::open(const common::ObString &uri, void* obj_base_info)
+int ObStorageReader::open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info)
 {
   int ret = OB_SUCCESS;
   int tmp_ret = OB_SUCCESS;
@@ -575,6 +557,8 @@ int ObStorageReader::open(const common::ObString &uri, void* obj_base_info)
     STORAGE_LOG(WARN, "failed to get type", K(ret), K(uri));
   } else if (OB_STORAGE_OSS == type) {
     reader_ = &oss_reader_;
+  } else if (OB_STORAGE_COS == type) {
+    reader_ = &cos_reader_;
   } else if (OB_STORAGE_FILE == type) {
     reader_ = &file_reader_;
   } else {
@@ -586,7 +570,7 @@ int ObStorageReader::open(const common::ObString &uri, void* obj_base_info)
     if (OB_ISNULL(reader_)) {
       ret = OB_ERR_UNEXPECTED;
       STORAGE_LOG(WARN, "reader_ is null", K(ret), K(uri));
-    } else if (OB_FAIL(reader_->open(uri, obj_base_info))) {
+    } else if (OB_FAIL(reader_->open(uri, storage_info))) {
       STORAGE_LOG(WARN, "failed to open reader", K(ret), K(uri));
     } else {
       file_length_ = reader_->get_length();
@@ -661,6 +645,7 @@ ObStorageWriter::ObStorageWriter()
   : writer_(NULL),
     file_writer_(),
     oss_writer_(),
+    cos_writer_(),
     start_ts_(0)
 {
     uri_[0] = '\0';
@@ -673,7 +658,7 @@ ObStorageWriter::~ObStorageWriter()
   }
 }
 
-int ObStorageWriter::open(const common::ObString &uri, void* obj_base_info)
+int ObStorageWriter::open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info)
 {
   int ret = OB_SUCCESS;
   int tmp_ret = OB_SUCCESS;
@@ -696,6 +681,8 @@ int ObStorageWriter::open(const common::ObString &uri, void* obj_base_info)
     STORAGE_LOG(WARN, "failed to get type", K(ret), K(uri));
   } else if (OB_STORAGE_OSS == type) {
     writer_ = &oss_writer_;
+  } else if (OB_STORAGE_COS == type) {
+    writer_ = &cos_writer_;
   } else if (OB_STORAGE_FILE == type) {
     writer_ = &file_writer_;
   } else {
@@ -707,7 +694,7 @@ int ObStorageWriter::open(const common::ObString &uri, void* obj_base_info)
     if (OB_ISNULL(writer_)) {
       ret = OB_ERR_UNEXPECTED;
       STORAGE_LOG(WARN, "writer_ is null", K(ret), K(uri));
-    } else if (OB_FAIL(writer_->open(uri, obj_base_info))) {
+    } else if (OB_FAIL(writer_->open(uri, storage_info))) {
       STORAGE_LOG(WARN, "failed to open writer", K(ret), K(uri));
     }
   }
@@ -774,9 +761,10 @@ ObStorageAppender::ObStorageAppender()
   : appender_(NULL),
     file_appender_(),
     oss_appender_(),
+    cos_appender_(),
     start_ts_(0),
     is_opened_(false),
-    storage_info_(NULL)
+    storage_info_()
 {
     uri_[0] = '\0';
 }
@@ -785,9 +773,10 @@ ObStorageAppender::ObStorageAppender(StorageOpenMode mode)
   : appender_(NULL),
     file_appender_(mode),
     oss_appender_(),
+    cos_appender_(),
     start_ts_(0),
     is_opened_(false),
-    storage_info_(NULL)
+    storage_info_()
 {
     uri_[0] = '\0';
 }
@@ -801,8 +790,7 @@ ObStorageAppender::~ObStorageAppender()
 
 int ObStorageAppender::open(
     const common::ObString &uri, 
-    void* obj_base_info, 
-	const AppenderParam &param)
+    common::ObObjectStorageInfo *storage_info)
 {
   int ret = OB_SUCCESS;
   int tmp_ret = OB_SUCCESS;
@@ -819,17 +807,20 @@ int ObStorageAppender::open(
   } else if (NULL != appender_) {
     ret = OB_INIT_TWICE;
     STORAGE_LOG(WARN, "cannot open twice", K(ret), K(uri));
+  } else if (OB_UNLIKELY(uri.empty()) || OB_ISNULL(storage_info)) {
+    ret = OB_INVALID_ARGUMENT;
+    STORAGE_LOG(WARN, "invalid argument", K(ret), K(uri), KP(storage_info));
   } else if (OB_FAIL(databuff_printf(uri_, sizeof(uri_), "%.*s", uri.length(), uri.ptr()))) {
     STORAGE_LOG(WARN, "failed to fill uri", K(ret), K(uri));
   } else if (OB_FAIL(get_storage_type_from_path(uri, type))) {
     STORAGE_LOG(WARN, "failed to get type", K(ret), K(uri));
-  } else if (OB_STORAGE_OSS == type) {
-    appender_ = &oss_appender_;
-    if (OB_ISNULL(storage_info_ = allocator_.alloc(sizeof(ObOssAccount)))) {
-      ret = OB_ALLOCATE_MEMORY_FAILED;
-      STORAGE_LOG(WARN, "failed to alloc memory", K(ret), K(sizeof(ObOssAccount)));
-    } else {
-      MEMCPY(storage_info_, obj_base_info, sizeof(ObOssAccount));
+  } else if (OB_STORAGE_OSS == type || OB_STORAGE_COS == type) {
+    if (OB_FAIL(storage_info_.assign(*storage_info))) {
+      STORAGE_LOG(WARN, "failed to copy storage info", K(ret));
+    } else if (OB_STORAGE_OSS == type) {
+      appender_ = &oss_appender_;
+    } else if (OB_STORAGE_COS == type) {
+      appender_ = &cos_appender_;
     }
   } else if (OB_STORAGE_FILE == type) {
     appender_ = &file_appender_;
@@ -842,7 +833,7 @@ int ObStorageAppender::open(
     if (OB_ISNULL(appender_)) {
       ret = OB_ERR_UNEXPECTED;
       STORAGE_LOG(WARN, "appender_ is null", K(ret), K(uri));
-    } else if (OB_FAIL(appender_->open(uri, obj_base_info))) {
+    } else if (OB_FAIL(appender_->open(uri, storage_info))) {
       STORAGE_LOG(WARN, "failed to open writer", K(ret), K(uri));
     } else {
       is_opened_ = true;
@@ -900,7 +891,7 @@ int ObStorageAppender::repeatable_pwrite_(const char *buf, const int64_t size, c
   if (OB_ISNULL(appender_)) {
     ret = OB_NOT_INIT;
     STORAGE_LOG(WARN, "not opened", K(ret));
-  } else if (OB_FAIL(reader.open(uri_, storage_info_))) {
+  } else if (OB_FAIL(reader.open(uri_, &storage_info_))) {
     STORAGE_LOG(WARN, "failed to open reader", K(ret));
   } else if (reader.get_length() <= offset) {
     // This situation also has concurrency issues.
@@ -982,7 +973,7 @@ int ObStorageAppender::close()
     print_access_storage_log("storage appender_", uri_, start_ts_, appender_->get_length());
   }
 
-  if (OB_ISNULL(appender_) || !is_opened_) {
+  if (OB_ISNULL(appender_)) {
     ret = OB_NOT_INIT;
     STORAGE_LOG(WARN, "not opened", K(ret));
   } else if (OB_FAIL(appender_->close())) {

deps/oblib/src/lib/restore/ob_storage.h

@@ -15,6 +15,7 @@
 #include "ob_i_storage.h"
 #include "ob_storage_file.h"
 #include "ob_storage_oss_base.h"
+#include "ob_storage_cos_base.h"
 
 namespace oceanbase
 {
@@ -112,7 +113,7 @@ public:
   // When physical backup lease is timeout, retry won't stop until 300s.
   explicit ObStorageUtil();
   virtual ~ObStorageUtil() {}
-  int open(void* obj_base, int device_type);
+  int open(common::ObObjectStorageInfo *storage_info);
   void close();
   int is_exist(const common::ObString &uri, bool &exist);
   int get_file_length(const common::ObString &uri, int64_t &file_length);
@@ -121,18 +122,18 @@ public:
   int write_single_file(const common::ObString &uri, const char *buf, const int64_t size);
   int list_files(const common::ObString &dir_path, common::ObBaseDirEntryOperator &op);
   int del_dir(const common::ObString &uri);
-  int check_backup_dest_lifecycle(const common::ObString &path, bool &is_set_lifecycle);
   int list_directories(const common::ObString &dir_path, common::ObBaseDirEntryOperator &op);
   int is_tagging(const common::ObString &uri, bool &is_tagging);
 
 private:
-//we does not use obj_base_info_ to judge init, since for nfs&local, obj_base_info_ is null
+  // we does not use storage_info_ to judge init, since for nfs&local, storage_info_ is null
   bool is_init() {return init_state;}
 
   ObStorageFileUtil file_util_;
   ObStorageOssUtil oss_util_;
+  ObStorageCosUtil cos_util_;
   ObIStorageUtil* util_;
-  void* obj_base_info_;
+  common::ObObjectStorageInfo* storage_info_;
   bool init_state;
   DISALLOW_COPY_AND_ASSIGN(ObStorageUtil);
 };
@@ -142,7 +143,7 @@ class ObStorageReader
 public:
   ObStorageReader();
   virtual ~ObStorageReader();
-  int open(const common::ObString &uri, void* obj_base_info);
+  int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info);
   int pread(char *buf,const int64_t buf_size, int64_t offset, int64_t &read_size);
   int close();
   int64_t get_length() const { return file_length_; }
@@ -151,6 +152,7 @@ private:
   ObIStorageReader *reader_;
   ObStorageFileReader file_reader_;
   ObStorageOssReader oss_reader_;
+  ObStorageCosReader cos_reader_;
   int64_t start_ts_;
   char uri_[OB_MAX_URI_LENGTH];
   DISALLOW_COPY_AND_ASSIGN(ObStorageReader);
@@ -161,13 +163,14 @@ class ObStorageWriter
 public:
   ObStorageWriter();
   virtual ~ObStorageWriter();
-  int open(const common::ObString &uri, void* obj_base_info);
+  int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info);
   int write(const char *buf,const int64_t size);
   int close();
 private:
   ObIStorageWriter *writer_;
   ObStorageFileWriter file_writer_;
   ObStorageOssWriter oss_writer_;
+  ObStorageCosWriter cos_writer_;
   int64_t start_ts_;
   char uri_[OB_MAX_URI_LENGTH];
   DISALLOW_COPY_AND_ASSIGN(ObStorageWriter);
@@ -186,7 +189,7 @@ public:
     ObStorageObjectVersionParam version_param_;
   };
 
-  int open(const common::ObString &uri, void* obj_base_info, const AppenderParam &param);
+  int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info);
   int write(const char *buf,const int64_t size);
   int pwrite(const char *buf, const int64_t size, const int64_t offset);
   int close();
@@ -198,10 +201,11 @@ private:
   ObIStorageWriter *appender_;
   ObStorageFileAppender file_appender_;
   ObStorageOssAppendWriter oss_appender_;
+  ObStorageCosAppendWriter cos_appender_;
   int64_t start_ts_;
   bool is_opened_;
   char uri_[OB_MAX_URI_LENGTH];
-  void *storage_info_;
+  common::ObObjectStorageInfo storage_info_;
   ObArenaAllocator allocator_;
 
   int repeatable_pwrite_(const char *buf, const int64_t size, const int64_t offset);

deps/oblib/src/lib/restore/ob_storage_cos_base.cpp

@@ -0,0 +1,934 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include "lib/restore/ob_storage_cos_base.h"
+#include <stdlib.h>
+#include <libgen.h>
+#include "common/ob_string_buf.h"
+#include "apr_errno.h"
+#include "ob_storage.h"
+#include "lib/hash/ob_hashset.h"
+#include "lib/utility/ob_tracepoint.h"
+
+namespace oceanbase
+{
+namespace common
+{
+using namespace oceanbase::common;
+
+/*--------------------------------GLOBAL---------------------------*/
+int init_cos_env()
+{
+  return qcloud_cos::ObCosEnv::get_instance().init();
+}
+
+void fin_cos_env()
+{
+  // wait doing io finish before destroy cos env.
+  const int64_t start_time = ObTimeUtility::current_time();
+  const int64_t timeout = ObExternalIOCounter::FLYING_IO_WAIT_TIMEOUT;
+  int64_t flying_io_cnt = ObExternalIOCounter::get_flying_io_cnt();
+  while(0 < flying_io_cnt) {
+    const int64_t end_time = ObTimeUtility::current_time();
+    if (end_time - start_time > timeout) {
+      OB_LOG(INFO, "force fin_cos_env", K(flying_io_cnt));
+      break;
+    }
+    usleep(100 * 1000L); // 100ms
+    flying_io_cnt = ObExternalIOCounter::get_flying_io_cnt();
+  }
+
+  qcloud_cos::ObCosEnv::get_instance().destroy();
+}
+
+struct CosListFilesCbArg
+{
+  common::ObIAllocator &allocator;
+  ObString &dir_path;
+  ObBaseDirEntryOperator &list_op;
+
+  CosListFilesCbArg(
+    common::ObIAllocator &alloc,
+    ObString &dir,
+    ObBaseDirEntryOperator &op)
+    : allocator(alloc),
+      dir_path(dir),
+      list_op(op) {}
+
+  ~CosListFilesCbArg() {}
+};
+
+static int execute_list_callback(
+  ObBaseDirEntryOperator &op,
+  qcloud_cos::ObCosWrapper::CosListObjPara &para,
+  ObString &file_name)
+{
+  int ret = OB_SUCCESS;
+  int64_t dname_size = sizeof(para.last_container_name_.d_name);
+  if (dname_size >= (file_name.length() + 1)) {
+    if (OB_FAIL(databuff_printf(para.last_container_name_.d_name, dname_size, "%s", file_name.ptr()))) {
+      OB_LOG(WARN, "fail to copy file name to entry buf!", K(ret), K(dname_size));
+    } else if (OB_FAIL(op.func(&(para.last_container_name_)))) {
+      OB_LOG(WARN, "fail to execute list callback!", K(ret), KCSTRING(para.last_container_name_.d_name),
+            K(para.last_container_name_.d_type), K(DT_REG), K(dname_size));
+    }
+  } else {
+    ret = OB_BUF_NOT_ENOUGH;
+    OB_LOG(WARN, "file name len is too long!", K(ret), K(file_name.length()), K(sizeof(dirent)));
+  }
+  return ret;
+}
+
+static int handle_object_name_cb(qcloud_cos::ObCosWrapper::CosListObjPara &para)
+{
+  int ret = OB_SUCCESS;
+  para.next_flag_ = true;
+  if (OB_ISNULL(para.arg_)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "arg is empty", K(ret));
+  } else if (OB_ISNULL(para.cur_full_path_slice_name_) || 0 >= para.full_path_size_) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "current object_name is empty", K(ret));
+  } else if (qcloud_cos::ObCosWrapper::CosListObjPara::CosListType::COS_LIST_CB_ARG == para.type_) {
+    CosListFilesCbArg *ctx = static_cast<CosListFilesCbArg *>(para.arg_);
+
+    // Returned object name is the whole object path, but we donot need the prefix dir_path.
+    // So, we trim the dir_name from object_name path.
+    const int dir_name_str_len = strlen(ctx->dir_path.ptr());
+    const int object_name_len = para.full_path_size_ - dir_name_str_len;
+    const int object_name_start_pos = dir_name_str_len;
+    //tmp_file_name like slice name
+    ObString tmp_file_name(object_name_len, object_name_len,
+      para.cur_full_path_slice_name_ + object_name_start_pos);
+    if (OB_FAIL(execute_list_callback(ctx->list_op, para, tmp_file_name))) {
+      OB_LOG(WARN, "fail to execute list callback!", K(ret));
+    }
+  } else {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "not supported type", K(ret), K(para.type_));
+  }
+
+  return ret;
+}
+
+static int handle_directory_name_cb(
+    void *arg,
+    const qcloud_cos::ObCosWrapper::CosListObjPara::CosListType type,
+    const char *object_name,
+    int64_t object_size)
+{
+  int ret = OB_SUCCESS;
+  const char delimiter = '/';
+  if (OB_ISNULL(arg)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "arg is empty", K(ret), KP(arg));
+  } else if (OB_ISNULL(object_name) || 0 >= object_size) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "object_name is empty", K(ret), KP(object_name), K(object_size));
+  } else if (delimiter != object_name[object_size - 1]) {
+    ret = OB_ERR_UNEXPECTED;
+    OB_LOG(WARN, "object name is unexpected", K(ret), KCSTRING(object_name), K(object_size));
+  } else {
+    const int64_t name_length = object_size - 1;
+    if (name_length <= 0) {
+      ret = OB_ERR_UNEXPECTED;
+      OB_LOG(WARN, "object is not exist", K(ret), KCSTRING(object_name), K(object_size));
+    } else if (qcloud_cos::ObCosWrapper::CosListObjPara::CosListType::COS_LIST_CB_ARG == type) {
+      CosListFilesCbArg *ctx = static_cast<CosListFilesCbArg *>(arg);
+      ObString directory_name;
+      dirent dir_name_entry;
+      dir_name_entry.d_type = DT_DIR;
+      ObString tmp_directory_name(name_length, name_length, object_name);
+      if (OB_FAIL(databuff_printf(dir_name_entry.d_name, sizeof(dir_name_entry.d_name), "%.*s",
+                                  tmp_directory_name.length(), tmp_directory_name.ptr()))) {
+        OB_LOG(WARN, "fail to set dir name entry", K(ret), K(tmp_directory_name));
+      } else if (OB_FAIL(ctx->list_op.func(&dir_name_entry))) {
+        OB_LOG(WARN, "fail to call list directories operation callback", K(ret), K(tmp_directory_name));
+      }
+    } else {
+      ret = OB_INVALID_ARGUMENT;
+      OB_LOG(WARN, "not supported type", K(ret), K(type));
+    }
+  }
+  return ret;
+}
+
+/*--------------------------------ObStorageCosUtil---------------------------*/
+
+ObStorageCosUtil::ObStorageCosUtil()
+  : is_opened_(false), storage_info_(NULL)
+{
+}
+
+ObStorageCosUtil::~ObStorageCosUtil()
+{
+}
+
+int ObStorageCosUtil::open(ObObjectStorageInfo *storage_info)
+{
+  int ret = OB_SUCCESS;
+  if (OB_UNLIKELY(is_opened_)) {
+    ret = OB_COS_ERROR;
+    OB_LOG(WARN, "cos util already open, cannot open again", K(ret));
+  } else if (OB_ISNULL(storage_info)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "storage info is null", K(ret));
+  } else {
+    storage_info_ = storage_info;
+    is_opened_ = true;
+  }
+  return ret;
+}
+
+void ObStorageCosUtil::close()
+{
+  is_opened_ = false;
+  storage_info_ = NULL;
+}
+
+int ObStorageCosUtil::is_exist(const ObString &uri, bool &is_exist)
+{
+  int ret = OB_SUCCESS;
+  ObExternalIOCounterGuard io_guard;
+  is_exist = false;
+  int64_t file_length = 0;
+  if (OB_UNLIKELY(!is_opened_)) {
+    ret = OB_COS_ERROR;
+    OB_LOG(WARN, "cos util not opened", K(ret));
+  } else if (OB_UNLIKELY(uri.empty())) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid argument", K(ret), K(uri));
+  } else if (OB_FAIL(get_object_meta_(uri, is_exist, file_length))) {
+    OB_LOG(WARN, "fail to get object meta", K(ret), K(uri));
+  }
+  return ret;
+}
+
+int ObStorageCosUtil::get_file_length(const ObString &uri, int64_t &file_length)
+{
+  int ret = OB_SUCCESS;
+  ObExternalIOCounterGuard io_guard;
+  bool is_file_exist = false;
+  if (OB_UNLIKELY(!is_opened_)) {
+    ret = OB_COS_ERROR;
+    OB_LOG(WARN, "cos util not opened", K(ret));
+  } else if (OB_UNLIKELY(uri.empty())) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid argument", K(ret), K(uri));
+  } else if (OB_FAIL(get_object_meta_(uri, is_file_exist, file_length))) {
+    OB_LOG(WARN, "fail to get object meta", K(ret), K(uri));
+  } else if (!is_file_exist) {
+    ret = OB_BACKUP_FILE_NOT_EXIST;
+    OB_LOG(WARN, "backup file is not exist", K(ret), K(uri));
+  }
+  return ret;
+}
+
+// inner function, won't check params valid or not.
+int ObStorageCosUtil::get_object_meta_(
+    const ObString &uri,
+    bool &is_file_exist,
+    int64_t &file_length)
+{
+  int ret = OB_SUCCESS;
+
+  ObStorageCosBase cos_base;
+  qcloud_cos::CosObjectMeta obj_meta;
+  is_file_exist = false;
+  if (OB_FAIL(cos_base.open(uri, storage_info_))) {
+    OB_LOG(WARN, "fail to open cos base", K(ret), K(uri));
+  } else if (OB_FAIL(cos_base.get_cos_file_meta(is_file_exist, obj_meta))) {
+    OB_LOG(WARN, "fail to get object meta", K(ret));
+  } else {
+    file_length = obj_meta.file_length_;
+  }
+  cos_base.reset();
+
+  return ret;
+}
+
+int ObStorageCosUtil::write_single_file(
+    const ObString &uri,
+    const char *buf,
+    const int64_t size)
+{
+  int ret = OB_SUCCESS;
+  ObExternalIOCounterGuard io_guard;
+  ObStorageCosWriter writer;
+  if (OB_FAIL(writer.open(uri, storage_info_))) {
+    OB_LOG(WARN, "fail to open cos writer", K(ret), K(uri), KP_(storage_info));
+  } else if (OB_FAIL(writer.write(buf, size))) {
+    OB_LOG(WARN, "fail to write into cos", K(ret), K(size), KP(buf));
+  } else if (OB_FAIL(writer.close())) {
+    OB_LOG(WARN, "fail to close cos writer", K(ret));
+  }
+  return ret;
+}
+
+int ObStorageCosUtil::mkdir(const ObString &uri)
+{
+  int ret = OB_SUCCESS;
+  OB_LOG(DEBUG, "no need to create dir in cos", K(uri));
+  UNUSED(uri);
+  return ret;
+}
+
+int ObStorageCosUtil::del_dir(const ObString &uri)
+{
+  int ret = OB_SUCCESS;
+  OB_LOG(DEBUG, "no need to del dir in cos", K(uri));
+  UNUSED(uri);
+  return ret;
+}
+
+int ObStorageCosUtil::is_tagging(
+    const ObString &uri,
+    bool &is_tagging)
+{
+  int ret = OB_SUCCESS;
+  ObExternalIOCounterGuard io_guard;
+  is_tagging = false;
+
+  if (OB_UNLIKELY(!is_opened_)) {
+    ret = OB_COS_ERROR;
+    OB_LOG(WARN, "cos util not opened", K(ret));
+  } else if (OB_UNLIKELY(uri.empty())) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid argument", K(ret), K(uri));
+  } else {
+    ObStorageCosBase cos_base;
+    if (OB_FAIL(cos_base.open(uri, storage_info_))) {
+      OB_LOG(WARN, "fail to open cos base", K(ret), K(uri));
+    } else if (OB_FAIL(cos_base.is_object_tagging(uri, is_tagging))) {
+      OB_LOG(WARN, "fail to check object tag", K(ret), K(uri));
+    }
+    cos_base.reset();
+  }
+  return ret;
+}
+
+int ObStorageCosUtil::del_file(const ObString &uri)
+{
+  int ret = OB_SUCCESS;
+  ObExternalIOCounterGuard io_guard;
+
+  if (OB_UNLIKELY(!is_opened_)) {
+    ret = OB_COS_ERROR;
+    OB_LOG(WARN, "cos util not opened", K(ret));
+  } else if (OB_UNLIKELY(uri.empty())) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid argument", K(ret), K(uri));
+  } else {
+    ObStorageCosBase cos_base;
+    if (OB_FAIL(cos_base.open(uri, storage_info_))) {
+      OB_LOG(WARN, "fail to open cos base", K(ret), K(uri));
+    } else if (OB_FAIL(cos_base.delete_object(uri))) {
+      OB_LOG(WARN, "fail to get object meta", K(ret));
+    } else {
+      OB_LOG(DEBUG, "succ to delete object", K(uri));
+    }
+    cos_base.reset();
+  }
+  return ret;
+}
+
+int ObStorageCosUtil::list_files(
+    const ObString &uri,
+    ObBaseDirEntryOperator &op)
+{
+  int ret = OB_SUCCESS;
+  ObExternalIOCounterGuard io_guard;
+  ObArenaAllocator allocator;
+
+  if (OB_UNLIKELY(!is_opened_)) {
+    ret = OB_COS_ERROR;
+    OB_LOG(WARN, "cos util not opened", K(ret));
+  } else if (OB_UNLIKELY(uri.empty())) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid argument", K(ret), K(uri));
+  } else {
+    char dir_path_array[OB_MAX_URI_LENGTH] = {0};
+    const char *ptr = NULL; // point to actual dir path
+    const int min_dir_path_len = 2; // cuz dir_path end with '\0'
+    const char *separator = "/";
+    const char end_marker = '\0';
+
+    ObStorageCosBase cos_base;
+    if (OB_FAIL(cos_base.open(uri, storage_info_))) {
+      OB_LOG(WARN, "fail to open cos base", K(ret), K(uri));
+    } else {
+      const ObString &dir_path_str = cos_base.get_handle().get_object_name();
+      const ObString::obstr_size_t dir_path_len = dir_path_str.length();
+      int32_t actual_dir_path_len = dir_path_len;
+      if (dir_path_len < min_dir_path_len) {
+        ret = OB_ERR_UNEXPECTED;
+        OB_LOG(WARN, "dir_path_str length is unepxected", K(ret), K(dir_path_str), K(dir_path_len));
+      } else {
+        // Let dir path end with '/'
+        if (dir_path_str.ptr()[dir_path_len - 1] != end_marker) {
+          ret = OB_ERR_UNEXPECTED;
+          OB_LOG(WARN, "dir_path_str not end with '\0'");
+        } else if (dir_path_str.ptr()[dir_path_len - 2] == *separator) {
+          ptr = dir_path_str.ptr();
+        } else if (OB_FAIL(databuff_printf(dir_path_array, OB_MAX_URI_LENGTH, "%s/", dir_path_str.ptr()))) {
+          OB_LOG(WARN, "fail to deep copy dir path", K(ret), K(dir_path_len), K(OB_MAX_URI_LENGTH));
+        } else {
+          ptr = dir_path_array;
+          actual_dir_path_len += 1;
+        }
+      }
+
+      if (OB_SUCC(ret)) {
+        ObString tmp_dir;
+        tmp_dir.assign(const_cast<char *>(ptr), actual_dir_path_len);
+
+        // Construct list object callback arg
+        CosListFilesCbArg arg(allocator, tmp_dir, op);
+        if (OB_FAIL(cos_base.list_objects(uri, tmp_dir, separator, arg))) {
+          OB_LOG(WARN, "fail to list object in cos_base", K(ret), K(uri), K(tmp_dir));
+        }
+      }
+    }
+    cos_base.reset();
+  }
+  return ret;
+}
+
+int ObStorageCosUtil::list_directories(
+    const ObString &uri,
+    ObBaseDirEntryOperator &op)
+{
+  int ret = OB_SUCCESS;
+  ObExternalIOCounterGuard io_guard;
+  ObArenaAllocator allocator;
+
+  if (OB_UNLIKELY(!is_opened_)) {
+    ret = OB_COS_ERROR;
+    OB_LOG(WARN, "cos util not opened", K(ret));
+  } else if (OB_UNLIKELY(uri.empty())) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid argument", K(ret), K(uri));
+  } else {
+    char dir_path_array[OB_MAX_URI_LENGTH] = {0};
+    const char *ptr = NULL; // point to actual dir path
+    const int min_dir_path_len = 2; // cuz dir_path end with '\0'
+    const char *delimiter_string = "/";
+    const char *next_marker_string = "";
+    const char end_marker = '\0';
+
+    ObStorageCosBase cos_base;
+    if (OB_FAIL(cos_base.open(uri, storage_info_))) {
+      OB_LOG(WARN, "fail to open cos base", K(ret), K(uri));
+    } else {
+      const ObString &dir_path_str = cos_base.get_handle().get_object_name();
+      const ObString::obstr_size_t dir_path_len = dir_path_str.length();
+      int32_t actual_dir_path_len = dir_path_len;
+      if (dir_path_len < min_dir_path_len) {
+        ret = OB_ERR_UNEXPECTED;
+        OB_LOG(WARN, "dir_path_str length is unepxected", K(ret), K(dir_path_str), K(dir_path_len));
+      } else {
+        // Let dir path end with '/'
+        if (dir_path_str.ptr()[dir_path_len - 1] != end_marker) {
+          ret = OB_ERR_UNEXPECTED;
+          OB_LOG(WARN, "dir_path_str not end with '\0'");
+        } else if (dir_path_str.ptr()[dir_path_len - 2] == *delimiter_string) {
+          ptr = dir_path_str.ptr();
+        } else if (OB_FAIL(databuff_printf(dir_path_array, OB_MAX_URI_LENGTH, "%s/", dir_path_str.ptr()))) {
+          OB_LOG(WARN, "fail to deep copy dir path", K(ret), K(dir_path_len), K(OB_MAX_URI_LENGTH));
+        } else {
+          ptr = dir_path_array;
+          actual_dir_path_len += 1;
+        }
+      }
+
+      if (OB_SUCC(ret)) {
+        ObString tmp_dir;
+        tmp_dir.assign(const_cast<char *>(ptr), actual_dir_path_len);
+
+        CosListFilesCbArg arg(allocator, tmp_dir, op);
+        if (OB_FAIL(cos_base.list_directories(uri, tmp_dir, next_marker_string, delimiter_string, arg))) {
+          OB_LOG(WARN, "fail to list directories in cos_base", K(ret), K(uri), K(tmp_dir));
+        }
+      }
+    }
+    cos_base.reset();
+  }
+  return ret;
+}
+
+/*--------------------------------ObStorageCosBase---------------------------*/
+
+ObStorageCosBase::ObStorageCosBase()
+  : is_opened_(false), handle_()
+{
+}
+
+ObStorageCosBase::~ObStorageCosBase()
+{
+  reset();
+}
+
+void ObStorageCosBase::reset()
+{
+  handle_.reset();
+  is_opened_ = false;
+}
+
+int ObStorageCosBase::init_handle(const ObObjectStorageInfo &storage_info)
+{
+  int ret = OB_SUCCESS;
+  if (OB_UNLIKELY(handle_.is_inited())) {
+    ret = OB_INIT_TWICE;
+    OB_LOG(WARN, "handle in cos base already inited", K(ret));
+  } else if (OB_FAIL(handle_.init(&storage_info))) {
+    OB_LOG(WARN, "fail to init cos wrapper handle", K(ret));
+  }
+  return ret;
+}
+
+int ObStorageCosBase::open(
+    const ObString &uri,
+    ObObjectStorageInfo *storage_info)
+{
+  int ret = OB_SUCCESS;
+  if (OB_UNLIKELY(uri.empty()) || OB_ISNULL(storage_info)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "uri is empty", K(ret), K(uri), KP(storage_info));
+  } else if (OB_FAIL(init_handle(*storage_info))) {
+    OB_LOG(WARN, "failed to init cos wrapper handle", K(ret), K(uri));
+  } else if (OB_FAIL(handle_.create_cos_handle())) {
+    OB_LOG(WARN, "failed to create cos handle", K(ret), K(uri));
+  } else if (OB_FAIL(handle_.build_bucket_and_object_name(uri))) {
+    OB_LOG(WARN, "failed to build bucket and object name", K(ret), K(uri));
+  }
+  return ret;
+}
+
+int ObStorageCosBase::get_cos_file_meta(
+    bool &is_file_exist,
+    qcloud_cos::CosObjectMeta &obj_meta)
+{
+  int ret = OB_SUCCESS;
+  if (OB_LIKELY(is_valid())) {
+    qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer(
+        handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length());
+    qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer(
+        handle_.get_object_name().ptr(), handle_.get_object_name().length());
+
+    is_file_exist = false;
+    if (OB_FAIL(qcloud_cos::ObCosWrapper::head_object_meta(handle_.get_ptr(), bucket_name,
+        object_name, is_file_exist, obj_meta))) {
+      OB_LOG(WARN, "fail to get object meta", K(ret));
+    }
+  } else {
+    ret = OB_ERR_UNEXPECTED;
+    OB_LOG(WARN, "cos wrapper handle not init or create", K(ret));
+  }
+  return ret;
+}
+
+int ObStorageCosBase::delete_object(const ObString &uri)
+{
+  int ret = OB_SUCCESS;
+  if (OB_LIKELY(is_valid())) {
+    qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer(
+        handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length());
+    qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer(
+        handle_.get_object_name().ptr(), handle_.get_object_name().length());
+
+    if (ObIStorageUtil::DELETE == handle_.get_delete_mode()) {
+      if (OB_FAIL(qcloud_cos::ObCosWrapper::del(handle_.get_ptr(), bucket_name, object_name))) {
+        OB_LOG(WARN, "fail to delete object meta", K(ret), K(uri));
+      }
+    } else if (ObIStorageUtil::TAGGING == handle_.get_delete_mode()) {
+      if (OB_FAIL(qcloud_cos::ObCosWrapper::tag(handle_.get_ptr(), bucket_name, object_name))) {
+        OB_LOG(WARN, "fail to tag object", K(ret), K(uri));
+      }
+    } else {
+      ret = OB_INVALID_ARGUMENT;
+      OB_LOG(WARN, "cos delete mode invalid", K(ret), K(uri));
+    }
+  } else {
+    ret = OB_ERR_UNEXPECTED;
+    OB_LOG(WARN, "cos wrapper handle not init or create", K(ret));
+  }
+  return ret;
+}
+
+int ObStorageCosBase::list_objects(
+    const ObString &uri,
+    const ObString &dir_name_str,
+    const char *separator,
+    CosListFilesCbArg &arg)
+{
+  int ret = OB_SUCCESS;
+  if (OB_ISNULL(separator)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid argument", K(ret));
+  } else if (OB_LIKELY(is_valid())) {
+    qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer(
+        handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length());
+    qcloud_cos::CosStringBuffer dir_name(dir_name_str.ptr(), dir_name_str.length());
+    qcloud_cos::CosStringBuffer next_marker(separator, strlen(separator) + 1);
+    if (OB_FAIL(qcloud_cos::ObCosWrapper::list_objects(handle_.get_ptr(), bucket_name,
+        dir_name, next_marker, handle_object_name_cb, (void *)(&arg)))) {
+      OB_LOG(WARN, "fail to list objects", K(ret), K(uri));
+    }
+  } else {
+    ret = OB_ERR_UNEXPECTED;
+    OB_LOG(WARN, "cos wrapper handle not init or create", K(ret));
+  }
+  return ret;
+}
+
+int ObStorageCosBase::list_directories(
+    const ObString &uri,
+    const ObString &dir_name_str,
+    const char *next_marker_str,
+    const char *delimiter_str,
+    CosListFilesCbArg &arg)
+{
+  int ret = OB_SUCCESS;
+  if (OB_ISNULL(next_marker_str) || OB_ISNULL(delimiter_str)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid argument", K(ret), KP(next_marker_str), KP(delimiter_str));
+  } else if (OB_LIKELY(is_valid())) {
+    qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer(
+        handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length());
+    qcloud_cos::CosStringBuffer dir_name(dir_name_str.ptr(), dir_name_str.length());
+    qcloud_cos::CosStringBuffer next_marker(next_marker_str, strlen(next_marker_str) + 1);
+    qcloud_cos::CosStringBuffer delimiter(delimiter_str, strlen(delimiter_str) + 1);
+    if (OB_FAIL(qcloud_cos::ObCosWrapper::list_directories(handle_.get_ptr(), bucket_name, dir_name,
+        next_marker, delimiter, handle_directory_name_cb, (void *)(&arg)))) {
+      OB_LOG(WARN, "failed to list directories", K(ret), K(uri));
+    }
+  } else {
+    ret = OB_ERR_UNEXPECTED;
+    OB_LOG(WARN, "cos wrapper handle not init or create", K(ret));
+  }
+  return ret;
+}
+
+int ObStorageCosBase::is_object_tagging(
+    const ObString &uri,
+    bool &is_tagging)
+{
+  int ret = OB_SUCCESS;
+  if (OB_LIKELY(is_valid())) {
+    qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer(
+        handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length());
+    qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer(
+        handle_.get_object_name().ptr(), handle_.get_object_name().length());
+    if (OB_FAIL(qcloud_cos::ObCosWrapper::is_object_tagging(handle_.get_ptr(),
+        bucket_name, object_name, is_tagging))) {
+      OB_LOG(WARN, "fail to check object tagging", K(ret), K(uri));
+    }
+  } else {
+    ret = OB_ERR_UNEXPECTED;
+    OB_LOG(WARN, "cos wrapper handle not init or create", K(ret));
+  }
+  return ret;
+}
+
+/*--------------------------------ObStorageCosReader---------------------------*/
+
+ObStorageCosReader::ObStorageCosReader()
+  : ObStorageCosBase(), file_length_(-1)
+{
+}
+
+ObStorageCosReader::~ObStorageCosReader()
+{
+}
+
+int ObStorageCosReader::open(const ObString &uri, ObObjectStorageInfo *storage_info)
+{
+  int ret = OB_SUCCESS;
+  ObExternalIOCounterGuard io_guard;
+
+  if (OB_UNLIKELY(is_opened_)) {
+    ret = OB_COS_ERROR;
+    OB_LOG(WARN, "cos reader already open, cannot open again", K(ret), K(uri));
+  } else if (OB_FAIL(ObStorageCosBase::open(uri, storage_info))) {
+    OB_LOG(WARN, "fail to open in cos_base", K(ret), K(uri));
+  } else {
+    const ObString &bucket_str = handle_.get_bucket_name();
+    const ObString &object_str = handle_.get_object_name();
+    qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer(
+        bucket_str.ptr(), bucket_str.length());
+    qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer(
+        object_str.ptr(), object_str.length());
+    bool is_file_exist = false;
+    qcloud_cos::CosObjectMeta obj_meta;
+    if (OB_FAIL(qcloud_cos::ObCosWrapper::head_object_meta(handle_.get_ptr(), bucket_name,
+        object_name, is_file_exist, obj_meta))) {
+      OB_LOG(WARN, "fail to get object meta", K(ret), K(bucket_str), K(object_str));
+    } else if (!is_file_exist) {
+      ret = OB_BACKUP_FILE_NOT_EXIST;
+      OB_LOG(WARN, "backup file is not exist", K(ret), K(bucket_str), K(object_str));
+    } else {
+      file_length_ = obj_meta.file_length_;
+      is_opened_ = true;
+    }
+  }
+  return ret;
+}
+
+int ObStorageCosReader::pread(
+    char *buf,
+    const int64_t buf_size,
+    int64_t offset,
+    int64_t &read_size)
+{
+  int ret = OB_SUCCESS;
+  ObExternalIOCounterGuard io_guard;
+
+  if (!is_opened_) {
+    ret = OB_COS_ERROR;
+    OB_LOG(WARN, "cos reader cannot read before it is opened", K(ret));
+  } else if (NULL == buf || buf_size <= 0 || offset < 0) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid argument", K(ret), KP(buf), K(buf_size), K(offset));
+  } else {
+    qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer(
+        handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length());
+    qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer(
+        handle_.get_object_name().ptr(), handle_.get_object_name().length());
+    if (OB_FAIL(qcloud_cos::ObCosWrapper::pread(handle_.get_ptr(), bucket_name,
+        object_name, offset, buf, buf_size, read_size))) {
+      OB_LOG(WARN, "fail to read object from cos", K(ret), KP(buf), K(buf_size), K(offset));
+    }
+  }
+  return ret;
+}
+
+int ObStorageCosReader::close()
+{
+  int ret = OB_SUCCESS;
+  ObExternalIOCounterGuard io_guard;
+  file_length_ = -1;
+  reset();
+  return ret;
+}
+
+/*--------------------------------ObStorageCosWriter---------------------------*/
+
+ObStorageCosWriter::ObStorageCosWriter()
+  : ObStorageCosBase(), file_length_(-1)
+{
+}
+
+ObStorageCosWriter::~ObStorageCosWriter()
+{
+  if (is_opened_) {
+    close();
+  }
+}
+
+int ObStorageCosWriter::open(
+    const ObString &uri,
+    ObObjectStorageInfo *storage_info)
+{
+  int ret = OB_SUCCESS;
+  if (OB_UNLIKELY(is_opened_)) {
+    ret = OB_COS_ERROR;
+    OB_LOG(WARN, "cos writer already open, cannot open again", K(ret), K(uri));
+  } else if (OB_FAIL(ObStorageCosBase::open(uri, storage_info))) {
+    OB_LOG(WARN, "fail to open in cos_base", K(ret), K(uri));
+  } else {
+    file_length_ = 0;
+    is_opened_ = true;
+  }
+  return ret;
+}
+
+int ObStorageCosWriter::close()
+{
+  int ret = OB_SUCCESS;
+  file_length_ = -1;
+  reset();
+  return ret;
+}
+
+int ObStorageCosWriter::pwrite(const char *buf, const int64_t size, const int64_t offset)
+{
+  int ret = OB_NOT_SUPPORTED;
+  UNUSED(buf);
+  UNUSED(size);
+  UNUSED(offset);
+  return ret;
+}
+
+int ObStorageCosWriter::write(const char *buf, const int64_t size)
+{
+  int ret = OB_SUCCESS;
+  if (!is_opened_) {
+    ret = OB_NOT_INIT;
+    OB_LOG(WARN, "cos writer not opened", K(ret));
+  } else if (NULL == buf || size < 0) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "buf is NULL or size is invalid", K(ret), KP(buf), K(size));
+  } else {
+    qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer(
+        handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length());
+    qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer(
+        handle_.get_object_name().ptr(), handle_.get_object_name().length());
+    if (OB_FAIL(qcloud_cos::ObCosWrapper::put(handle_.get_ptr(), bucket_name,
+        object_name, buf, size))) {
+      OB_LOG(WARN, "fail to write object into cos", K(ret), KP(buf), K(size));
+    } else {
+      file_length_ += size;
+    }
+  }
+  return ret;
+}
+
+/*--------------------------------ObStorageCosAppendWriter---------------------------*/
+
+ObStorageCosAppendWriter::ObStorageCosAppendWriter()
+  : ObStorageCosBase(),
+    file_length_(-1)
+{
+}
+
+ObStorageCosAppendWriter::~ObStorageCosAppendWriter()
+{
+}
+
+int ObStorageCosAppendWriter::open(
+    const ObString &uri,
+    ObObjectStorageInfo *storage_info)
+{
+  int ret = OB_SUCCESS;
+  ObExternalIOCounterGuard io_guard;
+
+  if (OB_UNLIKELY(is_opened_)) {
+    ret = OB_COS_ERROR;
+    OB_LOG(WARN, "cos appender already open, cannot open again", K(ret), K(uri));
+  } else if (OB_FAIL(ObStorageCosBase::open(uri, storage_info))) {
+    OB_LOG(WARN, "fail to open in cos_base", K(ret), K(uri));
+  } else {
+    file_length_ = 0;
+    is_opened_ = true;
+  }
+  return ret;
+}
+
+int ObStorageCosAppendWriter::write(
+    const char *buf,
+    const int64_t size)
+{
+  int ret = OB_SUCCESS;
+  ObExternalIOCounterGuard io_guard;
+  const int64_t fake_offset = 0;
+  const bool is_pwrite = false;
+
+  if(OB_UNLIKELY(!is_opened_)) {
+    ret = OB_COS_ERROR;
+    OB_LOG(WARN, "cos append writer cannot write before it is not opened", K(ret));
+  } else if(NULL == buf || size < 0) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "buf is NULL or size is invalid", KP(buf), K(size), K(ret));
+  } else if (OB_FAIL(do_write(buf, size, fake_offset, is_pwrite))) {
+    OB_LOG(WARN, "failed to do write", K(ret), KP(buf), K(size));
+  }
+  return ret;
+}
+
+int ObStorageCosAppendWriter::pwrite(const char *buf, const int64_t size, const int64_t offset)
+{
+  int ret = OB_SUCCESS;
+  ObExternalIOCounterGuard io_guard;
+  const bool is_pwrite = true;
+
+  if(OB_UNLIKELY(!is_opened_)) {
+    ret = OB_COS_ERROR;
+    OB_LOG(WARN, "cos append writer cannot write before it is not opened", K(ret));
+  } else if(NULL == buf || size < 0 || offset < 0) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "buf is NULL or size is invalid", KP(buf), K(size), K(ret));
+  } else if (OB_FAIL(do_write(buf, size, offset, is_pwrite))) {
+    OB_LOG(WARN, "failed to do write", K(ret), KP(buf), K(size), K(offset));
+  }
+  return ret;
+}
+
+int ObStorageCosAppendWriter::close()
+{
+  int ret = OB_SUCCESS;
+  ObExternalIOCounterGuard io_guard;
+  file_length_ = -1;
+  reset();
+  return ret;
+}
+
+int ObStorageCosAppendWriter::do_write(
+    const char *buf,
+    const int64_t size,
+    const int64_t offset,
+    const bool is_pwrite)
+{
+  int ret = OB_SUCCESS;
+
+  const int64_t start_time = ObTimeUtility::current_time();
+  if(NULL == buf || size < 0) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "buf is NULL or size is invalid", K(ret), KP(buf), K(size));
+  } else {
+    qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer(
+        handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length());
+    qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer(
+        handle_.get_object_name().ptr(), handle_.get_object_name().length());
+
+    int64_t pos = 0;
+    bool is_exist = false;
+    qcloud_cos::CosObjectMeta obj_meta;
+    bool is_appendable = true;
+    if (OB_FAIL(qcloud_cos::ObCosWrapper::head_object_meta(handle_.get_ptr(), bucket_name,
+        object_name, is_exist, obj_meta))) {
+      OB_LOG(WARN, "fail to get object meta", K(ret));
+    } else if (is_exist) {
+      pos = obj_meta.file_length_;
+      is_appendable = (obj_meta.type_ == qcloud_cos::CosObjectMeta::COS_OBJ_APPENDABLE);
+    }
+
+    if (OB_FAIL(ret)) {
+    } else if (!is_appendable) {
+      ret = OB_CLOUD_OBJECT_NOT_APPENDABLE;
+      OB_LOG(WARN, "we can only append an appendable obj", K(ret), K(is_appendable));
+    } else if (is_pwrite && pos != offset) {
+      ret = OB_BACKUP_PWRITE_OFFSET_NOT_MATCH;
+      OB_LOG(WARN, "offset is not match with real length", K(ret), K(pos), K(offset), K(obj_meta.type_));
+    } else if (OB_FAIL(qcloud_cos::ObCosWrapper::append(handle_.get_ptr(), bucket_name,
+               object_name, buf, size, offset))) {
+      OB_LOG(WARN, "fail to append object in cos", K(ret), KP(buf), K(size), K(offset), K(is_pwrite));
+
+      // If append failed, print the current object meta, to help debugging.
+      int tmp_ret = OB_SUCCESS;
+      obj_meta.reset();
+      is_exist = false;
+      if (OB_TMP_FAIL(qcloud_cos::ObCosWrapper::head_object_meta(handle_.get_ptr(), bucket_name,
+          object_name, is_exist, obj_meta))) {
+        OB_LOG(WARN, "fail to get object meta", K(tmp_ret));
+      } else {
+        OB_LOG(INFO, "after append fail, we got the object meta", K(is_exist), K(obj_meta.type_),
+          K(obj_meta.file_length_));
+      }
+    } else {
+      file_length_ += size;
+    }
+  }
+  return ret;
+}
+
+} //common
+} //oceanbase

deps/oblib/src/lib/restore/ob_storage_cos_base.h

@@ -0,0 +1,154 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_AGENTSERVER_OSS_STORAGE_COS_BASE_H_
+#define OCEANBASE_AGENTSERVER_OSS_STORAGE_COS_BASE_H_
+
+#include "lib/allocator/page_arena.h"
+#include "lib/container/ob_array.h"
+#include "lib/allocator/page_arena.h"
+#include "lib/lock/ob_spin_rwlock.h"
+#include "ob_i_storage.h"
+#include "cos/ob_cos_wrapper_handle.h"
+
+namespace oceanbase
+{
+namespace common
+{
+class ObStorageCosBase;
+struct CosListFilesCbArg;
+
+// Before using cos, you need to initialize cos enviroment.
+// Thread safe guaranteed by user.
+int init_cos_env();
+
+// You need to clean cos resource when not use cos any more.
+// Thread safe guaranteed by user.
+void fin_cos_env();
+
+class ObStorageCosUtil: public ObIStorageUtil
+{
+public:
+  ObStorageCosUtil();
+  virtual ~ObStorageCosUtil();
+  virtual int open(common::ObObjectStorageInfo *storage_info);
+  virtual void close();
+  virtual int is_exist(const common::ObString &uri, bool &is_exist);
+  virtual int get_file_length(const common::ObString &uri, int64_t &file_length);
+  virtual int write_single_file(const common::ObString &uri, const char *buf,
+                                const int64_t size);
+
+  //cos no dir
+  virtual int mkdir(const common::ObString &uri);
+  virtual int del_file(const common::ObString &uri);
+  virtual int list_files(const common::ObString &uri, common::ObBaseDirEntryOperator &op);
+  virtual int del_dir(const common::ObString &uri);
+  virtual int list_directories(const common::ObString &uri, common::ObBaseDirEntryOperator &op);
+  virtual int is_tagging(const common::ObString &uri, bool &is_tagging);
+private:
+  int get_object_meta_(const common::ObString &uri, bool &is_file_exist, int64_t &file_length);
+
+private:
+  bool is_opened_;
+  common::ObObjectStorageInfo *storage_info_;
+};
+
+class ObStorageCosBase
+{
+public:
+  ObStorageCosBase();
+  virtual ~ObStorageCosBase();
+
+  virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info);
+  void reset();
+  const ObCosWrapperHandle &get_handle() { return handle_; }
+
+  // some cos function
+  int get_cos_file_meta(bool &is_file_exist, common::qcloud_cos::CosObjectMeta &obj_meta);
+  int delete_object(const common::ObString &uri);
+  int list_objects(const common::ObString &uri, const common::ObString &dir_name_str,
+      const char *separator, common::CosListFilesCbArg &arg);
+  int list_directories(const common::ObString &uri, const common::ObString &dir_name_str,
+      const char *next_marker_str, const char *delimiter_str, common::CosListFilesCbArg &arg);
+  int is_object_tagging(const common::ObString &uri, bool &is_tagging);
+
+private:
+  int init_handle(const common::ObObjectStorageInfo &storage_info);
+  bool is_valid() const { return handle_.is_valid(); }
+
+protected:
+  bool is_opened_;
+  ObCosWrapperHandle handle_;
+
+private:
+  DISALLOW_COPY_AND_ASSIGN(ObStorageCosBase);
+};
+
+class ObStorageCosWriter : public ObStorageCosBase, public ObIStorageWriter
+{
+public:
+  ObStorageCosWriter();
+  ~ObStorageCosWriter();
+  virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info) override;
+  int write(const char *buf, const int64_t size);
+  int pwrite(const char *buf, const int64_t size, const int64_t offset);
+  int close();
+  int64_t get_length() const { return file_length_;}
+  bool is_opened() const { return is_opened_; }
+private:
+  int64_t file_length_;
+  DISALLOW_COPY_AND_ASSIGN(ObStorageCosWriter);
+};
+
+class ObStorageCosReader: public ObStorageCosBase, public ObIStorageReader
+{
+public:
+  ObStorageCosReader();
+  virtual ~ObStorageCosReader();
+  virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info) override;
+  int pread(char *buf, const int64_t buf_size, int64_t offset, int64_t &read_size);
+  int close();
+  int64_t get_length() const { return file_length_; }
+  bool is_opened() const { return is_opened_; }
+
+private:
+  int64_t file_length_;
+
+  DISALLOW_COPY_AND_ASSIGN(ObStorageCosReader);
+};
+
+class ObStorageCosAppendWriter : public ObStorageCosBase, public ObIStorageWriter
+{
+public:
+  ObStorageCosAppendWriter();
+  virtual ~ObStorageCosAppendWriter();
+
+public:
+  virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info) override;
+  int write(const char *buf, const int64_t size);
+  int pwrite(const char *buf, const int64_t size, const int64_t offset);
+  int close();
+  int64_t get_length() const { return file_length_; }
+  bool is_opened() const { return is_opened_; }
+
+private:
+  int do_write(const char *buf, const int64_t size, const int64_t offset, const bool is_pwrite);
+
+private:
+  int64_t file_length_;
+
+  DISALLOW_COPY_AND_ASSIGN(ObStorageCosAppendWriter);
+};
+
+} //common
+} //oceanbase
+#endif

deps/oblib/src/lib/restore/ob_storage_file.cpp

@@ -426,16 +426,6 @@ int ObStorageFileUtil::del_dir(const common::ObString &uri)
   return ret;
 }
 
-int ObStorageFileUtil::check_backup_dest_lifecycle(
-    const common::ObString &path,
-    bool &is_set_lifecycle)
-{
-  int ret = OB_SUCCESS;
-  UNUSED(path);
-  is_set_lifecycle = false;
-  return ret;
-}
-
 int ObStorageFileUtil::list_directories(
     const ObString &uri,
     ObBaseDirEntryOperator &op)
@@ -547,13 +537,13 @@ ObStorageFileReader::~ObStorageFileReader()
   }
 }
 
-int ObStorageFileReader::open(const common::ObString &uri, void* device_handle)
+int ObStorageFileReader::open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info)
 {
   int ret = OB_SUCCESS;
   int tmp_ret = OB_SUCCESS;
   char errno_buf[OB_MAX_ERROR_MSG_LEN] = "";
   struct stat64 file_info;
-  UNUSED(device_handle);
+  UNUSED(storage_info);
 
   if (is_opened_) {
     ret = OB_INIT_TWICE;
@@ -813,9 +803,9 @@ ObStorageFileWriter::~ObStorageFileWriter()
   }
 }
 
-int ObStorageFileWriter::open(const common::ObString &uri, void* device_handle)
+int ObStorageFileWriter::open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info)
 {
-  UNUSED(device_handle);
+  UNUSED(storage_info);
   int ret = OB_SUCCESS;
   const char *TMP_NAME_FORMAT = "%s.tmp.%ld";
   ObStorageFileUtil util;
@@ -901,9 +891,9 @@ ObStorageFileAppender::~ObStorageFileAppender()
   }
 }
 
-int ObStorageFileAppender::open(const common::ObString &uri, void* device_hanlde)
+int ObStorageFileAppender::open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info)
 {
-  UNUSED(device_hanlde);
+  UNUSED(storage_info);
   int ret = OB_SUCCESS;
   int tmp_ret = OB_SUCCESS;
   ObStorageFileUtil util;

deps/oblib/src/lib/restore/ob_storage_file.h

@@ -24,15 +24,13 @@ class ObStorageFileUtil: public ObIStorageUtil
 public:
   ObStorageFileUtil();
   virtual ~ObStorageFileUtil();
-  virtual int open(void* base_info)
+  virtual int open(common::ObObjectStorageInfo *storage_info)
   {
-    UNUSED(base_info);
+    UNUSED(storage_info);
     return OB_SUCCESS;
   }
 
-  virtual void close()
-  {
-  }
+  virtual void close() {}
 
   virtual int is_exist(const common::ObString &uri, bool &exist);
   virtual int get_file_length(const common::ObString &uri, int64_t &file_length);
@@ -41,14 +39,10 @@ public:
   virtual int mkdir(const common::ObString &uri);
   virtual int list_files(const common::ObString &uri, common::ObBaseDirEntryOperator &op);
   virtual int del_dir(const common::ObString &uri);
-  virtual int check_backup_dest_lifecycle(const common::ObString &dir_path, bool &is_set_lifecycle);
   virtual int list_directories(const common::ObString &uri, common::ObBaseDirEntryOperator &op);
   virtual int is_tagging(const common::ObString &uri, bool &is_tagging);
 private:
-  int get_tmp_file_format_timestamp(
-      const char *file_name,
-      bool &is_tmp_file,
-      int64_t &timestamp);
+  int get_tmp_file_format_timestamp(const char *file_name, bool &is_tmp_file, int64_t &timestamp);
 
 private:
   DISALLOW_COPY_AND_ASSIGN(ObStorageFileUtil);
@@ -59,7 +53,7 @@ class ObStorageFileReader: public ObIStorageReader
 public:
   ObStorageFileReader();
   virtual ~ObStorageFileReader();
-  virtual int open(const common::ObString &uri, void* device_handle = NULL);
+  virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info = NULL);
   virtual int pread(char *buf,const int64_t buf_size, int64_t offset, int64_t &read_size);
   virtual int close();
   virtual int64_t get_length() const { return file_length_; }
@@ -78,7 +72,7 @@ class ObStorageFileBaseWriter: public ObIStorageWriter
 public:
   ObStorageFileBaseWriter();
   virtual ~ObStorageFileBaseWriter();
-  virtual int open(const common::ObString &uri, void* device_handle = NULL) = 0;
+  virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info = NULL) = 0;
   virtual int open(const int flags);
   virtual int write(const char *buf,const int64_t size);
   virtual int pwrite(const char *buf, const int64_t size, const int64_t offset);
@@ -101,7 +95,7 @@ class ObStorageFileWriter: public ObStorageFileBaseWriter
 public:
   ObStorageFileWriter();
   virtual ~ObStorageFileWriter();
-  virtual int open(const common::ObString &uri, void* device_handle = NULL);
+  virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info = NULL);
   virtual int close() override;
 private:
   char real_path_[OB_MAX_URI_LENGTH];
@@ -116,7 +110,7 @@ public:
   ObStorageFileAppender();
   ObStorageFileAppender(StorageOpenMode mode);
   virtual ~ObStorageFileAppender();
-  virtual int open(const common::ObString &uri, void* device_handle = NULL);
+  virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info = NULL);
   virtual int close() override;
   void set_open_mode(StorageOpenMode mode) {open_mode_ = mode;}
 private:

deps/oblib/src/lib/restore/ob_storage_info.cpp

@@ -0,0 +1,297 @@
+/*
+ *  Copyright (c) 2022 OceanBase
+ *  OceanBase is licensed under Mulan PubL v2.
+ *  You can use this software according to the terms and conditions of the Mulan PubL v2.
+ *  You may obtain a copy of Mulan PubL v2 at:
+ *           http://license.coscl.org.cn/MulanPubL-2.0
+ *  THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ *  EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ *  MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ *  See the Mulan PubL v2 for more details.
+ *  Authors:
+ *
+ */
+#define USING_LOG_PREFIX STORAGE
+
+#include "ob_storage_info.h"
+#include "ob_storage.h"
+
+namespace oceanbase
+{
+
+namespace common
+{
+//***********************ObObjectStorageInfo***************************
+ObObjectStorageInfo::ObObjectStorageInfo()
+  : device_type_(ObStorageType::OB_STORAGE_MAX_TYPE)
+{
+  endpoint_[0] = '\0';
+  access_id_[0] = '\0';
+  access_key_[0] = '\0';
+  extension_[0] = '\0';
+}
+
+ObObjectStorageInfo::~ObObjectStorageInfo()
+{
+  reset();
+}
+
+void ObObjectStorageInfo::reset()
+{
+  device_type_ = ObStorageType::OB_STORAGE_MAX_TYPE;
+  endpoint_[0] = '\0';
+  access_id_[0] = '\0';
+  access_key_[0] = '\0';
+  extension_[0] = '\0';
+}
+
+bool ObObjectStorageInfo::is_valid() const
+{
+  return device_type_ >= 0 && device_type_ < ObStorageType::OB_STORAGE_MAX_TYPE;
+}
+
+int64_t ObObjectStorageInfo::hash() const
+{
+  int64_t hash_value = 0;
+  hash_value = murmurhash(&device_type_, static_cast<int32_t>(sizeof(device_type_)), hash_value);
+  hash_value = murmurhash(endpoint_, static_cast<int32_t>(strlen(endpoint_)), hash_value);
+  hash_value = murmurhash(access_id_, static_cast<int32_t>(strlen(access_id_)), hash_value);
+  hash_value = murmurhash(access_key_, static_cast<int32_t>(strlen(access_key_)), hash_value);
+  hash_value = murmurhash(extension_, static_cast<int32_t>(strlen(extension_)), hash_value);
+  return hash_value;
+}
+
+bool ObObjectStorageInfo::operator ==(const ObObjectStorageInfo &storage_info) const
+{
+  return device_type_ == storage_info.device_type_
+      && (0 == STRCMP(endpoint_, storage_info.endpoint_))
+      && (0 == STRCMP(access_id_, storage_info.access_id_))
+      && (0 == STRCMP(access_key_, storage_info.access_key_))
+      && (0 == STRCMP(extension_, storage_info.extension_));
+}
+
+bool ObObjectStorageInfo::operator !=(const ObObjectStorageInfo &storage_info) const
+{
+  return !(*this == storage_info);
+}
+
+const char *ObObjectStorageInfo::get_type_str() const
+{
+  return get_storage_type_str(device_type_);
+}
+
+ObStorageType ObObjectStorageInfo::get_type() const
+{
+  return device_type_;
+}
+
+// oss:host=xxxx&access_id=xxx&access_key=xxx
+// cos:host=xxxx&access_id=xxx&access_key=xxxappid=xxx
+int ObObjectStorageInfo::set(const common::ObStorageType device_type, const char *storage_info)
+{
+  bool has_appid = false;
+  int ret = OB_SUCCESS;
+  if (is_valid()) {
+    ret = OB_INIT_TWICE;
+    LOG_WARN("storage info init twice", K(ret));
+  } else if (OB_ISNULL(storage_info) || strlen(storage_info) >= OB_MAX_BACKUP_STORAGE_INFO_LENGTH) {
+    ret = OB_INVALID_BACKUP_DEST;
+    LOG_WARN("storage info is invalid", K(ret), K(storage_info));
+  } else if (FALSE_IT(device_type_ = device_type)) {
+  } else if (0 == strlen(storage_info)) {
+    if (OB_STORAGE_FILE != device_type_) {
+      ret = OB_INVALID_BACKUP_DEST;
+      LOG_WARN("storage info is empty", K(ret), K_(device_type));
+    }
+  } else if (OB_FAIL(parse_storage_info_(storage_info, has_appid))) {
+    LOG_WARN("parse storage info failed", K(ret));
+  } else if (OB_STORAGE_FILE != device_type
+      && (0 == strlen(endpoint_) || 0 == strlen(access_id_) || 0 == strlen(access_key_))) {
+    ret = OB_INVALID_BACKUP_DEST;
+    LOG_WARN("backup device is not nfs, endpoint/access_id/access_key do not allow to be empty",
+        K(ret), K_(device_type), K_(endpoint), K_(access_id));
+  } else if (OB_STORAGE_COS == device_type && !has_appid) {
+    ret = OB_INVALID_BACKUP_DEST;
+    LOG_WARN("invalid cos info, appid do not allow to be empty", K(ret), K_(extension));
+  } else if (OB_STORAGE_FILE == device_type
+      && (0 != strlen(endpoint_) || 0 != strlen(access_id_) || 0 != strlen(access_key_))) {
+    ret = OB_INVALID_BACKUP_DEST;
+    LOG_WARN("backup device is nfs, endpoint/access_id/access_key must be empty",
+        K(ret), K_(device_type), K_(endpoint), K_(access_id));
+  } else {
+  }
+
+
+  if (OB_SUCC(ret)) {
+    LOG_INFO("succ to parse storage info",
+        K_(device_type), K_(endpoint), K_(access_id), K_(extension));
+  } else {
+    reset();
+  }
+  return ret;
+}
+
+int ObObjectStorageInfo::set(const char *uri, const char *storage_info)
+{
+  int ret = OB_SUCCESS;
+  common::ObStorageType device_type;
+  if (OB_FAIL(get_storage_type_from_path(uri, device_type))) {
+    LOG_WARN("failed to get storage type from path", K(ret), KPC(this));
+  } else if (OB_FAIL(set(device_type, storage_info))) {
+    LOG_WARN("failed to set storage info", K(ret), KPC(this));
+  }
+  return ret;
+}
+
+int ObObjectStorageInfo::parse_storage_info_(const char *storage_info, bool &has_appid)
+{
+  int ret = OB_SUCCESS;
+
+  if (OB_ISNULL(storage_info) || strlen(storage_info) >= OB_MAX_BACKUP_STORAGE_INFO_LENGTH) {
+    ret = OB_INVALID_BACKUP_DEST;
+    LOG_WARN("storage info is invalid", K(ret), K(storage_info), K(strlen(storage_info)));
+  } else {
+    char tmp[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 };
+    char *token = NULL;
+    char *saved_ptr = NULL;
+    int64_t info_len = strlen(storage_info);
+
+    MEMCPY(tmp, storage_info, info_len);
+    tmp[info_len] = '\0';
+    token = tmp;
+    for (char *str = token; OB_SUCC(ret); str = NULL) {
+      token = ::strtok_r(str, "&", &saved_ptr);
+      if (NULL == token) {
+        break;
+      } else if (0 == strncmp(HOST, token, strlen(HOST))) {
+        if (OB_FAIL(set_storage_info_field_(token, endpoint_, sizeof(endpoint_)))) {
+          LOG_WARN("failed to set endpoint",K(ret), K(token));
+        }
+      } else if (0 == strncmp(ACCESS_ID, token, strlen(ACCESS_ID))) {
+        if (OB_FAIL(set_storage_info_field_(token, access_id_, sizeof(access_id_)))) {
+          LOG_WARN("failed to set access id", K(ret), K(token));
+        }
+      } else if (0 == strncmp(ACCESS_KEY, token, strlen(ACCESS_KEY))) {
+        if (OB_FAIL(set_storage_info_field_(token, access_key_, sizeof(access_key_)))) {
+          LOG_WARN("failed to set access key", K(ret), K(token));
+        }
+      } else if (OB_STORAGE_FILE != device_type_ && 0 == strncmp(APPID, token, strlen(APPID))) {
+        has_appid = true;
+        if (OB_FAIL(set_storage_info_field_(token, extension_, sizeof(extension_)))) {
+          LOG_WARN("failed to set appid", K(ret), K(token));
+        }
+      } else if (0 == strncmp(DELETE_MODE, token, strlen(DELETE_MODE))) {
+        if (OB_STORAGE_FILE == device_type_) {
+          ret = OB_INVALID_BACKUP_DEST;
+          OB_LOG(WARN, "OB_STORAGE_FILE don't support delete mode yet",
+              K(ret), K_(device_type), K(token));
+        } else if (OB_FAIL(check_delete_mode_(token + strlen(DELETE_MODE)))) {
+          OB_LOG(WARN, "failed to check delete mode", K(ret), K(token));
+        } else if (OB_FAIL(set_storage_info_field_(token, extension_, sizeof(extension_)))) {
+          LOG_WARN("failed to set delete mode", K(ret), K(token));
+        }
+      } else {
+      }
+    }
+  }
+  return ret;
+}
+
+int ObObjectStorageInfo::check_delete_mode_(const char *delete_mode) const
+{
+  int ret = OB_SUCCESS;
+  if (OB_ISNULL(delete_mode)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid args", K(ret), KP(delete_mode));
+  } else if (0 != strcmp(delete_mode, "delete") && 0 != strcmp(delete_mode, "tagging")) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "delete mode is invalid", K(ret), K(delete_mode));
+  }
+  return ret;
+}
+
+int ObObjectStorageInfo::set_storage_info_field_(const char *info, char *field, const int64_t length)
+{
+  int ret = OB_SUCCESS;
+  if (OB_ISNULL(info) || OB_ISNULL(field)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid args", K(ret), KP(info), KP(field));
+  } else {
+    const int64_t info_len = strlen(info);
+    int64_t pos = strlen(field);
+    if (info_len >= length) {
+      ret = OB_INVALID_ARGUMENT;
+      LOG_WARN("info is too long ", K(ret), K(info), K(length));
+    } else if (pos > 0 && OB_FAIL(databuff_printf(field, length, pos, "&"))) {
+      // cos:host=xxxx&access_id=xxx&access_key=xxxappid=xxx&delete_mode=xxx
+      // extension_ may contain both appid and delete_mode
+      // so delimiter '&' should be included
+      LOG_WARN("failed to add delimiter to storage info field", K(ret), K(info), K(field), K(length));
+    } else if (OB_FAIL(databuff_printf(field, length, pos, "%s", info))) {
+      LOG_WARN("failed to set storage info field", K(ret), K(info), K(field), K(length));
+    }
+  }
+  return ret;
+}
+
+int ObObjectStorageInfo::assign(const ObObjectStorageInfo &storage_info)
+{
+  int ret = OB_SUCCESS;
+  device_type_ = storage_info.device_type_;
+  MEMCPY(endpoint_, storage_info.endpoint_, sizeof(endpoint_));
+  MEMCPY(access_id_, storage_info.access_id_, sizeof(access_id_));
+  MEMCPY(access_key_, storage_info.access_key_, sizeof(access_key_));
+  MEMCPY(extension_, storage_info.extension_, sizeof(extension_));
+
+  return ret;
+}
+
+int ObObjectStorageInfo::get_storage_info_str(char *storage_info, const int64_t info_len) const
+{
+  int ret = OB_SUCCESS;
+  const int64_t key_len = MAX(OB_MAX_BACKUP_SERIALIZEKEY_LENGTH, OB_MAX_BACKUP_ACCESSKEY_LENGTH);
+  char key[key_len] = { 0 };
+  if (!is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("storage info not init", K(ret));
+  } else if (OB_ISNULL(storage_info) || (info_len <= 0)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid args", K(ret), KP(storage_info), K(info_len));
+  } else if (OB_STORAGE_FILE != device_type_) {
+    if (OB_FAIL(get_access_key_(key, sizeof(key)))) {
+      LOG_WARN("failed to get access key", K(ret));
+    } else if (OB_FAIL(databuff_printf(storage_info, info_len, "%s&%s&%s",
+                                       endpoint_, access_id_, key))) {
+      LOG_WARN("failed to set storage info", K(ret), K(info_len));
+    }
+  }
+
+  if (OB_SUCC(ret) && 0 != strlen(extension_) && info_len > strlen(storage_info)) {
+    // if OB_STORAGE_FILE's extension is not empty
+    // delimiter should be included
+    int64_t str_len = strlen(storage_info);
+    if (str_len > 0 && OB_FAIL(databuff_printf(storage_info, info_len, str_len, "&"))) {
+      LOG_WARN("failed to add delimiter to storage info", K(ret), K(info_len), K(str_len));
+    } else if (OB_FAIL(databuff_printf(storage_info, info_len, str_len, "%s", extension_))) {
+      LOG_WARN("failed to add extension", K(ret), K(info_len), K(str_len), K_(extension));
+    }
+  }
+
+  return ret;
+}
+
+int ObObjectStorageInfo::get_access_key_(char *key_buf, const int64_t key_buf_len) const
+{
+  int ret = OB_SUCCESS;
+  if (OB_ISNULL(key_buf) || key_buf_len <= strlen(access_key_)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid args", K(ret), KP(key_buf), K(key_buf_len));
+  } else {
+    MEMCPY(key_buf, access_key_, strlen(access_key_));
+    key_buf[strlen(access_key_)] = '\0';
+  }
+  return ret;
+}
+
+}
+}

deps/oblib/src/lib/restore/ob_storage_info.h

@@ -0,0 +1,80 @@
+/*
+ *  Copyright (c) 2022 OceanBase
+ *  OceanBase is licensed under Mulan PubL v2.
+ *  You can use this software according to the terms and conditions of the Mulan PubL v2.
+ *  You may obtain a copy of Mulan PubL v2 at:
+ *           http://license.coscl.org.cn/MulanPubL-2.0
+ *  THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ *  EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ *  MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ *  See the Mulan PubL v2 for more details.
+ *  Authors:
+ *
+ */
+
+#ifndef OCEANBASE_LIB_RESTORE_OB_STORAGE_INFO_H_
+#define OCEANBASE_LIB_RESTORE_OB_STORAGE_INFO_H_
+
+#include "common/storage/ob_device_common.h"
+#include "lib/utility/ob_print_utils.h"
+#include "lib/utility/ob_unify_serialize.h"
+
+namespace oceanbase
+{
+
+namespace common
+{
+
+const int64_t OB_MAX_BACKUP_EXTENSION_LENGTH = 512;
+const int64_t OB_MAX_BACKUP_ENDPOINT_LENGTH = 256;
+const int64_t OB_MAX_BACKUP_ACCESSID_LENGTH = 256;
+const int64_t OB_MAX_BACKUP_ACCESSKEY_LENGTH = 256;
+const int64_t OB_MAX_BACKUP_STORAGE_INFO_LENGTH = 1536;
+const int64_t OB_MAX_BACKUP_ENCRYPTKEY_LENGTH = OB_MAX_BACKUP_ACCESSKEY_LENGTH + 32;
+const int64_t OB_MAX_BACKUP_SERIALIZEKEY_LENGTH = OB_MAX_BACKUP_ENCRYPTKEY_LENGTH * 2;
+
+const char *const ACCESS_ID = "access_id=";
+const char *const ACCESS_KEY = "access_key=";
+const char *const HOST = "host=";
+const char *const APPID = "appid=";
+const char *const DELETE_MODE = "delete_mode=";
+
+class ObObjectStorageInfo
+{
+  OB_UNIS_VERSION(1);
+public:
+  ObObjectStorageInfo();
+  virtual ~ObObjectStorageInfo();
+
+  int set(const common::ObStorageType device_type, const char *storage_info);
+  int set(const char *uri, const char *storage_info);
+  virtual int assign(const ObObjectStorageInfo &storage_info);
+  ObStorageType get_type() const;
+  const char *get_type_str() const;
+  int get_storage_info_str(char *storage_info, const int64_t info_len) const;
+
+  bool is_valid() const;
+  void reset();
+  int64_t hash() const;
+  bool operator ==(const ObObjectStorageInfo &storage_info) const;
+  bool operator !=(const ObObjectStorageInfo &storage_info) const;
+  TO_STRING_KV(K_(endpoint), K_(access_id), K_(extension), "type", get_type_str());
+
+protected:
+  virtual int get_access_key_(char *key_buf, const int64_t key_buf_len) const;
+  virtual int parse_storage_info_(const char *storage_info, bool &has_appid);
+  int check_delete_mode_(const char *delete_mode) const;
+  int set_storage_info_field_(const char *info, char *field, const int64_t length);
+
+public:
+  common::ObStorageType device_type_;
+  char endpoint_[OB_MAX_BACKUP_ENDPOINT_LENGTH];
+  char access_id_[OB_MAX_BACKUP_ACCESSID_LENGTH];
+  char access_key_[OB_MAX_BACKUP_ACCESSKEY_LENGTH];
+  char extension_[OB_MAX_BACKUP_EXTENSION_LENGTH];
+};
+
+}
+}
+
+#endif

deps/oblib/src/lib/restore/ob_storage_oss_base.cpp

@@ -351,7 +351,7 @@ ObStorageOssBase::ObStorageOssBase()
   :aos_pool_(NULL),
    oss_option_(NULL),
    is_inited_(false),
-   oss_account_(NULL)
+   oss_account_()
 {
   memset(oss_endpoint_, 0, MAX_OSS_ENDPOINT_LENGTH);
 }
@@ -374,16 +374,21 @@ void ObStorageOssBase::reset()
   }
 }
 
-int ObStorageOssBase::init_with_oss_account(void* account)
+int ObStorageOssBase::init_with_storage_info(common::ObObjectStorageInfo *storage_info)
 {
   int ret = OB_SUCCESS;
-  oss_account_ = static_cast<ObOssAccount*>(account);
+  char info_str[common::OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 };
+
   if(is_inited_) {
     ret = OB_INIT_TWICE;
     OB_LOG(WARN, "oss client init twice", K(ret));
-  } else if (OB_ISNULL(oss_account_)) {
+  } else if (OB_ISNULL(storage_info)) {
     ret = OB_INVALID_ARGUMENT;
     OB_LOG(WARN, "oss account is null, fail to init oss base!", K(ret));
+  } else if (OB_FAIL(storage_info->get_storage_info_str(info_str, sizeof(info_str)))) {
+    OB_LOG(WARN, "fail to get storage info str", K(ret), KPC(storage_info));
+  } else if (OB_FAIL(oss_account_.parse_oss_arg(info_str))) {
+    OB_LOG(WARN, "fail to build oss account", K(ret));
   } else if (OB_FAIL(init_oss_options(aos_pool_, oss_option_))) {
     OB_LOG(WARN, "fail to init oss options", K(aos_pool_), K(oss_option_), K(ret));
   } else if (OB_ISNULL(aos_pool_) || OB_ISNULL(oss_option_)) {
@@ -510,8 +515,8 @@ int ObStorageOssBase::init_oss_options(aos_pool_t *&aos_pool, oss_request_option
     OB_LOG(WARN, "fail to init oss endpoind", K(ret));
   } else {
     aos_str_set(&oss_option->config->endpoint, oss_endpoint_);
-    aos_str_set(&oss_option->config->access_key_id, oss_account_->oss_id_);
-    aos_str_set(&oss_option->config->access_key_secret, oss_account_->oss_key_);
+    aos_str_set(&oss_option->config->access_key_id, oss_account_.oss_id_);
+    aos_str_set(&oss_option->config->access_key_secret, oss_account_.oss_key_);
     oss_option->config->is_cname = 0;
     oss_option->ctl = aos_http_controller_create(oss_option->pool, 0);
 
@@ -549,16 +554,16 @@ int ObStorageOssBase::reinit_oss_option()
 int ObStorageOssBase::init_oss_endpoint()
 {
   int ret = OB_SUCCESS;
-  if (0 == strlen(oss_account_->oss_domain_)) {
+  if (0 == strlen(oss_account_.oss_domain_)) {
     ret = OB_OSS_ERROR;
     OB_LOG(WARN, "oss domain is empty", K(ret));
   } else {
-    int64_t str_len = strlen(oss_account_->oss_domain_);
+    int64_t str_len = strlen(oss_account_.oss_domain_);
     if (str_len >= sizeof(oss_endpoint_)) {
       ret = OB_BUF_NOT_ENOUGH;
-      OB_LOG(WARN, "oss domain is too long", K(ret), KCSTRING(oss_account_->oss_domain_));
+      OB_LOG(WARN, "oss domain is too long", K(ret), KCSTRING(oss_account_.oss_domain_));
     } else {
-      MEMCPY(oss_endpoint_, oss_account_->oss_domain_, str_len);
+      MEMCPY(oss_endpoint_, oss_account_.oss_domain_, str_len);
       oss_endpoint_[str_len] = '\0';
     }
   }
@@ -641,9 +646,9 @@ void ObStorageOssBase::print_oss_info(aos_table_t *resp_headers, aos_status_s *a
     }
     OB_LOG_RET(WARN, OB_SUCCESS, "oss info ", K(aos_ret->code), KCSTRING(aos_ret->error_code),
         KCSTRING(aos_ret->error_msg), KCSTRING(aos_ret->req_id),  KCSTRING(delay_time),
-        KCSTRING(oss_account_->oss_domain_), KCSTRING(oss_endpoint_), KCSTRING(oss_account_->oss_id_));
+        KCSTRING(oss_account_.oss_domain_), KCSTRING(oss_endpoint_), KCSTRING(oss_account_.oss_id_));
   } else {
-    OB_LOG_RET(WARN, OB_SUCCESS, "oss info ", KCSTRING(oss_account_->oss_domain_), KCSTRING(oss_endpoint_), KCSTRING(oss_account_->oss_id_));
+    OB_LOG_RET(WARN, OB_SUCCESS, "oss info ", KCSTRING(oss_account_.oss_domain_), KCSTRING(oss_endpoint_), KCSTRING(oss_account_.oss_id_));
   }
 }
 
@@ -667,7 +672,7 @@ ObStorageOssMultiPartWriter::~ObStorageOssMultiPartWriter()
 {
 }
 
-int ObStorageOssMultiPartWriter::open(const ObString &uri, void* oss_account)
+int ObStorageOssMultiPartWriter::open(const ObString &uri, common::ObObjectStorageInfo *storage_info)
 {
   int ret = OB_SUCCESS;
   ObExternalIOCounterGuard io_guard;
@@ -677,7 +682,7 @@ int ObStorageOssMultiPartWriter::open(const ObString &uri, void* oss_account)
   } else if (is_opened_) {
     ret = OB_OSS_ERROR;
     OB_LOG(WARN, "already open, cannot open again", K(ret));
-  } else if (OB_FAIL(init_with_oss_account(oss_account))) {
+  } else if (OB_FAIL(init_with_storage_info(storage_info))) {
     OB_LOG(WARN, "failed to init oss", K(ret));
   } else if (OB_FAIL(get_bucket_object_name(uri, bucket_, object_, allocator_))) {
     OB_LOG(WARN, "get bucket object error", K(uri), K(ret));
@@ -1031,7 +1036,7 @@ ObStorageOssReader::~ObStorageOssReader()
 {
 }
 
-int ObStorageOssReader::open(const ObString &uri, void* base_info)
+int ObStorageOssReader::open(const ObString &uri, common::ObObjectStorageInfo *storage_info)
 {
   int ret = OB_SUCCESS;
   ObExternalIOCounterGuard io_guard;
@@ -1045,7 +1050,7 @@ int ObStorageOssReader::open(const ObString &uri, void* base_info)
   } else if(is_opened_) {
     ret = OB_OSS_ERROR;
     OB_LOG(WARN, "already open, cannot open again", K(ret));
-  } else if (OB_FAIL(init_with_oss_account(base_info))) {
+  } else if (OB_FAIL(init_with_storage_info(storage_info))) {
     OB_LOG(WARN, "failed to init oss", K(ret));
   } else if (OB_SUCCESS != (ret = get_bucket_object_name(uri, bucket_, object_, allocator_))) {
     OB_LOG(WARN, "bucket name of object name is empty", K(ret));
@@ -1192,7 +1197,7 @@ int ObStorageOssReader::close()
   return ret;
 }
 
-ObStorageOssUtil::ObStorageOssUtil() :is_opened_(false), oss_account_(NULL)
+ObStorageOssUtil::ObStorageOssUtil() :is_opened_(false), storage_info_(NULL)
 {
 }
 
@@ -1200,18 +1205,18 @@ ObStorageOssUtil::~ObStorageOssUtil()
 {
 }
 
-int ObStorageOssUtil::open(void* account)
+int ObStorageOssUtil::open(common::ObObjectStorageInfo *storage_info)
 {
   int ret = OB_SUCCESS;
   if (is_opened_) {
     ret = OB_OSS_ERROR;
     OB_LOG(WARN, "already open, cannot open again", K(ret));
-  } else if (OB_ISNULL(account)) {
+  } else if (OB_ISNULL(storage_info)) {
     ret = OB_INVALID_ARGUMENT;
     OB_LOG(WARN, "oss account is null", K(ret));
   } else {
     is_opened_ = true;
-    oss_account_ = static_cast<ObOssAccount*>(account);
+    storage_info_ = storage_info;
   }
   return ret;
 }
@@ -1222,7 +1227,7 @@ void ObStorageOssUtil::close()
     OB_LOG_RET(WARN, OB_ERR_UNEXPECTED, "oss util cannot close before it is opened");
   } else {
     is_opened_ = false;
-    oss_account_ = NULL;
+    storage_info_ = NULL;
   }
 }
 
@@ -1244,7 +1249,7 @@ int ObStorageOssUtil::is_exist(const ObString &uri, bool &exist)
   } else if (!is_opened_) {
     ret = OB_INVALID_ARGUMENT;
     OB_LOG(WARN, "oss util is not inited", K(ret), K(uri));
-  } else if (OB_FAIL(oss_base.init_with_oss_account(oss_account_))) {
+  } else if (OB_FAIL(oss_base.init_with_storage_info(storage_info_))) {
     OB_LOG(WARN, "fail to init oss base with account", K(ret), K(uri));
   } else if (OB_FAIL(get_bucket_object_name(uri, bucket_ob_string,
       object_ob_string, allocator))) {
@@ -1277,7 +1282,7 @@ int ObStorageOssUtil::get_file_length(const common::ObString &uri, int64_t &file
   } else if (!is_opened_) {
     ret = OB_INVALID_ARGUMENT;
     OB_LOG(WARN, "oss util is not inited", K(ret), K(uri));
-  } else if (OB_FAIL(oss_base.init_with_oss_account(oss_account_))) {
+  } else if (OB_FAIL(oss_base.init_with_storage_info(storage_info_))) {
     OB_LOG(WARN, "fail to init oss base with account", K(ret), K(uri));
   } else if (OB_SUCCESS != (ret = get_bucket_object_name(uri, bucket_ob_string,
       object_ob_string, allocator))) {
@@ -1302,7 +1307,7 @@ int ObStorageOssUtil::write_single_file(
   int ret = OB_SUCCESS;
   ObExternalIOCounterGuard io_guard;
   ObStorageOssWriter writer;
-  if (OB_FAIL(writer.open(uri, oss_account_))) {
+  if (OB_FAIL(writer.open(uri, storage_info_))) {
     OB_LOG(WARN, "fail to open the writer(overwrite)", K(ret));
   } else if (OB_FAIL(writer.write(buf, size))) {
     OB_LOG(WARN, "fail to write the writer(overwrite)", K(ret), K(size), KP(buf));
@@ -1333,7 +1338,7 @@ int ObStorageOssUtil::is_tagging(
   if (uri.empty()) {
     ret = OB_INVALID_ARGUMENT;
     OB_LOG(WARN, "name is empty", K(ret));
-  } else if (OB_FAIL(oss_base.init_with_oss_account(oss_account_))) {
+  } else if (OB_FAIL(oss_base.init_with_storage_info(storage_info_))) {
     OB_LOG(WARN, "fail to init oss base with account", K(ret), K(uri));
   } else if (OB_FAIL(get_bucket_object_name(uri, bucket_str, object_str, allocator))) {
     OB_LOG(WARN, "bucket or object name is empty", K(ret), K(uri), K(bucket_str), K(object_str));
@@ -1446,15 +1451,15 @@ int ObStorageOssUtil::del_file(const common::ObString &uri)
   } else if (!is_opened_) {
     ret = OB_INVALID_ARGUMENT;
     OB_LOG(WARN, "oss util is not inited", K(ret), K(uri));
-  } else if (OB_FAIL(oss_base.init_with_oss_account(oss_account_))) {
+  } else if (OB_FAIL(oss_base.init_with_storage_info(storage_info_))) {
     OB_LOG(WARN, "fail to init oss base with account", K(ret), K(uri));
   } else if (OB_FAIL(get_bucket_object_name(uri, bucket_str, object_str, allocator))) {
     OB_LOG(WARN, "bucket or object name is empty", K(ret), K(uri), K(bucket_str), K(object_str));
-  } else if (ObIStorageUtil::DELETE == oss_base.oss_account_->delete_mode_) {
+  } else if (ObIStorageUtil::DELETE == oss_base.oss_account_.delete_mode_) {
     if (OB_FAIL(delete_object_(uri, oss_base, bucket_str, object_str))) {
       OB_LOG(WARN, "failed to delete object", K(ret), K(uri));
     }
-  } else if (ObIStorageUtil::TAGGING == oss_base.oss_account_->delete_mode_) {
+  } else if (ObIStorageUtil::TAGGING == oss_base.oss_account_.delete_mode_) {
     if (OB_FAIL(tagging_object_(uri, oss_base, bucket_str, object_str))) {
       OB_LOG(WARN, "failed to tagging file", K(ret), K(uri));
     }
@@ -1508,7 +1513,7 @@ int ObStorageOssUtil::list_files(const common::ObString &dir_path,
   } else if (!is_opened_) {
     ret = OB_INVALID_ARGUMENT;
     OB_LOG(WARN, "oss util is not inited", K(ret), K(dir_path));
-  } else if (OB_FAIL(oss_base.init_with_oss_account(oss_account_))) {
+  } else if (OB_FAIL(oss_base.init_with_storage_info(storage_info_))) {
     OB_LOG(WARN, "fail to init oss base with account", K(ret), K(dir_path));
   } else if (OB_FAIL(get_bucket_object_name(dir_path, bucket_str, object_str, tmp_allocator))) {
     OB_LOG(WARN, "bucket or object name is empty", K(ret), K(dir_path), K(bucket_str), K(object_str));
@@ -1634,65 +1639,6 @@ int ObStorageOssUtil::strtotime(const char *date_time, int64_t &time)
   return ret;
 }
 
-int ObStorageOssUtil::check_backup_dest_lifecycle(
-    const common::ObString &path,
-    bool &is_set_lifecycle)
-{
-  int ret = OB_SUCCESS;
-  ObExternalIOCounterGuard io_guard;
-  common::ObArenaAllocator allocator;
-  ObString bucket_str;
-  ObString object_str;
-  aos_status_t *aos_ret = NULL;
-  aos_list_t lifecycle_rule_list;
-  aos_table_t *resp_headers = NULL;
-  oss_lifecycle_rule_content_t *rule_content = NULL;
-  aos_string_t bucket;
-  ObString rule_id;
-  ObString prefix;
-  ObString status;
-  ObString date;
-  ObStorageOssBase oss_base;
-  int days = 0;
-  is_set_lifecycle = false;
-
-  if (path.empty()) {
-    ret = OB_INVALID_ARGUMENT;
-    OB_LOG(WARN, "name is empty", K(ret));
-  } else if (!is_opened_) {
-    ret = OB_INVALID_ARGUMENT;
-    OB_LOG(WARN, "oss util is not inited", K(ret), K(path));
-  } else if (OB_FAIL(oss_base.init_with_oss_account(oss_account_))) {
-    OB_LOG(WARN, "fail to init oss base with account", K(ret), K(path));
-  } else if (OB_SUCCESS != (ret = get_bucket_object_name(path, bucket_str,
-      object_str, allocator))) {
-    OB_LOG(WARN, "bucket or object name is empty",
-        K(bucket_str), K(object_str), K(ret));
-  } else {
-    aos_str_set(&bucket, bucket_str.ptr());
-    aos_list_init(&lifecycle_rule_list);
-    if (OB_ISNULL(aos_ret = oss_get_bucket_lifecycle(oss_base.oss_option_, &bucket, &lifecycle_rule_list, &resp_headers)) || !aos_status_is_ok(aos_ret)) {
-      convert_io_error(aos_ret, ret);
-      OB_LOG(WARN, "fail to list all object", K(ret));
-      oss_base.print_oss_info(resp_headers, aos_ret);
-    } else {
-      aos_list_for_each_entry(oss_lifecycle_rule_content_t, rule_content, &lifecycle_rule_list, node) {
-        prefix = apr_psprintf(oss_base.aos_pool_, "%.*s", rule_content->prefix.len, rule_content->prefix.data);
-        status = apr_psprintf(oss_base.aos_pool_,"%.*s", rule_content->status.len, rule_content->status.data);
-        date = apr_psprintf(oss_base.aos_pool_, "%.*s", rule_content->date.len, rule_content->date.data);
-        days = rule_content->days;
-        if (object_str.prefix_match(prefix)) {
-          if (days > 0 || !date.empty()) {
-            is_set_lifecycle = true;
-            break;
-          }
-        }
-      }
-    }
-  }
-  return ret;
-}
-
 int ObStorageOssUtil::list_directories(
       const common::ObString &dir_path,
       common::ObBaseDirEntryOperator &op)
@@ -1719,7 +1665,7 @@ int ObStorageOssUtil::list_directories(
   } else if (!is_opened_) {
     ret = OB_INVALID_ARGUMENT;
     OB_LOG(WARN, "oss util is not inited", K(ret), K(dir_path));
-  } else if (OB_FAIL(oss_base.init_with_oss_account(oss_account_))) {
+  } else if (OB_FAIL(oss_base.init_with_storage_info(storage_info_))) {
     OB_LOG(WARN, "fail to init oss base with account", K(ret), K(dir_path));
   } else if (OB_FAIL(get_bucket_object_name(dir_path, bucket_str, object_str, tmp_allocator))) {
     OB_LOG(WARN, "bucket or object name is empty", K(ret), K(dir_path), K(bucket_str), K(object_str));
@@ -1845,7 +1791,9 @@ ObStorageOssAppendWriter::~ObStorageOssAppendWriter()
 {
 }
 
-int ObStorageOssAppendWriter::open(const common::ObString &uri, void* oss_account)
+int ObStorageOssAppendWriter::open(
+    const common::ObString &uri,
+    common::ObObjectStorageInfo *storage_info)
 {
   int ret = OB_SUCCESS;
   ObExternalIOCounterGuard io_guard;
@@ -1856,7 +1804,7 @@ int ObStorageOssAppendWriter::open(const common::ObString &uri, void* oss_accoun
   } else if (is_opened_) {
     ret = OB_OSS_ERROR;
     OB_LOG(WARN, "already open, cannot open again", K(ret), K(uri));
-  } else if (OB_FAIL(init_with_oss_account(oss_account))) {
+  } else if (OB_FAIL(init_with_storage_info(storage_info))) {
     OB_LOG(WARN, "failed to init oss", K(ret), K(uri));
   } else if (OB_FAIL(get_bucket_object_name(uri, bucket_, object_, allocator_))) {
     OB_LOG(WARN, "get bucket object error", K(uri), K(ret));
@@ -1971,7 +1919,7 @@ int ObStorageOssAppendWriter::do_write(const char *buf, const int64_t size, cons
           OB_LOG(ERROR, "oss type is null", K(ret));
         } else if (0 != strncmp(OSS_OBJECT_TYPE_APPENDABLE, object_type, strlen(OSS_OBJECT_TYPE_APPENDABLE))) {
           ret = OB_CLOUD_OBJECT_NOT_APPENDABLE;
-          OB_LOG(WARN, "oss object not match", K(ret), KCSTRING(object_type));
+          OB_LOG(WARN, "oss object must be appendable", K(ret), KCSTRING(object_type));
         } else {
           char *next_append_position = (char*)(apr_table_get(resp_headers, OSS_NEXT_APPEND_POSITION));
           position = aos_atoi64(next_append_position);
@@ -2005,6 +1953,18 @@ int ObStorageOssAppendWriter::do_write(const char *buf, const int64_t size, cons
             print_oss_info(resp_headers, aos_ret);
             convert_io_error(aos_ret, ret);
             OB_LOG(WARN, "fail to append", K(content), K(ret));
+
+            // If append failed, print the current object meta, to help debugging.
+            aos_table_t *headers3 = NULL;
+            if(OB_NOT_NULL(headers3 = aos_table_make(aos_pool_, 0))) {
+              if(OB_NOT_NULL(aos_ret = oss_head_object(oss_option_, &bucket, &object, headers3, &resp_headers))) {
+                if ((0 != aos_status_is_ok(aos_ret))) {
+                  char *append_pos_str = (char*)(apr_table_get(resp_headers, OSS_NEXT_APPEND_POSITION));
+                  int64_t cur_pos = aos_atoi64(append_pos_str);
+                  OB_LOG(WARN, "after append fail, we got the object meta", K(cur_pos));
+                }
+              }
+            }
           }
         }
       }
@@ -2044,7 +2004,9 @@ ObStorageOssWriter::~ObStorageOssWriter()
   }
 }
 
-int ObStorageOssWriter::open(const common::ObString &uri, void* oss_base)
+int ObStorageOssWriter::open(
+    const common::ObString &uri,
+    common::ObObjectStorageInfo *storage_info)
 {
   int ret = OB_SUCCESS;
   if (uri.empty()) {
@@ -2053,7 +2015,7 @@ int ObStorageOssWriter::open(const common::ObString &uri, void* oss_base)
   } else if (is_opened_) {
     ret = OB_OSS_ERROR;
     OB_LOG(WARN, "already open, cannot open again", K(ret));
-  } else if (OB_FAIL(init_with_oss_account(oss_base))) {
+  } else if (OB_FAIL(init_with_storage_info(storage_info))) {
     OB_LOG(WARN, "failed to init oss", K(ret), K(uri));
   } else if (OB_FAIL(get_bucket_object_name(uri, bucket_, object_, allocator_))) {
     OB_LOG(WARN, "get bucket object error", K(uri), K(ret));

deps/oblib/src/lib/restore/ob_storage_oss_base.h

@@ -124,14 +124,14 @@ public:
                         bool &is_file_exist, char *&remote_md5, int64_t &file_length);
   void print_oss_info(aos_table_t *resp_headers, aos_status_s *aos_ret);
 
-  int init_with_oss_account(void* account);
+  int init_with_storage_info(common::ObObjectStorageInfo *storage_info);
   int init_oss_endpoint();
 
   aos_pool_t *aos_pool_;
   oss_request_options_t *oss_option_;
   char oss_endpoint_[MAX_OSS_ENDPOINT_LENGTH];
   bool is_inited_;
-  ObOssAccount* oss_account_;
+  ObOssAccount oss_account_;
   
   DISALLOW_COPY_AND_ASSIGN(ObStorageOssBase);
 };
@@ -141,7 +141,7 @@ class ObStorageOssWriter : public ObStorageOssBase, public ObIStorageWriter
 public:
   ObStorageOssWriter();
   ~ObStorageOssWriter();
-  int open(const common::ObString &uri, void* oss_base);
+  int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info);
   int write(const char *buf,const int64_t size);
   int pwrite(const char *buf, const int64_t size, const int64_t offset);
   int close();
@@ -162,7 +162,7 @@ class ObStorageOssMultiPartWriter: public ObStorageOssBase, public ObIStorageWri
 public:
   ObStorageOssMultiPartWriter();
   virtual ~ObStorageOssMultiPartWriter();
-  int open(const common::ObString &uri, void* oss_base);
+  int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info);
   int write(const char *buf,const int64_t size);
   int pwrite(const char *buf, const int64_t size, const int64_t offset);
   int close();
@@ -196,7 +196,7 @@ class ObStorageOssReader: public ObStorageOssBase, public ObIStorageReader
 public:
   ObStorageOssReader();
   virtual ~ObStorageOssReader();
-  int open(const common::ObString &uri, void* oss_base);
+  int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info);
   int pread(char *buf,const int64_t buf_size, int64_t offset, int64_t &read_size);
   int close();
   int64_t get_length() const { return file_length_; }
@@ -217,7 +217,7 @@ class ObStorageOssUtil: public ObIStorageUtil
 public:
   ObStorageOssUtil();
   virtual ~ObStorageOssUtil();
-  virtual int open(void* account);
+  virtual int open(common::ObObjectStorageInfo *storage_info);
   virtual void close();
   virtual int is_exist(const common::ObString &uri, bool &exist);
   virtual int get_file_length(const common::ObString &uri, int64_t &file_length);
@@ -229,8 +229,6 @@ public:
   virtual int del_file(const common::ObString &uri);
   virtual int list_files(const common::ObString &dir_path, common::ObBaseDirEntryOperator &op);
   virtual int del_dir(const common::ObString &uri);
-  virtual int check_backup_dest_lifecycle(const common::ObString &path, 
-                                          bool &is_set_lifecycle);
   virtual int list_directories(const common::ObString &uri, common::ObBaseDirEntryOperator &op);
   virtual int is_tagging(const common::ObString &uri, bool &is_tagging);
 private:
@@ -246,7 +244,7 @@ private:
       const common::ObString &bucket_str,
       const common::ObString &object_str);
   bool is_opened_;
-  ObOssAccount* oss_account_;
+  common::ObObjectStorageInfo *storage_info_;
 };
 
 class ObStorageOssAppendWriter : public ObStorageOssBase, public ObIStorageWriter
@@ -256,7 +254,7 @@ public:
   virtual ~ObStorageOssAppendWriter();
 
 public:
-  int open(const common::ObString &uri, void* oss_base);
+  int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info);
   int write(const char *buf, const int64_t size);
   int pwrite(const char *buf, const int64_t size, const int64_t offset);
   int close();

deps/oblib/src/lib/statistic_event/ob_stat_event.h

@@ -293,16 +293,23 @@ STAT_EVENT_ADD_DEF(COS_IO_READ_DELAY, "cos io read delay", ObStatClassIds::STORA
 STAT_EVENT_ADD_DEF(COS_IO_WRITE_DELAY, "cos io write delay", ObStatClassIds::STORAGE, "cos io write delay", 69015, true, true)
 STAT_EVENT_ADD_DEF(COS_IO_LS_DELAY, "cos io list delay", ObStatClassIds::STORAGE, "cos io list delay", 69016, true, true)
 STAT_EVENT_ADD_DEF(BACKUP_DELETE_DELAY, "backup delete delay", ObStatClassIds::STORAGE, "backup delete delay", 69017, true, true)
-
 STAT_EVENT_ADD_DEF(BACKUP_IO_LS_COUNT, "backup io list count", ObStatClassIds::STORAGE, "backup io list count", 69019, true, true)
 STAT_EVENT_ADD_DEF(BACKUP_IO_READ_FAIL_COUNT, "backup io read failed count", ObStatClassIds::STORAGE, "backup io read failed count", 69020, true, true)
 STAT_EVENT_ADD_DEF(BACKUP_IO_WRITE_FAIL_COUNT, "backup io write failed count", ObStatClassIds::STORAGE, "backup io write failed count", 69021, true, true)
 STAT_EVENT_ADD_DEF(BACKUP_IO_DEL_FAIL_COUNT, "backup io delete failed count", ObStatClassIds::STORAGE, "backup io delete failed count", 69022, true, true)
 STAT_EVENT_ADD_DEF(BACKUP_IO_LS_FAIL_COUNT, "backup io list failed count", ObStatClassIds::STORAGE, "backup io list failed count", 69023, true, true)
-
 STAT_EVENT_ADD_DEF(BACKUP_TAGGING_COUNT, "backup io tagging count", ObStatClassIds::STORAGE, "backup io tagging count", 69024, true, true)
 STAT_EVENT_ADD_DEF(BACKUP_TAGGING_FAIL_COUNT, "backup io tagging failed count", ObStatClassIds::STORAGE, "backup io tagging count", 69025, true, true)
 
+STAT_EVENT_ADD_DEF(COS_IO_WRITE_BYTES, "cos io write bytes", ObStatClassIds::STORAGE, "cos io write bytes", 69026, true, true)
+STAT_EVENT_ADD_DEF(COS_IO_WRITE_COUNT, "cos io write count", ObStatClassIds::STORAGE, "cos io write count", 69027, true, true)
+STAT_EVENT_ADD_DEF(COS_DELETE_COUNT, "cos delete count", ObStatClassIds::STORAGE, "cos delete count", 69028, true, true)
+STAT_EVENT_ADD_DEF(COS_DELETE_DELAY, "cos delete delay", ObStatClassIds::STORAGE, "cos delete delay", 69029, true, true)
+STAT_EVENT_ADD_DEF(COS_IO_LS_LIMIT_COUNT, "cos list io limit count", ObStatClassIds::STORAGE, "cos list io limit count", 69030, true, true)
+STAT_EVENT_ADD_DEF(COS_IO_LS_COUNT, "cos io list count", ObStatClassIds::STORAGE, "cos io list count", 69031, true, true)
+STAT_EVENT_ADD_DEF(COS_IO_READ_COUNT, "cos io read count", ObStatClassIds::STORAGE, "cos io read count", 69032, true, true)
+STAT_EVENT_ADD_DEF(COS_IO_READ_BYTES, "cos io read bytes", ObStatClassIds::STORAGE, "cos io read bytes", 69033, true, true)
+
 // DEBUG
 STAT_EVENT_ADD_DEF(REFRESH_SCHEMA_COUNT, "refresh schema count", ObStatClassIds::DEBUG, "refresh schema count", 70000, false, true)
 STAT_EVENT_ADD_DEF(REFRESH_SCHEMA_TIME, "refresh schema time", ObStatClassIds::DEBUG, "refresh schema time", 70001, false, true)

deps/oblib/unittest/lib/CMakeLists.txt

@@ -85,6 +85,8 @@ oblib_addtest(rc/test_context.cpp)
 oblib_addtest(resource/test_resource_mgr.cpp)
 #oblib_addtest(restore/test_storage_file.cpp)
 #oblib_addtest(restore/test_storage_oss.cpp)
+oblib_addtest(restore/test_storage_cos.cpp)
+oblib_addtest(restore/test_storage_info.cpp)
 #oblib_addtest(restore/test_storage.cpp)
 oblib_addtest(stat/test_di_cache.cpp)
 oblib_addtest(stat/test_diagnose_info.cpp)

deps/oblib/unittest/lib/restore/test_storage_cos.cpp

@@ -0,0 +1,681 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include <gtest/gtest.h>
+#include "lib/utility/ob_test_util.h"
+#include "lib/restore/ob_storage.h"
+#include "lib/restore/ob_storage_cos_base.h"
+#include "lib/allocator/page_arena.h"
+#include "test_storage_cos.h"
+
+using namespace oceanbase::common;
+
+class TestStorageCosCommon {
+public:
+TestStorageCosCommon() {}
+~TestStorageCosCommon() {}
+
+void init()
+{
+  init_cos_env();
+  ASSERT_EQ(OB_SUCCESS,
+            databuff_printf(account, sizeof(account),
+            "host=%s&access_id=%s&access_key=%s&appid=%s",
+            endpoint, secretid, secretkey, appid));
+  //build cos_base
+  const ObString cos_storage_info(account);
+  ASSERT_EQ(OB_SUCCESS, cos_base.set(ObStorageType::OB_STORAGE_COS, cos_storage_info.ptr()));
+}
+void destory()
+{
+  fin_cos_env();
+}
+protected:
+  char account[OB_MAX_URI_LENGTH];
+  const char *dir_name = "cos_unittest_dir";
+  char uri[OB_MAX_URI_LENGTH];
+  char dir_uri[OB_MAX_URI_LENGTH];
+  ObObjectStorageInfo cos_base;
+
+  int object_prefix_len = 5;
+};
+
+//use to clean the env
+class TestCosCleanOp : public ObBaseDirEntryOperator, public TestStorageCosCommon
+{
+public:
+  TestCosCleanOp()
+  {
+  }
+  ~TestCosCleanOp() {}
+  int func(const dirent *entry) override;
+private :
+};
+
+int TestCosCleanOp::func(const dirent *entry)
+{
+  int ret = OB_SUCCESS;
+  ObStorageCosUtil util;
+  if (OB_FAIL(util.open(&cos_base))) {
+  } else if (OB_FAIL(databuff_printf(uri, sizeof(uri), "%s/%s/%s", bucket, dir_name, entry->d_name))) {
+  } else if (OB_FAIL(util.del_file(uri))) {
+  }
+  util.close();
+
+  return ret;
+}
+
+class TestStorageCos: public ::testing::Test, public TestStorageCosCommon
+{
+public:
+  TestStorageCos() : enable_test_(enable_test) {}
+  virtual ~TestStorageCos(){}
+  virtual void SetUp()
+  {
+    init();
+  }
+  virtual void TearDown()
+  {
+    destory();
+  }
+
+  static void SetUpTestCase()
+  {
+  }
+
+  static void TearDownTestCase()
+  {
+  }
+
+private:
+  // disallow copy
+  DISALLOW_COPY_AND_ASSIGN(TestStorageCos);
+protected:
+  bool enable_test_;
+};
+
+TEST_F(TestStorageCos, test_append)
+{
+  int ret = OB_SUCCESS;
+  if (enable_test_) {
+    ObStorageAppender appender;
+    ObStorageReader reader;
+    ObStorageUtil util;
+    ASSERT_EQ(OB_SUCCESS, util.open(&cos_base));
+    const char *tmp_append_dir = "test_append";
+    const int64_t ts = ObTimeUtility::current_time();
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld",
+        bucket, dir_name, tmp_append_dir, ts));
+
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/test_append_file", dir_uri));
+
+    // operate before open
+    char tmp_buf[10] = {0};
+    ASSERT_FALSE(appender.is_opened());
+    ASSERT_EQ(OB_NOT_INIT, appender.write(tmp_buf, sizeof(tmp_buf)));
+    ASSERT_EQ(OB_NOT_INIT, appender.pwrite(tmp_buf, sizeof(tmp_buf), 0));
+    ASSERT_EQ(OB_NOT_INIT, appender.pwrite(tmp_buf, sizeof(tmp_buf), 0));
+    ASSERT_EQ(OB_NOT_INIT, appender.close());
+
+    // wrong uri
+    uri[0] = '\0';
+    ASSERT_EQ(OB_INVALID_ARGUMENT, appender.open(uri, &cos_base));
+    ASSERT_EQ(OB_NOT_INIT, appender.write(tmp_buf, sizeof(tmp_buf)));
+    ASSERT_EQ(OB_NOT_INIT, appender.pwrite(tmp_buf, sizeof(tmp_buf), 0));
+    ASSERT_EQ(OB_NOT_INIT, appender.close());
+
+    uri[0] = 'a';
+    uri[1] = '\0';
+    ASSERT_EQ(OB_INVALID_BACKUP_DEST, appender.open(uri, &cos_base));
+    ASSERT_EQ(OB_NOT_INIT, appender.write(tmp_buf, sizeof(tmp_buf)));
+    ASSERT_EQ(OB_NOT_INIT, appender.pwrite(tmp_buf, sizeof(tmp_buf), 0));
+    ASSERT_EQ(OB_NOT_INIT, appender.close());
+
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/test_append_file_////", dir_uri));
+    ASSERT_EQ(OB_INVALID_ARGUMENT, appender.open(uri, &cos_base));
+    ASSERT_EQ(OB_NOT_INIT, appender.write(tmp_buf, sizeof(tmp_buf)));
+    ASSERT_EQ(OB_NOT_INIT, appender.pwrite(tmp_buf, sizeof(tmp_buf), 0));
+    ASSERT_EQ(OB_NOT_INIT, appender.close());
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/test_append_file", dir_uri));
+    ASSERT_EQ(OB_SUCCESS, appender.open(uri, &cos_base));
+    ASSERT_EQ(OB_SUCCESS, appender.close());
+
+    // invalid storage info
+    ASSERT_EQ(OB_INVALID_ARGUMENT, appender.open(uri, NULL));
+    ASSERT_EQ(OB_NOT_INIT, appender.write(tmp_buf, sizeof(tmp_buf)));
+    ASSERT_EQ(OB_NOT_INIT, appender.pwrite(tmp_buf, sizeof(tmp_buf), 0));
+    ASSERT_EQ(OB_NOT_INIT, appender.close());
+
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/test_append_file", dir_uri));
+    ASSERT_EQ(OB_SUCCESS, appender.open(uri, &cos_base));
+    ASSERT_TRUE(appender.is_opened());
+    ASSERT_EQ(0LL, appender.get_length());
+
+    // first append
+    const char first_write[] = "123";
+    ASSERT_EQ(OB_SUCCESS, appender.pwrite(first_write, strlen(first_write), 0));
+    ASSERT_EQ(strlen(first_write), appender.get_length());
+
+    // second append
+    const char second_write[] = "4567";
+    // repeatable_pwrite returned err code
+    ASSERT_EQ(OB_BACKUP_PWRITE_CONTENT_NOT_MATCH,
+        appender.pwrite(second_write, strlen(second_write), strlen(first_write) - 1));
+    ASSERT_EQ(OB_BACKUP_PWRITE_OFFSET_NOT_MATCH,
+        appender.pwrite(second_write, strlen(second_write) - 1, strlen(first_write) + 1));
+    ASSERT_EQ(OB_SUCCESS, appender.pwrite(second_write, strlen(second_write), strlen(first_write)));
+    ASSERT_EQ(strlen(first_write) + strlen(second_write), appender.get_length());
+
+    // check size
+    int64_t file_length = 0;
+    ASSERT_EQ(OB_SUCCESS, util.get_file_length(uri, file_length));
+    ASSERT_EQ(strlen(first_write) + strlen(second_write), file_length);
+
+    // check data and clean
+    ASSERT_EQ(OB_SUCCESS, reader.open(uri, &cos_base));
+    ASSERT_EQ(strlen(first_write) + strlen(second_write), reader.get_length());
+    char read_buf[5] = {0};
+    int64_t read_size = 0;
+    ASSERT_EQ(OB_SUCCESS, reader.pread(read_buf, 5, 2, read_size));
+    ASSERT_EQ('3', read_buf[0]);
+    ASSERT_EQ('7', read_buf[4]);
+    ASSERT_EQ(5, read_size);
+    ASSERT_EQ(OB_SUCCESS, reader.close());
+    ASSERT_EQ(OB_SUCCESS, util.del_file(uri));
+    ASSERT_EQ(OB_SUCCESS, appender.close());
+    ASSERT_FALSE(appender.is_opened());
+
+    // append to not appendable object
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/test_append_to_not_appendable_file", dir_uri));
+    ASSERT_EQ(OB_SUCCESS, util.write_single_file(uri, first_write, strlen(first_write)));
+    ASSERT_EQ(OB_SUCCESS, appender.open(uri, &cos_base));
+    ASSERT_EQ(OB_CLOUD_OBJECT_NOT_APPENDABLE, appender.pwrite(second_write, strlen(second_write), strlen(first_write)));
+    ASSERT_EQ(OB_SUCCESS, appender.close());
+    ASSERT_FALSE(appender.is_opened());
+    ASSERT_EQ(OB_SUCCESS, util.del_file(uri));
+
+    // operate after close
+    ASSERT_EQ(OB_NOT_INIT, appender.write(tmp_buf, sizeof(tmp_buf)));
+    ASSERT_EQ(OB_NOT_INIT, appender.pwrite(tmp_buf, sizeof(tmp_buf), 0));
+    ASSERT_EQ(OB_NOT_INIT, appender.close());
+
+    util.close();
+  }
+}
+
+#define WRITE_SINGLE_FILE(file_suffix, file_content) \
+  ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%stest_write_file_%ld", dir_uri, file_suffix)); \
+  ASSERT_EQ(OB_SUCCESS, writer.open(uri, &cos_base)); \
+  const char write_content[] = file_content; \
+  ASSERT_EQ(OB_SUCCESS, writer.write(write_content, strlen(write_content))); \
+  ASSERT_EQ(OB_SUCCESS, writer.close()); \
+
+TEST_F(TestStorageCos, test_basic_rw)
+{
+  int ret = OB_SUCCESS;
+  if (enable_test_) {
+    ObStorageWriter writer;
+    ObStorageReader reader;
+    ObStorageUtil util;
+    ASSERT_EQ(OB_SUCCESS, util.open(&cos_base));
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/", bucket, dir_name));
+
+    // operate before open
+    char tmp_buf[10] = {0};
+    int64_t tmp_read_size = 0;
+    ASSERT_EQ(OB_NOT_INIT, writer.write(tmp_buf, sizeof(tmp_buf)));
+    ASSERT_EQ(OB_NOT_INIT, writer.close());
+    ASSERT_EQ(OB_NOT_INIT, reader.pread(tmp_buf, sizeof(tmp_buf), 0, tmp_read_size));
+    ASSERT_EQ(OB_NOT_INIT, reader.close());
+
+    // wrong uri
+    uri[0] = '\0';
+    ASSERT_EQ(OB_INVALID_BACKUP_DEST, writer.open(uri, NULL));
+    ASSERT_EQ(OB_NOT_INIT, writer.write(tmp_buf, sizeof(tmp_buf)));
+    ASSERT_EQ(OB_NOT_INIT, writer.close());
+    ASSERT_EQ(OB_INVALID_BACKUP_DEST, reader.open(uri, NULL));
+    ASSERT_EQ(OB_NOT_INIT, reader.pread(tmp_buf, sizeof(tmp_buf), 0, tmp_read_size));
+    ASSERT_EQ(OB_NOT_INIT, reader.close());
+
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%stest_write_file_////", dir_uri));
+    ASSERT_EQ(OB_INVALID_ARGUMENT, writer.open(uri, &cos_base));
+    ASSERT_EQ(OB_NOT_INIT, writer.write(tmp_buf, sizeof(tmp_buf)));
+    ASSERT_EQ(OB_NOT_INIT, writer.close());
+    ASSERT_EQ(OB_INVALID_ARGUMENT, reader.open(uri, &cos_base));
+    ASSERT_EQ(OB_NOT_INIT, reader.pread(tmp_buf, sizeof(tmp_buf), 0, tmp_read_size));
+    ASSERT_EQ(OB_NOT_INIT, reader.close());
+
+    const int64_t tmp_ts = ObTimeUtility::current_time();
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%stest_write_file_%ld", dir_uri, tmp_ts));
+    ASSERT_EQ(OB_SUCCESS, writer.open(uri, &cos_base));
+    ASSERT_EQ(OB_SUCCESS, writer.close());
+
+    // invalid storage info
+    ASSERT_EQ(OB_INVALID_ARGUMENT, writer.open(uri, NULL));
+    ASSERT_EQ(OB_NOT_INIT, writer.write(tmp_buf, sizeof(tmp_buf)));
+    ASSERT_EQ(OB_NOT_INIT, writer.close());
+    ASSERT_EQ(OB_INVALID_ARGUMENT, reader.open(uri, NULL));
+    ASSERT_EQ(OB_NOT_INIT, reader.pread(tmp_buf, sizeof(tmp_buf), 0, tmp_read_size));
+    ASSERT_EQ(OB_NOT_INIT, reader.close());
+
+    {
+      // rw empty object
+      const int64_t ts = ObTimeUtility::current_time();
+      WRITE_SINGLE_FILE(ts, "");
+
+      ASSERT_EQ(OB_SUCCESS, reader.open(uri, &cos_base));
+      ASSERT_EQ(strlen(write_content), reader.get_length());
+      ASSERT_EQ(OB_SUCCESS, reader.close());
+      ASSERT_EQ(OB_SUCCESS, util.del_file(uri));
+    }
+
+    {
+      // ObStorageWriter writer;
+      const int64_t ts = ObTimeUtility::current_time();
+      WRITE_SINGLE_FILE(ts, "123456789");
+
+      // ObStorageReader reader;
+      ASSERT_EQ(OB_SUCCESS, reader.open(uri, &cos_base));
+      ASSERT_EQ(strlen(write_content), reader.get_length());
+      char read_buf[7] = {0};
+      int64_t read_size = 0;
+      ASSERT_EQ(OB_SUCCESS, reader.pread(read_buf, 7, 2, read_size));
+      ASSERT_EQ('3', read_buf[0]);
+      ASSERT_EQ('9', read_buf[6]);
+      ASSERT_EQ(7, read_size);
+      ASSERT_EQ(OB_SUCCESS, reader.close());
+
+      ASSERT_EQ(OB_SUCCESS, util.del_file(uri));
+    }
+
+    {
+      // ObStorageWriter writer;
+      const int64_t ts = ObTimeUtility::current_time();
+      WRITE_SINGLE_FILE(ts, "123456789ABCDEF");
+
+      // ObStorageReader reader;
+      ASSERT_EQ(OB_SUCCESS, reader.open(uri, &cos_base));
+      ASSERT_EQ(strlen(write_content), reader.get_length());
+      char read_buf[10] = {0};
+      int64_t read_size = 0;
+      ASSERT_EQ(OB_SUCCESS, reader.pread(read_buf, 10, 9, read_size));
+      ASSERT_EQ('A', read_buf[0]);
+      ASSERT_EQ('F', read_buf[5]);
+      ASSERT_EQ(6, read_size);
+      ASSERT_EQ(OB_SUCCESS, reader.close());
+
+      ASSERT_EQ(OB_SUCCESS, util.del_file(uri));
+    }
+
+    {
+      // read not exist object
+      const int64_t ts = ObTimeUtility::current_time();
+      ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%snot_exist_%ld", dir_uri, ts));
+      ASSERT_EQ(OB_BACKUP_FILE_NOT_EXIST, reader.open(uri, &cos_base));
+      ASSERT_EQ(OB_NOT_INIT, reader.close());
+
+      // open after fail
+      WRITE_SINGLE_FILE(ts, "123456789ABCDEF");
+      ASSERT_EQ(OB_SUCCESS, reader.open(uri, &cos_base));
+      ASSERT_EQ(OB_SUCCESS, reader.close());
+    }
+
+    // open twice
+    ASSERT_EQ(OB_SUCCESS, writer.open(uri, &cos_base));
+    ASSERT_EQ(OB_INIT_TWICE, writer.open(uri, &cos_base));
+    ASSERT_EQ(OB_NOT_INIT, writer.close());   // reader/writer will be closed if init twice
+    ASSERT_EQ(OB_SUCCESS, reader.open(uri, &cos_base));
+    ASSERT_EQ(OB_INIT_TWICE, reader.open(uri, &cos_base));
+    ASSERT_EQ(OB_NOT_INIT, reader.close());
+    ASSERT_EQ(OB_SUCCESS, util.del_file(uri));
+
+    // operate after close
+    ASSERT_EQ(OB_NOT_INIT, writer.write(tmp_buf, sizeof(tmp_buf)));
+    ASSERT_EQ(OB_NOT_INIT, writer.close());
+    ASSERT_EQ(OB_NOT_INIT, reader.pread(tmp_buf, sizeof(tmp_buf), 0, tmp_read_size));
+    ASSERT_EQ(OB_NOT_INIT, reader.close());
+
+    util.close();
+  }
+}
+
+TEST_F(TestStorageCos, test_util)
+{
+  int ret = OB_SUCCESS;
+  if (enable_test_) {
+    ObStorageUtil util;
+    const char *tmp_util_dir = "test_util";
+    const int64_t ts = ObTimeUtility::current_time();
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld/",
+      bucket, dir_name, tmp_util_dir, ts));
+    ASSERT_EQ(OB_SUCCESS, util.open(&cos_base));
+
+    ObStorageWriter writer;
+    {
+      WRITE_SINGLE_FILE(1L, "123456789ABC");
+
+      bool is_obj_exist = false;
+      ASSERT_EQ(OB_SUCCESS, util.is_exist(uri, is_obj_exist));
+      ASSERT_TRUE(is_obj_exist);
+
+      ASSERT_EQ(OB_SUCCESS, util.del_file(uri));
+      ASSERT_EQ(OB_SUCCESS, util.is_exist(uri, is_obj_exist));
+      ASSERT_FALSE(is_obj_exist);
+    }
+    {
+      WRITE_SINGLE_FILE(2L, "123456789ABCDEF");
+
+      int64_t file_length = 0;
+      ASSERT_EQ(OB_SUCCESS, util.get_file_length(uri, file_length));
+      ASSERT_EQ(strlen(write_content), file_length);
+      ASSERT_EQ(OB_SUCCESS, util.del_file(uri));
+    }
+
+    // open twice
+    ASSERT_EQ(OB_INIT_TWICE, util.open(&cos_base));
+    util.close();
+
+    // invalid storage info
+    ASSERT_EQ(OB_INVALID_ARGUMENT, util.open(NULL));
+  }
+}
+
+TEST_F(TestStorageCos, test_util_write_single_file)
+{
+  int ret = OB_SUCCESS;
+  if (enable_test_) {
+    ObStorageReader reader;
+    ObStorageUtil util;
+    const char *tmp_util_dir = "test_util_write_single_file";
+    const int64_t ts = ObTimeUtility::current_time();
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld",
+      bucket, dir_name, tmp_util_dir, ts));
+    ASSERT_EQ(OB_SUCCESS, util.open(&cos_base));
+
+    // write data
+    const char *write_content = "123456789ABCDEF";
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/0", dir_uri));
+    ASSERT_EQ(OB_SUCCESS, util.write_single_file(uri, write_content, strlen(write_content)));
+
+    // check data
+    ASSERT_EQ(OB_SUCCESS, reader.open(uri, &cos_base));
+    ASSERT_EQ(strlen(write_content), reader.get_length());
+    char read_buf[10] = {0};
+    int64_t read_size = 0;
+    ASSERT_EQ(OB_SUCCESS, reader.pread(read_buf, 10, 9, read_size));
+    ASSERT_EQ('A', read_buf[0]);
+    ASSERT_EQ('F', read_buf[5]);
+    ASSERT_EQ(6, read_size);
+    ASSERT_EQ(OB_SUCCESS, reader.close());
+
+    ASSERT_EQ(OB_SUCCESS, util.del_file(uri));
+
+    // wrong uri
+    uri[0] = '\0';
+    ASSERT_EQ(OB_INVALID_BACKUP_DEST, util.write_single_file(uri, write_content, strlen(write_content)));
+
+    util.close();
+  }
+}
+
+class TestCosListOp : public ObBaseDirEntryOperator
+{
+public:
+  TestCosListOp() {}
+  virtual ~TestCosListOp() {}
+  int func(const dirent *entry) override;
+
+  ObArray<char *> object_names_;
+private :
+  ObArenaAllocator allocator_;
+};
+
+int TestCosListOp::func(const dirent *entry)
+{
+  int ret = OB_SUCCESS;
+  char *tmp_object_name = (char *)allocator_.alloc(strlen(entry->d_name) + 1);
+  if (OB_ISNULL(tmp_object_name)) {
+    ret = OB_ALLOCATE_MEMORY_FAILED;
+    OB_LOG(WARN,"fail to alloc buf for tmp object name", K(ret));
+  } else {
+    STRCPY(tmp_object_name, entry->d_name);
+    object_names_.push_back(tmp_object_name);
+  }
+  return ret;
+}
+
+#define UTIL_WRITE_FILES(format, object_prefix_len, file_num, file_content) \
+  for (int64_t file_idx = 0; file_idx < file_num; file_idx++) { \
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), format, \
+                                          dir_uri, object_prefix_len, file_idx, file_idx)); \
+    ASSERT_EQ(OB_SUCCESS, util.write_single_file(uri, file_content, strlen(file_content))); \
+  } \
+
+#define UTIL_DELETE_FILES(format, object_prefix_len, file_num) \
+  for (int64_t file_idx = 0; file_idx < file_num; file_idx++) { \
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), format, \
+                                          dir_uri, object_prefix_len, file_idx, file_idx)); \
+    ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); \
+  } \
+
+TEST_F(TestStorageCos, test_util_list_files)
+{
+  int ret = OB_SUCCESS;
+  if (enable_test_) {
+    ObStorageReader reader;
+    ObStorageUtil util;
+    const char *tmp_util_dir = "test_util_list_files";
+    const int64_t ts = ObTimeUtility::current_time();
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld",
+      bucket, dir_name, tmp_util_dir, ts));
+    ASSERT_EQ(OB_SUCCESS, util.open(&cos_base));
+
+    {
+      // wrong uri
+      uri[0] = '\0';
+      TestCosListOp op;
+      ASSERT_EQ(OB_INVALID_BACKUP_DEST, util.list_files(uri, op));
+    }
+
+    int64_t file_num = 1001;
+    const char *write_content = "0123456789";
+
+    // list objects
+    {
+      const char *format = "%s/%0*ld_%ld";
+      UTIL_WRITE_FILES(format, object_prefix_len, file_num, write_content);
+
+      // list and check and clean
+      TestCosListOp op;
+      ASSERT_EQ(OB_SUCCESS, util.list_files(dir_uri, op));
+      ASSERT_EQ(file_num, op.object_names_.size());
+      for (int64_t i = 0; i < file_num; i++) {
+        // listed files do not contain prefix
+        ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%0*ld_%ld", object_prefix_len, i, i));
+        ASSERT_STREQ(uri, op.object_names_[i]);
+      }
+      UTIL_DELETE_FILES(format, object_prefix_len, file_num);
+    }
+
+    // list subfolders' objects
+    {
+      const char *format = "%s/%0*ld/%ld";
+      UTIL_WRITE_FILES(format, object_prefix_len, file_num, write_content);
+
+      // list and check and clean
+      TestCosListOp op;
+      ASSERT_EQ(OB_SUCCESS, util.list_files(dir_uri, op));
+      ASSERT_EQ(file_num, op.object_names_.size());
+      for (int64_t i = 0; i < file_num; i++) {
+        // listed files do not contain prefix
+        ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%0*ld/%ld", object_prefix_len, i, i));
+        ASSERT_STREQ(uri, op.object_names_[i]);
+      }
+      UTIL_DELETE_FILES(format, object_prefix_len, file_num);
+    }
+
+    // list empty dir, now dir_uri should be empty after delete
+    TestCosListOp list_empty_op;
+    ASSERT_EQ(OB_SUCCESS, util.list_files(dir_uri, list_empty_op));
+    ASSERT_EQ(0, list_empty_op.object_names_.size());
+
+    util.close();
+  }
+}
+
+TEST_F(TestStorageCos, test_util_list_directories)
+{
+  int ret = OB_SUCCESS;
+  if (enable_test_) {
+    ObStorageReader reader;
+    ObStorageUtil util;
+    const char *tmp_util_dir = "test_util_list_directories";
+    const int64_t ts = ObTimeUtility::current_time();
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld",
+      bucket, dir_name, tmp_util_dir, ts));
+    ASSERT_EQ(OB_SUCCESS, util.open(&cos_base));
+
+    {
+      // wrong uri
+      uri[0] = '\0';
+      TestCosListOp op;
+      ASSERT_EQ(OB_INVALID_ARGUMENT, util.list_directories(uri, op));
+    }
+
+    int64_t file_num = 1001;
+    const char *write_content = "0123456789";
+
+    {
+      const char *format = "%s/%0*ld/%ld";
+      UTIL_WRITE_FILES(format, object_prefix_len, file_num, write_content);
+
+      // list and check and clean
+      TestCosListOp op;
+      ASSERT_EQ(OB_SUCCESS, util.list_directories(dir_uri, op));
+      ASSERT_EQ(file_num, op.object_names_.size());
+      for (int64_t i = 0; i < file_num; i++) {
+        // listed directories do not contain prefix
+        ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%0*ld", object_prefix_len, i));
+        ASSERT_STREQ(uri, op.object_names_[i]);
+      }
+      UTIL_DELETE_FILES(format, object_prefix_len, file_num);
+    }
+
+    // no sub directories
+    {
+      const char *format = "%s/%0*ld_%ld";
+      UTIL_WRITE_FILES(format, object_prefix_len, file_num, write_content);
+
+      // list and check and clean
+      TestCosListOp op;
+      ASSERT_EQ(OB_SUCCESS, util.list_directories(dir_uri, op));
+      ASSERT_EQ(0, op.object_names_.size());
+      UTIL_DELETE_FILES(format, object_prefix_len, file_num);
+    }
+
+    util.close();
+  }
+}
+
+TEST_F(TestStorageCos, test_util_is_tagging)
+{
+  int ret = OB_SUCCESS;
+  if (enable_test_) {
+    ObStorageUtil util;
+    const char *tmp_util_dir = "test_util_is_tagging";
+    const int64_t ts = ObTimeUtility::current_time();
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld",
+      bucket, dir_name, tmp_util_dir, ts));
+
+    bool is_tagging = true;
+    char tmp_account[OB_MAX_URI_LENGTH];
+    ObObjectStorageInfo tmp_cos_base;
+    const char *write_content = "123456789ABCDEF";
+
+    // wrong tag mode
+    ASSERT_EQ(OB_SUCCESS,
+              databuff_printf(tmp_account, sizeof(tmp_account),
+              "host=%s&access_id=%s&access_key=%s&appid=%s&delete_mode=tag",
+              endpoint, secretid, secretkey, appid));
+    ASSERT_EQ(OB_INVALID_ARGUMENT, tmp_cos_base.set(ObStorageType::OB_STORAGE_COS, tmp_account));
+    tmp_cos_base.reset();
+
+    ASSERT_EQ(OB_SUCCESS,
+              databuff_printf(tmp_account, sizeof(tmp_account),
+              "host=%s&access_id=%s&access_key=%s&appid=%s&delete_mode=delete_delete",
+              endpoint, secretid, secretkey, appid));
+    ASSERT_EQ(OB_INVALID_ARGUMENT, tmp_cos_base.set(ObStorageType::OB_STORAGE_COS, tmp_account));
+    tmp_cos_base.reset();
+
+    // delete mode
+    ASSERT_EQ(OB_SUCCESS,
+              databuff_printf(tmp_account, sizeof(tmp_account),
+              "host=%s&access_id=%s&access_key=%s&appid=%s&delete_mode=delete",
+              endpoint, secretid, secretkey, appid));
+    ASSERT_EQ(OB_SUCCESS, tmp_cos_base.set(ObStorageType::OB_STORAGE_COS, tmp_account));
+
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/delete_mode", dir_uri));
+    ASSERT_EQ(OB_SUCCESS, util.open(&tmp_cos_base));
+    ASSERT_EQ(OB_SUCCESS, util.write_single_file(uri, write_content, strlen(write_content)));
+
+    ASSERT_EQ(OB_SUCCESS, util.is_tagging(uri, is_tagging));
+    ASSERT_FALSE(is_tagging);
+
+    ASSERT_EQ(OB_SUCCESS, util.del_file(uri));
+    ASSERT_EQ(OB_BACKUP_FILE_NOT_EXIST, util.is_tagging(uri, is_tagging));
+    tmp_cos_base.reset();
+    util.close();
+
+    // tagging mode
+    ASSERT_EQ(OB_SUCCESS,
+              databuff_printf(tmp_account, sizeof(tmp_account),
+              "host=%s&access_id=%s&access_key=%s&appid=%s&delete_mode=tagging",
+              endpoint, secretid, secretkey, appid));
+    ASSERT_EQ(OB_SUCCESS, tmp_cos_base.set(ObStorageType::OB_STORAGE_COS, tmp_account));
+
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/tagging_mode", dir_uri));
+    ASSERT_EQ(OB_SUCCESS, util.open(&tmp_cos_base));
+    ASSERT_EQ(OB_SUCCESS, util.write_single_file(uri, write_content, strlen(write_content)));
+
+    is_tagging = true;
+    ASSERT_EQ(OB_SUCCESS, util.is_tagging(uri, is_tagging));
+    ASSERT_FALSE(is_tagging);
+
+    ASSERT_EQ(OB_SUCCESS, util.del_file(uri));
+    ASSERT_EQ(OB_SUCCESS, util.is_tagging(uri, is_tagging));
+    ASSERT_TRUE(is_tagging);
+    tmp_cos_base.reset();
+    util.close();
+
+    // clean
+    ASSERT_EQ(OB_SUCCESS,
+              databuff_printf(tmp_account, sizeof(tmp_account),
+              "host=%s&access_id=%s&access_key=%s&appid=%s",
+              endpoint, secretid, secretkey, appid));
+    ASSERT_EQ(OB_SUCCESS, tmp_cos_base.set(ObStorageType::OB_STORAGE_COS, tmp_account));
+
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/tagging_mode", dir_uri));
+    ASSERT_EQ(OB_SUCCESS, util.open(&tmp_cos_base));
+    ASSERT_EQ(OB_SUCCESS, util.del_file(uri));
+    ASSERT_EQ(OB_BACKUP_FILE_NOT_EXIST, util.is_tagging(uri, is_tagging));
+    util.close();
+  }
+}
+
+int main(int argc, char **argv)
+{
+  system("rm -f test_storage_cos.log*");
+  OB_LOGGER.set_file_name("test_storage_cos.log", true, true);
+  OB_LOGGER.set_log_level("INFO");
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

deps/oblib/unittest/lib/restore/test_storage_cos.h

@@ -0,0 +1,23 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef TEST_STORAGE_COS_H_
+#define TEST_STORAGE_COS_H_
+
+const bool enable_test = false;
+const char *bucket = "cos://xxx";
+const char *endpoint = "xxx";
+const char *secretid = "xxx";
+const char *secretkey = "xxx";
+const char *appid = "xxx";
+
+#endif

deps/oblib/unittest/lib/restore/test_storage_info.cpp

@@ -0,0 +1,114 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include <gtest/gtest.h>
+#include "lib/utility/ob_test_util.h"
+#include "lib/restore/ob_storage_info.h"
+
+using namespace oceanbase::common;
+
+TEST(ObObjectStorageInfo, file)
+{
+  const char *uri = "file:///backup_dir/?&delete_mode=tagging";
+  ObObjectStorageInfo info1;
+  ASSERT_FALSE(info1.is_valid());
+
+  // wrong storage info
+  const char *storage_info = NULL;
+  ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info));
+  storage_info = "access_key=xxx";    // nfs, endpoint/access_id/access_key should be empty
+  ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info));
+  storage_info = "access_key=xxx&encrypt_key=xxx";
+  ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info));
+  storage_info = "delete_mode=wrong_mode";
+  ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info));
+  storage_info = "delete_mode=delete";
+  ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info));
+
+  ObObjectStorageInfo info2;
+  ObObjectStorageInfo info3;
+  storage_info = "";
+  ObStorageType device_type = OB_STORAGE_FILE;
+  ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info));
+  ASSERT_EQ(OB_SUCCESS, info2.set(device_type, storage_info));
+  ASSERT_EQ(OB_SUCCESS, info3.assign(info1));
+
+  ASSERT_EQ(info1, info2);
+  ASSERT_EQ(info1, info3);
+
+  // get
+  ASSERT_EQ(device_type, info1.get_type());
+
+  char buf[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 };
+  ASSERT_EQ(OB_SUCCESS, info1.get_storage_info_str(buf, sizeof(buf)));
+  ASSERT_STREQ("", buf);
+  ASSERT_EQ(OB_SUCCESS, info1.get_storage_info_str(buf, sizeof(buf)));
+  ASSERT_STREQ("", buf);
+
+  // clean
+  ASSERT_TRUE(info1.is_valid());
+  info1.reset();
+  ASSERT_FALSE(info1.is_valid());
+}
+
+TEST(ObObjectStorageInfo, oss)
+{
+  const char *uri = "oss://backup_dir?host=xxx.com&access_id=111&access_key=222";
+  ObObjectStorageInfo info1;
+
+  const char *storage_info = "";
+  ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info));
+  storage_info = "access_id=111&access_key=222";
+  ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info));
+
+  storage_info = "host=xxx.com&access_id=111&access_key=222";
+  ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info));
+
+  char buf[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 };
+  ASSERT_EQ(OB_SUCCESS, info1.get_storage_info_str(buf, sizeof(buf)));
+  ASSERT_STREQ(storage_info, buf);
+
+  storage_info = "host=xxx.com&access_id=111&access_key=222&delete_mode=delete";
+  info1.reset();
+  ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info));
+}
+
+TEST(ObObjectStorageInfo, cos)
+{
+  const char *uri = "cos://backup_dir?host=xxx.com&access_id=111&access_key=222&appid=333";
+  ObObjectStorageInfo info1;
+
+  const char *storage_info = "";
+  ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info));
+  storage_info = "host=xxx.com&access_id=111&access_key=222";
+  ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info));
+
+  storage_info = "host=xxx.com&access_id=111&access_key=222&appid=333";
+  ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info));
+
+  char buf[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 };
+  ASSERT_EQ(OB_SUCCESS, info1.get_storage_info_str(buf, sizeof(buf)));
+  ASSERT_STREQ(storage_info, buf);
+
+  storage_info = "host=xxx.com&access_id=111&access_key=222&appid=333&delete_mode=delete";
+  info1.reset();
+  ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info));
+}
+
+int main(int argc, char **argv)
+{
+  system("rm -f test_storage_info.log*");
+  OB_LOGGER.set_file_name("test_storage_info.log", true, true);
+  OB_LOGGER.set_log_level("INFO");
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

src/logservice/archiveservice/ob_archive_sender.cpp

@@ -793,7 +793,8 @@ bool ObArchiveSender::is_retry_ret_code_(const int ret_code) const
   return is_io_error(ret_code)
     || OB_ALLOCATE_MEMORY_FAILED == ret_code
     || OB_BACKUP_DEVICE_OUT_OF_SPACE == ret_code
-    || OB_BACKUP_PWRITE_OFFSET_NOT_MATCH == ret_code;
+    || OB_BACKUP_PWRITE_OFFSET_NOT_MATCH == ret_code
+    || OB_IO_LIMIT == ret_code;
 }
 
 bool ObArchiveSender::is_ignore_ret_code_(const int ret_code) const

src/logservice/ob_log_external_storage_io_task.h

@@ -162,7 +162,7 @@ protected:
   ObString uri_ob_str_;
   char uri_str_[OB_MAX_URI_LENGTH];
   ObString storage_info_ob_str_;
-  char storage_info_str_[share::OB_MAX_BACKUP_STORAGE_INFO_LENGTH];
+  char storage_info_str_[common::OB_MAX_BACKUP_STORAGE_INFO_LENGTH];
   int64_t generate_ts_;
   int64_t do_task_ts_;
   RunningStatus *running_status_;

src/logservice/restoreservice/ob_remote_data_generator.cpp

@@ -472,7 +472,10 @@ int LocationDataGenerator::read_file_(const ObString &base,
     ObString uri(path.get_obstr());
     char storage_info_cstr[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = {'\0'};
     int64_t real_size = 0;
-    if (OB_FAIL(storage_info->get_storage_info_str(storage_info_cstr, OB_MAX_BACKUP_STORAGE_INFO_LENGTH, false))) {
+    common::ObObjectStorageInfo storage_info_base;
+    if (OB_FAIL(storage_info_base.assign(*storage_info))) {
+      OB_LOG(WARN, "fail to assign storage info base!", K(ret), KP(storage_info));
+    } else if (OB_FAIL(storage_info_base.get_storage_info_str(storage_info_cstr, OB_MAX_BACKUP_STORAGE_INFO_LENGTH))) {
       LOG_WARN("get_storage_info_str failed", K(ret), K(uri), K(storage_info));
     } else {
       ObString storage_info_ob_str(storage_info_cstr);

src/share/backup/ob_backup_io_adapter.cpp

@@ -86,16 +86,20 @@ int ObBackupIoAdapter::get_and_init_device(ObIODevice*& dev_handle,
                                            const common::ObString &storage_type_prefix)
 {
   int ret = OB_SUCCESS;
-  char storage_info_str[share::OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 };
+  char storage_info_str[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 };
   ObIODOpts opts;
   ObIODOpt opt; //only one para
   opts.opts_ = &(opt);
   opts.opt_cnt_ = 1;
   opt.key_ = "storage_info";
+  common::ObObjectStorageInfo storage_info_base;
   if (NULL == storage_info) {
     ret = OB_INVALID_ARGUMENT;
     OB_LOG(WARN, "storage info is null", K(ret)); 
-  } else if (OB_FAIL(storage_info->get_storage_info_str(storage_info_str, sizeof(storage_info_str), false/*no need encrypt*/))) {
+  } else if (OB_FAIL(storage_info_base.assign(*storage_info))) {
+    OB_LOG(WARN, "fail to assign storage info base!", K(ret), KP(storage_info));
+  } else if (OB_FAIL(storage_info_base.get_storage_info_str(storage_info_str, sizeof(storage_info_str)))) {
+    // no need encrypt
     OB_LOG(WARN, "fail to get storage info str!", K(ret), KP(storage_info));
   } else if (FALSE_IT(opt.value_.value_str = storage_info_str)) {
   } else if (OB_FAIL(ObDeviceManager::get_instance().get_device(storage_info_str, storage_type_prefix, dev_handle))) {
@@ -603,34 +607,6 @@ int ObBackupIoAdapter::is_empty_directory(const common::ObString &uri,
   return ret;
 }
 
-int ObBackupIoAdapter::check_backup_dest_lifecycle(const common::ObString &path,  
-                      const share::ObBackupStorageInfo *storage_info,
-                      bool &is_set_lifecycle)
-{
-  int ret = OB_SUCCESS;
-  is_set_lifecycle = false;
-  ObIODevice* device_handle = NULL; 
-  if (OB_FAIL(get_and_init_device(device_handle, storage_info, path))) {
-    OB_LOG(WARN, "fail to get device!", K(ret));
-  } else {
-    ObIODOpts opts;
-    ObIODOpt opt[2];
-    
-    opts.opts_ = opt;
-    opts.opt_cnt_ = 2;
-    opt[0].set("get_bucket_lifecycle", false);
-    opt[1].set("path", path.ptr());
-    
-    if (OB_FAIL(device_handle->get_config(opts))) {
-      OB_LOG(WARN, "fail to get config for backup lifecycle!", K(ret), K(path), KP(storage_info));
-    } else {
-      is_set_lifecycle = opt[0].value_.value_bool;
-    }
-  }
-  release_device(device_handle);
-  return ret;
-}
-
 int ObBackupIoAdapter::set_access_type(ObIODOpts* opts, bool is_appender, int max_opt_num)
 {
   int ret = OB_SUCCESS;

src/share/backup/ob_backup_io_adapter.h

@@ -54,9 +54,6 @@ public:
   int is_empty_directory(const common::ObString &uri, 
       const share::ObBackupStorageInfo *storage_info, 
       bool &is_empty_directory);
-  int check_backup_dest_lifecycle(const common::ObString &path,  
-      const share::ObBackupStorageInfo *storage_info,
-      bool &is_set_lifecycle);
   int open_with_access_type(ObIODevice*& device_handle, ObIOFd &fd, 
               const share::ObBackupStorageInfo *storage_info, const common::ObString &uri,
               ObStorageAccessType access_type);

src/share/backup/ob_backup_path.cpp

@@ -1292,7 +1292,7 @@ int ObBackupPathUtil::construct_backup_set_dest(const share::ObBackupDest &backu
     ret = OB_ERR_UNEXPECTED;
     LOG_WARN("storage info must not be nullptr", K(ret));
   } else if (OB_FAIL(storage_info->get_storage_info_str(
-        storage_info_buf, OB_MAX_BACKUP_STORAGE_INFO_LENGTH, true/*need_encrypt*/))) {
+        storage_info_buf, OB_MAX_BACKUP_STORAGE_INFO_LENGTH))) {
     LOG_WARN("fail to get storage info", K(ret));
   } else if (OB_FAIL(backup_set_dest.set(path.get_ptr(), storage_info_buf))) {
     LOG_WARN("fail to set backup set dest", K(ret), K(path), K(storage_info_buf));
@@ -1324,7 +1324,7 @@ int ObBackupPathUtil::construct_backup_complement_log_dest(const share::ObBackup
     ret = OB_ERR_UNEXPECTED;
     LOG_WARN("storage info must not be nullptr", K(ret));
   } else if (OB_FAIL(storage_info->get_storage_info_str(
-        storage_info_buf, OB_MAX_BACKUP_STORAGE_INFO_LENGTH, true/*need_encrypt*/))) {
+        storage_info_buf, OB_MAX_BACKUP_STORAGE_INFO_LENGTH))) {
     LOG_WARN("fail to get storage info", K(ret));
   } else if (OB_FAIL(backup_set_dest.set(path.get_ptr(), storage_info_buf))) {
     LOG_WARN("fail to set backup set dest", K(ret), K(path), K(storage_info_buf));
@@ -1353,7 +1353,7 @@ int ObBackupPathUtil::construct_backup_complement_log_dest(const share::ObBackup
     ret = OB_ERR_UNEXPECTED;
     LOG_WARN("storage info must not be nullptr", K(ret));
   } else if (OB_FAIL(storage_info->get_storage_info_str(
-        storage_info_buf, OB_MAX_BACKUP_STORAGE_INFO_LENGTH, true/*need_encrypt*/))) {
+        storage_info_buf, OB_MAX_BACKUP_STORAGE_INFO_LENGTH))) {
     LOG_WARN("fail to get storage info", K(ret));
   } else if (OB_FAIL(backup_set_dest.set(path.get_ptr(), storage_info_buf))) {
     LOG_WARN("fail to set backup set dest", K(ret), K(path), K(storage_info_buf));

src/share/backup/ob_backup_struct.cpp

@@ -1102,39 +1102,11 @@ int ObLogArchiveBackupInfo::get_backup_dest(ObBackupDest &backup_dest) const
 }
 
 //***********************ObBackupStorageInfo***************************
-ObBackupStorageInfo::ObBackupStorageInfo()
-  : device_type_(ObStorageType::OB_STORAGE_MAX_TYPE)
-{
-  endpoint_[0] = '\0';
-  access_id_[0] = '\0';
-  access_key_[0] = '\0';
-  extension_[0] = '\0';
-}
-
 ObBackupStorageInfo::~ObBackupStorageInfo()
 {
   reset();
 }
 
-void ObBackupStorageInfo::reset()
-{
-  device_type_ = ObStorageType::OB_STORAGE_MAX_TYPE;
-  endpoint_[0] = '\0';
-  access_id_[0] = '\0';
-  access_key_[0] = '\0';
-  extension_[0] = '\0';
-}
-
-bool ObBackupStorageInfo::is_valid() const
-{
-  return device_type_ >= 0 && device_type_ < ObStorageType::OB_STORAGE_MAX_TYPE;
-}
-
-const char *ObBackupStorageInfo::get_type_str() const
-{
-  return get_storage_type_str(device_type_);
-}
-
 int ObBackupStorageInfo::set(
     const common::ObStorageType device_type,
     const char *endpoint,
@@ -1142,98 +1114,77 @@ int ObBackupStorageInfo::set(
     const char *extension)
 {
   int ret = OB_SUCCESS;
+  char storage_info[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 };
   if (is_valid()) {
     ret = OB_INIT_TWICE;
     LOG_WARN("storage info init twice", K(ret));
-  } else if (OB_ISNULL(endpoint) || OB_ISNULL(authorization) || OB_ISNULL(extension) || OB_STORAGE_MAX_TYPE == device_type) {
+  } else if (OB_ISNULL(endpoint)
+      || OB_ISNULL(authorization) || OB_ISNULL(extension) || OB_STORAGE_MAX_TYPE == device_type) {
     ret = OB_INVALID_BACKUP_DEST;
     LOG_WARN("invalid args", K(ret), KP(endpoint), KP(authorization), KP(extension), K(device_type));
-  } else if (FALSE_IT(device_type_ = device_type)) {
-  } else if (0 != strlen(endpoint) && OB_FAIL(databuff_printf(endpoint_, sizeof(endpoint_), "%s", endpoint))) {
-    LOG_WARN("failed to set endpoint", K(ret));
-  } else if (0 != strlen(authorization) && OB_FAIL(parse_authorization_(authorization))) {
-    LOG_WARN("failed to parse authorization", K(ret));
-  } else if (0 != strlen(extension) && OB_FAIL(databuff_printf(extension_, sizeof(extension_), "%s", extension))) {
-    LOG_WARN("failed to set extension", K(ret));
-  }
-
-  if (OB_FAIL(ret)) {
-  } else if (ObStorageType::OB_STORAGE_FILE != device_type_ && (0 == strlen(endpoint_) || 0 == strlen(access_id_) || 0 == strlen(access_key_))) {
-    ret = OB_INVALID_BACKUP_DEST;
-    LOG_WARN("backup device is not nfs, endpoint/access_id/access_key do not allow to be empty", K(ret), K_(device_type), K_(endpoint), K_(access_id));
-  } else if (ObStorageType::OB_STORAGE_FILE == device_type_ && (0 != strlen(endpoint_) || 0 != strlen(access_id_) || 0 != strlen(access_key_))) {
-    ret = OB_INVALID_BACKUP_DEST;
-    LOG_WARN("backup device is nfs, endpoint/access_id/access_key not be empty", K(ret), K_(device_type), K_(endpoint), K_(access_id));
+  } else if (0 != strlen(endpoint)
+      && OB_FAIL(set_storage_info_field_(endpoint, storage_info, sizeof(storage_info)))) {
+    LOG_WARN("failed to set storage info", K(ret));
+  } else if (0 != strlen(authorization)
+      && OB_FAIL(set_storage_info_field_(authorization, storage_info, sizeof(storage_info)))) {
+    LOG_WARN("failed to set storage info", K(ret));
+  } else if (0 != strlen(extension)
+      && OB_FAIL(set_storage_info_field_(extension, storage_info, sizeof(storage_info)))) {
+    LOG_WARN("failed to set storage info", K(ret));
+  } else if (OB_FAIL(set(device_type, storage_info))) {
+    LOG_WARN("failed to set storage info", K(ret), KPC(this));
   }
   return ret;
 }
 
-int ObBackupStorageInfo::set_access_key_(const char *buf, const bool need_decrypt)
+int ObBackupStorageInfo::get_authorization_info(char *authorization, const int64_t length) const
 {
   int ret = OB_SUCCESS;
-  if (OB_ISNULL(buf)) {
+  const int64_t key_len = MAX(OB_MAX_BACKUP_SERIALIZEKEY_LENGTH, OB_MAX_BACKUP_ACCESSKEY_LENGTH);
+  char access_key_buf[key_len] = { 0 };
+  STATIC_ASSERT(OB_MAX_BACKUP_AUTHORIZATION_LENGTH > (OB_MAX_BACKUP_ACCESSID_LENGTH + key_len), "array length overflow");
+  if (!is_valid()) {
     ret = OB_INVALID_ARGUMENT;
-    LOG_WARN("buf is null", K(ret), K(need_decrypt));
-  } else if (0 != strlen(access_key_)) {
+    LOG_WARN("storage info not init", K(ret));
+  } else if (OB_ISNULL(authorization) || length <= 0) {
     ret = OB_INVALID_ARGUMENT;
-    LOG_WARN("access_key has been set value", K(ret), K_(access_key));
+    LOG_WARN("invalid args", K(ret), KP(authorization), K(length));
+  } else if (OB_STORAGE_FILE == device_type_) {
+    // do nothing
+  } else if (OB_FAIL(get_access_key_(access_key_buf, sizeof(access_key_buf)))) {
+    LOG_WARN("failed to get access key", K(ret));
+  } else if (OB_FAIL(databuff_printf(authorization, length, "%s&%s",  access_id_, access_key_buf))) {
+    LOG_WARN("failed to set authorization", K(ret), K(length), K_(access_id), K(strlen(access_key_buf)));
+  }
+
+  return ret;
+}
+
 #ifdef OB_BUILD_TDE_SECURITY
-  } else if (need_decrypt) {
-    char encrypted_key[OB_MAX_BACKUP_ENCRYPTKEY_LENGTH] = { 0 };
-    if (OB_FAIL(set_storage_info_field_(buf, encrypted_key, sizeof(encrypted_key)))) {
-      LOG_WARN("failed to set access_key", K(ret), K(buf));
-    } else if (OB_FAIL(decrypt_access_key_(encrypted_key))) {
-      LOG_WARN("failed to set access key", K(ret), K(buf));
-    }
-#endif
-  } else if (OB_FAIL(set_storage_info_field_(buf, access_key_, sizeof(access_key_)))) {
-    LOG_WARN("failed to set endpoint",K(ret), K(buf));
+int ObBackupStorageInfo::get_access_key_(char *key_buf, const int64_t key_buf_len) const
+{
+  int ret = OB_SUCCESS;
+  // encrypt_access_key_ will check args' validity
+  if (OB_FAIL(encrypt_access_key_(key_buf, key_buf_len))) {
+    LOG_WARN("failed to encrypt access key", K(ret));
   }
   return ret;
 }
 
-int ObBackupStorageInfo::check_delete_mode_(const char *delete_mode)
+int ObBackupStorageInfo::parse_storage_info_(const char *storage_info, bool &has_appid)
 {
   int ret = OB_SUCCESS;
-  if (NULL == delete_mode) {
-    ret = OB_INVALID_ARGUMENT;
-    OB_LOG(WARN, "invalid args", K(ret), KP(delete_mode));
-  } else if (0 != strcmp(delete_mode, "delete")
-      && 0 != strcmp(delete_mode, "tagging")) {
-    ret = OB_INVALID_ARGUMENT;
-    OB_LOG(WARN, "delete mode is invalid", K(ret), K(delete_mode));
-  }
-  return ret;
-}
-
-// oss:host=xxxx&access_id=xxx&access_key=xxx
-// cos:host=xxxx&access_id=xxx&access_key=xxxappid=xxx
-int ObBackupStorageInfo::set(
-    const common::ObStorageType device_type,
-    const char *storage_info)
-{
-  int ret = OB_SUCCESS;
-  if (is_valid()) {
-    ret = OB_INIT_TWICE;
-    LOG_WARN("storage info init twice", K(ret));
-  } else if (OB_ISNULL(storage_info) || strlen(storage_info) >= OB_MAX_BACKUP_STORAGE_INFO_LENGTH) {
+  if (OB_ISNULL(storage_info) || strlen(storage_info) >= OB_MAX_BACKUP_STORAGE_INFO_LENGTH) {
     ret = OB_INVALID_BACKUP_DEST;
-    LOG_WARN("storage info is too long", K(ret), K(storage_info), K(strlen(storage_info)));
-  } else if (FALSE_IT(device_type_ = device_type)) {
-  } else if (0 == strlen(storage_info)) {
-    if (OB_STORAGE_FILE != device_type_) {
-      ret = OB_INVALID_BACKUP_DEST;
-      LOG_WARN("backup dest is valid", K(ret), K_(device_type));
-    }
+    LOG_WARN("storage info is invalid", K(ret), K(storage_info), K(strlen(storage_info)));
+  } else if (OB_FAIL(ObObjectStorageInfo::parse_storage_info_(storage_info, has_appid))) {
+    LOG_WARN("failed to parse storage info", K(ret), K(storage_info));
   } else {
-    int64_t info_len = strlen(storage_info);
     char tmp[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 };
+    char serialize_key[OB_MAX_BACKUP_SERIALIZEKEY_LENGTH] = { 0 };
     char *token = NULL;
     char *saved_ptr = NULL;
-    const char *HOST = "host=";
-    const char *APPID = "appid=";
-    const char *DELETE_MODE = "delete_mode=";
-    bool has_appid = false;
+    int64_t info_len = strlen(storage_info);
 
     MEMCPY(tmp, storage_info, info_len);
     tmp[info_len] = '\0';
@@ -1242,232 +1193,22 @@ int ObBackupStorageInfo::set(
       token = ::strtok_r(str, "&", &saved_ptr);
       if (NULL == token) {
         break;
-      } else if (0 == strncmp(HOST, token, strlen(HOST))) {
-        if (OB_FAIL(set_storage_info_field_(token, endpoint_, sizeof(endpoint_)))) {
-          LOG_WARN("failed to set endpoint",K(ret), K(token));
-        }
-      } else if (0 == strncmp(ACCESS_ID, token, strlen(ACCESS_ID))) {
-        if (OB_FAIL(set_storage_info_field_(token, access_id_, sizeof(access_id_)))) {
-          LOG_WARN("failed to set access id", K(ret), K(token));
-        }
-#ifdef OB_BUILD_TDE_SECURITY
       } else if (0 == strncmp(ENCRYPT_KEY, token, strlen(ENCRYPT_KEY))) {
-        if (OB_FAIL(set_access_key_(token, true/*need decrypt*/))) {
-          LOG_WARN("failed to set oss extension", K(ret), K(token));
-        }
-#endif
-      } else if (0 == strncmp(ACCESS_KEY, token, strlen(ACCESS_KEY))) {
-        if (OB_FAIL(set_access_key_(token, false/*need decrypt*/))) {
-          LOG_WARN("failed to set oss extension", K(ret), K(token));
-        }
-      } else if (OB_STORAGE_COS == device_type && 0 == strncmp(APPID, token, strlen(APPID))) {
-        has_appid = true;
-        if (OB_FAIL(set_storage_info_field_(token, extension_, sizeof(extension_)))) {
-          LOG_WARN("failed to set cos extension", K(ret), K(token));
-        }
-      } else if (OB_STORAGE_OSS == device_type && 0 == strncmp(DELETE_MODE, token, strlen(DELETE_MODE))) {
-        if (OB_FAIL(check_delete_mode_(token + strlen(DELETE_MODE)))) {
-          OB_LOG(WARN, "failed to set delete mode", K(ret), K(token));
-        } else if (OB_FAIL(set_storage_info_field_(token, extension_, sizeof(extension_)))) {
-          LOG_WARN("failed to set cos extension", K(ret), K(token));
-        }
-      } else {
-        LOG_DEBUG("unkown storage info", K(*token));
-      }
-    }
-
-    if (OB_FAIL(ret)) {
-    } else if (OB_STORAGE_OSS == device_type &&(0 == strlen(endpoint_) || 0 == strlen(access_id_) || 0 == strlen(access_key_))) {
-      ret = OB_INVALID_BACKUP_DEST;
-      LOG_WARN("failed to parse storage info", K(ret), K_(endpoint), K_(access_id), K_(access_key));
-    } else if (OB_STORAGE_COS == device_type && !has_appid) {
-      ret = OB_INVALID_BACKUP_DEST;
-      LOG_WARN("failed to parse  cos info", K(ret), K_(extension));
-    } else {
-      LOG_INFO("success to parse storage info", K(ret), K_(endpoint), K_(access_id), K_(extension));
-    }
-  }
-
-  return ret;
-}
-
-int ObBackupStorageInfo::set(
-    const char *uri,
-    const char *storage_info)
-{
-  int ret = OB_SUCCESS;
-  common::ObStorageType device_type;
-  if (OB_FAIL(get_storage_type_from_path(uri, device_type))) {
-    LOG_WARN("failed to get storage type from path", K(ret), KPC(this));
-  } else if (OB_FAIL(set(device_type, storage_info))) {
-    LOG_WARN("failed to set storage info", K(ret), KPC(this));
-  }
-  return ret;
-}
-
-int ObBackupStorageInfo::parse_authorization_(const char *authorization)
-{
-  int ret = OB_SUCCESS;
-  char tmp_authorization[OB_MAX_BACKUP_AUTHORIZATION_LENGTH] = { 0 };
-  char *token = NULL;
-  char *saved_ptr = NULL;
-  if (OB_ISNULL(authorization) || 0 == strlen(authorization)) {
-    ret = OB_INVALID_ARGUMENT;
-    LOG_WARN("authorization is invalid", K(ret));
-  } else if (OB_FAIL(databuff_printf(tmp_authorization, sizeof(tmp_authorization), "%s", authorization))) {
-    LOG_WARN("failed to set token", K(ret));
-  } else {
-    token = tmp_authorization;
-    for (char *str = token; OB_SUCC(ret); str = NULL) {
-      token = ::strtok_r(str, "&", &saved_ptr);
-      if (NULL == token) {
-        break;
-      } else if (0 == strncmp(ACCESS_ID, token, strlen(ACCESS_ID))) {
-        if (OB_FAIL(set_storage_info_field_(token, access_id_, sizeof(access_id_)))) {
-          LOG_WARN("failed to set access id", K(ret), K(token));
-        }
-#ifdef OB_BUILD_TDE_SECURITY
-      } else if (0 == strncmp(ENCRYPT_KEY, token, strlen(ENCRYPT_KEY))) {
-        char serialize_key[OB_MAX_BACKUP_SERIALIZEKEY_LENGTH] = { 0 };
-        if (OB_FAIL(set_storage_info_field_(token, serialize_key, sizeof(serialize_key)))) {
-          LOG_WARN("failed to set encrypt_key", K(ret), K(token));
+        if (0 != strlen(access_key_)) {
+          ret = OB_INVALID_ARGUMENT;
+          LOG_WARN("access_key has been set value", K(ret), K(strlen(access_key_)));
+        } else if (OB_FAIL(set_storage_info_field_(token, serialize_key, sizeof(serialize_key)))) {
+          LOG_WARN("failed to set encrypted_key", K(ret), K(token));
         } else if (OB_FAIL(decrypt_access_key_(serialize_key))) {
           LOG_WARN("failed to decrypt access key", K(ret), K(token));
         }
-#endif
-      } else if (0 == strncmp(ACCESS_KEY, token, strlen(ACCESS_KEY))) {
-        if (OB_FAIL(set_storage_info_field_(token, access_key_, sizeof(access_key_)))) {
-          LOG_WARN("failed to set access key", K(ret), K(token));
-        }
       } else {
-        ret = OB_INVALID_ARGUMENT;
-        LOG_WARN("invalid argument", K(ret), K(token));
       }
     }
   }
   return ret;
 }
 
-int ObBackupStorageInfo::get_authorization_info(char *authorization, int64_t length)
-{
-  int ret = OB_SUCCESS;
-  char encrypt_key[OB_MAX_BACKUP_SERIALIZEKEY_LENGTH] = { 0 };
-  STATIC_ASSERT(OB_MAX_BACKUP_AUTHORIZATION_LENGTH > (OB_MAX_BACKUP_ACCESSID_LENGTH + OB_MAX_BACKUP_SERIALIZEKEY_LENGTH), "array length overflow");
-  if (!is_valid()) {
-    ret = OB_INVALID_ARGUMENT;
-    LOG_WARN("storage info not init", K(ret));
-  } else if (OB_ISNULL(authorization) || length <= 0) {
-    ret = OB_INVALID_ARGUMENT;
-    LOG_WARN("storage info is invalid", K(ret));
-  } else if (OB_STORAGE_FILE == device_type_) {
-    // do nothing
-#ifndef OB_BUILD_TDE_SECURITY
-  } else if (OB_FAIL(databuff_printf(authorization, length, "%s&%s",  access_id_, access_key_))) {
-    LOG_WARN("failed to set authorization", K(ret), K_(access_id), K(access_key_));
-#else
-  } else if (OB_FAIL(encrypt_access_key_(encrypt_key, sizeof(encrypt_key)))) {
-    LOG_WARN("failed to encrept access key", K(ret));
-  } else if (OB_FAIL(databuff_printf(authorization, length, "%s&%s",  access_id_, encrypt_key))) {
-    LOG_WARN("failed to set authorization", K(ret), K_(access_id), K(encrypt_key));
-#endif
-  }
-  return ret;
-}
-
-int ObBackupStorageInfo::get_storage_info_str(char *storage_info, int64_t info_len, const bool need_encrypt) const
-{
-  int ret = OB_SUCCESS;
-  char encrypt_key[OB_MAX_BACKUP_SERIALIZEKEY_LENGTH] = { 0 };
-  const char *key = NULL;
-  if (!is_valid()) {
-    ret = OB_INVALID_ARGUMENT;
-    LOG_WARN("storage info not init", K(ret));
-  } else if (OB_ISNULL(storage_info) || info_len <= 0) {
-    ret = OB_INVALID_ARGUMENT;
-    LOG_WARN("storage info is invalid", K(ret));
-  } else if (OB_STORAGE_FILE == device_type_) {
-    // do nothing
-#ifdef OB_BUILD_TDE_SECURITY
-  } else if (need_encrypt) {
-    if (OB_FAIL(encrypt_access_key_(encrypt_key, sizeof(encrypt_key)))) {
-      LOG_WARN("failed to encrypt access key", K(ret));
-    } else {
-      key = encrypt_key;
-    }
-#endif
-  } else {
-    key = access_key_;
-  }
-
-  if (OB_FAIL(ret)) {
-  } else if (OB_STORAGE_FILE != device_type_ && OB_FAIL(databuff_printf(storage_info, info_len, "%s&%s&%s",  endpoint_, access_id_, key))) {
-    LOG_WARN("failed to set storage info", K(ret));
-  } else if (0 != strlen(extension_) && info_len > strlen(storage_info)) {
-    const int64_t str_len = strlen(storage_info);
-    if (OB_FAIL(databuff_printf(storage_info + str_len, info_len - str_len, "&%s", extension_))) {
-      LOG_WARN("failed to add extension", K(ret), K_(extension));
-    }
-  }
-  return ret;
-}
-
-int ObBackupStorageInfo::assign(const ObBackupStorageInfo &storage_info)
-{
-  int ret = OB_SUCCESS;
-  device_type_ = storage_info.device_type_;
-  MEMCPY(endpoint_, storage_info.endpoint_, sizeof(endpoint_));
-  MEMCPY(access_id_, storage_info.access_id_, sizeof(access_id_));
-  MEMCPY(access_key_, storage_info.access_key_, sizeof(access_key_));
-  MEMCPY(extension_, storage_info.extension_, sizeof(extension_));
-
-  return ret;
-}
-
-bool ObBackupStorageInfo::operator ==(const ObBackupStorageInfo &storage_info) const
-{
-  return device_type_ == storage_info.device_type_
-      && (0 == STRCMP(endpoint_, storage_info.endpoint_))
-      && (0 == STRCMP(access_id_, storage_info.access_id_))
-      && (0 == STRCMP(access_key_, storage_info.access_key_))
-      && (0 == STRCMP(extension_, storage_info.extension_));
-}
-
-bool ObBackupStorageInfo::operator !=(const ObBackupStorageInfo &storage_info) const
-{
-  return !(*this == storage_info);
-}
-
-int64_t ObBackupStorageInfo::hash() const
-{
-  int64_t hash_value = 0;
-  hash_value = murmurhash(&device_type_, static_cast<int32_t>(sizeof(device_type_)), hash_value);
-  hash_value = murmurhash(endpoint_, static_cast<int32_t>(strlen(endpoint_)), hash_value);
-  hash_value = murmurhash(access_id_, static_cast<int32_t>(strlen(access_id_)), hash_value);
-  hash_value = murmurhash(access_key_, static_cast<int32_t>(strlen(access_key_)), hash_value);
-  hash_value = murmurhash(extension_, static_cast<int32_t>(strlen(extension_)), hash_value);
-  return hash_value;
-}
-
-int ObBackupStorageInfo::set_storage_info_field_(const char *info, char *field, const int64_t length)
-{
-  int ret = OB_SUCCESS;
-  if (NULL == info || NULL == field) {
-    ret = OB_INVALID_ARGUMENT;
-    LOG_WARN("invalid args", K(ret), KP(info), KP(field));
-  } else {
-    const int64_t info_len = strlen(info);
-    int64_t pos = strlen(field);
-    if (info_len >= length) {
-      ret = OB_INVALID_ARGUMENT;
-      LOG_WARN("info is too long ", K(ret), K(info), K(length));
-    } else if (OB_FAIL(databuff_printf(field, length, pos, "%s", info))){
-      LOG_WARN("failed to set storage info field", K(ret), K(info), K(field));
-    }
-  }
-  return ret;
-}
-
-#ifdef OB_BUILD_TDE_SECURITY
 int ObBackupStorageInfo::encrypt_access_key_(char *encrypt_key, int64_t length) const
 {
   int ret = OB_SUCCESS;
@@ -1475,7 +1216,10 @@ int ObBackupStorageInfo::encrypt_access_key_(char *encrypt_key, int64_t length)
   char serialize_buf[OB_MAX_BACKUP_SERIALIZEKEY_LENGTH] = { 0 };
   int64_t serialize_pos = 0;
   int64_t key_len = 0;
-  if (0 == strlen(access_key_)) {
+  if (OB_ISNULL(encrypt_key) || length <= 0) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid args", K(ret), KP(encrypt_key), K(length));
+  } else if (0 == strlen(access_key_)) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("access_key is empty, shouldn't encrypt", K(ret));
   } else if (0 != strncmp(ACCESS_KEY, access_key_, strlen(ACCESS_KEY))) {
@@ -1504,11 +1248,12 @@ int ObBackupStorageInfo::decrypt_access_key_(const char *buf)
   int64_t key_len = 0;
   char deserialize_buf[OB_MAX_BACKUP_SERIALIZEKEY_LENGTH] = { 0 };
   char decrypt_key[OB_MAX_BACKUP_ACCESSKEY_LENGTH] = { 0 };
-  int64_t buf_len = strlen(buf);
+  int64_t buf_len = 0;
   int64_t deserialize_size = 0;
-  if (0 == buf_len) {
+  if (OB_ISNULL(buf) || strlen(buf) == 0) {
     ret = OB_INVALID_ARGUMENT;
-    LOG_WARN("encrypted_key is empty", K(ret));
+    LOG_WARN("encrypted_key is empty", K(ret), KP(buf), K(strlen(buf)));
+  } else if (FALSE_IT(buf_len = strlen(buf))) {
   } else if (OB_FAIL(hex_to_cstr(buf + strlen(ENCRYPT_KEY), buf_len - strlen(ENCRYPT_KEY),
       deserialize_buf, sizeof(deserialize_buf), deserialize_size))) {
     LOG_WARN("failed to get cstr from hex", KR(ret), K(buf_len), K(sizeof(deserialize_buf)));
@@ -1861,7 +1606,7 @@ int ObBackupDest::get_backup_dest_str(char *buf, const int64_t buf_size) const
     LOG_WARN("invalid argument", K(ret), KP(buf), K(buf_size));
   } else if (OB_FAIL(databuff_printf(buf, buf_size, "%s", root_path_))) {
     LOG_WARN("failed to get backup dest str", K(ret), K(root_path_), K(storage_info_));
-  } else if (OB_FAIL(storage_info_->get_storage_info_str(storage_info_str, sizeof(storage_info_str), true/*no need encrypt*/))) {
+  } else if (OB_FAIL(storage_info_->get_storage_info_str(storage_info_str, sizeof(storage_info_str)))) {
     OB_LOG(WARN, "fail to get storage info str!", K(ret), K(storage_info_));
   } else if (0 != strlen(storage_info_str) && OB_FAIL(databuff_printf(buf + strlen(buf), buf_size - strlen(buf), "?%s",storage_info_str))) {
     LOG_WARN("failed to get backup dest str", K(ret), K(root_path_), K(storage_info_));
@@ -1885,7 +1630,7 @@ int ObBackupDest::get_backup_dest_str_with_primary_attr(char *buf, const int64_t
     LOG_WARN("invalid argument", K(ret), KP(buf), K(buf_size));
   } else if (OB_FAIL(databuff_printf(buf, buf_size, "%s", root_path_))) {
     LOG_WARN("failed to get backup dest str", K(ret), K(root_path_), K(storage_info_));
-  } else if (OB_FAIL(storage_info_->get_storage_info_str(storage_info_str, sizeof(storage_info_str), true/*no need encrypt*/))) {
+  } else if (OB_FAIL(storage_info_->get_storage_info_str(storage_info_str, sizeof(storage_info_str)))) {
     OB_LOG(WARN, "fail to get storage info str!", K(ret), K(storage_info_));
   } else if (0 != strlen(storage_info_str) && OB_FAIL(databuff_printf(buf + strlen(buf), buf_size - strlen(buf), "?%s",storage_info_str))) {
     LOG_WARN("failed to get backup dest str", K(ret), K(root_path_), K(storage_info_));

src/share/backup/ob_backup_struct.h

@@ -55,14 +55,7 @@ const int64_t OB_BACKUP_DEFAULT_PG_NUM = 10000;
 const int64_t OB_MAX_BACKUP_DEST_LENGTH = 2048;
 const int64_t OB_MAX_RESTORE_DEST_LENGTH = OB_MAX_BACKUP_DEST_LENGTH * 10;
 const int64_t OB_MAX_BACKUP_PATH_LENGTH = 1024;
-const int64_t OB_MAX_BACKUP_STORAGE_INFO_LENGTH = 1536;
-const int64_t OB_MAX_BACKUP_ENDPOINT_LENGTH = 256;
-const int64_t OB_MAX_BACKUP_ACCESSID_LENGTH = 256;
-const int64_t OB_MAX_BACKUP_ACCESSKEY_LENGTH = 256;
-const int64_t OB_MAX_BACKUP_ENCRYPTKEY_LENGTH = OB_MAX_BACKUP_ACCESSKEY_LENGTH + 32;
-const int64_t OB_MAX_BACKUP_SERIALIZEKEY_LENGTH = OB_MAX_BACKUP_ENCRYPTKEY_LENGTH * 2;
 const int64_t OB_MAX_BACKUP_AUTHORIZATION_LENGTH = 1024;
-const int64_t OB_MAX_BACKUP_EXTENSION_LENGTH = 512;
 const int64_t OB_MAX_BACKUP_CHECK_FILE_LENGTH = OB_MAX_BACKUP_PATH_LENGTH;
 const int64_t OB_MAX_BACKUP_CHECK_FILE_NAME_LENGTH = 256;
 const int64_t OB_MAX_KEPT_LOG_ARCHIVE_BACKUP_ROUND = 10 * 10000; // 10w
@@ -358,9 +351,6 @@ const char *const OB_RESTORE_PREVIEW_BACKUP_CLUSTER_ID_SESSION_STR = "__ob_resto
 const char *const MULTI_BACKUP_SET_PATH_PREFIX = "BACKUPSET";
 const char *const MULTI_BACKUP_PIECE_PATH_PREFIX = "BACKUPPIECE";
 
-const char *const ACCESS_ID = "access_id=";
-const char *const ACCESS_KEY = "access_key=";
-
 const char *const ENCRYPT_KEY = "encrypt_key=";
 const char *const OB_STR_INITIATOR_JOB_ID = "initiator_job_id";
 const char *const OB_STR_EXECUTOR_TENANT_ID = "executor_tenant_id";
@@ -855,47 +845,29 @@ struct ObNonFrozenBackupPieceInfo final
   DECLARE_TO_STRING;
 };
 
-class ObBackupStorageInfo final
+class ObBackupStorageInfo final : public common::ObObjectStorageInfo
 {
 public:
-  ObBackupStorageInfo();
-  ~ObBackupStorageInfo();
-  int set(
-      const common::ObStorageType device_type,
-      const char *storage_info);
+  using common::ObObjectStorageInfo::set;
+
+public:
+  ObBackupStorageInfo() {}
+  virtual ~ObBackupStorageInfo();
+
   int set(
       const common::ObStorageType device_type,
       const char *endpoint,
       const char *authorization,
       const char *extension);
-  int set(
-      const char *uri,
-      const char *storage_info);
-  int get_storage_info_str(char *storage_info, int64_t info_len, const bool need_encrypt) const;
-  int get_authorization_info(char *authorization, int64_t length);
-  bool is_valid() const;
-  void reset();
-  bool operator ==(const ObBackupStorageInfo &storage_info) const;
-  bool operator !=(const ObBackupStorageInfo &storage_info) const;
-  const char *get_type_str() const;
-  int64_t hash() const;
-  int assign(const ObBackupStorageInfo &storage_info);
-  TO_STRING_KV(K_(endpoint), K_(access_id), K_(extension), "type", get_type_str());
+  int get_authorization_info(char *authorization, const int64_t length) const;
+
 private:
-  int set_storage_info_field_(const char *info, char *field, const int64_t length);
 #ifdef OB_BUILD_TDE_SECURITY
-  int encrypt_access_key_(char *encrypt_key, int64_t length) const;
+  virtual int get_access_key_(char *key_buf, const int64_t key_buf_len) const override;
+  virtual int parse_storage_info_(const char *storage_info, bool &has_appid) override;
+  int encrypt_access_key_(char *encrypt_key, const int64_t length) const;
   int decrypt_access_key_(const char *buf);
 #endif
-  int set_access_key_(const char *buf, const bool need_decrypt);
-  int parse_authorization_(const char *authorization);
-  int check_delete_mode_(const char *delete_mode);
-public:
-  common::ObStorageType device_type_;
-  char endpoint_[OB_MAX_BACKUP_ENDPOINT_LENGTH];
-  char access_id_[OB_MAX_BACKUP_ACCESSID_LENGTH];
-  char access_key_[OB_MAX_BACKUP_ACCESSKEY_LENGTH];
-  char extension_[OB_MAX_BACKUP_EXTENSION_LENGTH];
 };
 
 class ObBackupDest final

src/share/ob_device_manager.cpp

@@ -48,6 +48,8 @@ int ObDeviceManager::init_devices_env()
       OB_LOG(WARN, "Fail to init allocator ", K(ret));
     } else if (OB_FAIL(init_oss_env())) {
       OB_LOG(WARN, "fail to init oss storage", K(ret));
+    } else if (OB_FAIL(init_cos_env())) {
+      OB_LOG(WARN, "fail to init cos storage", K(ret));
     }
   }
 
@@ -81,6 +83,7 @@ void ObDeviceManager::destroy()
     }
     allocator_.reset();
     fin_oss_env();
+    fin_cos_env();
     is_init_ = false;
     device_count_ = 0;
     OB_LOG_RET(WARN, ret_dev, "release the init resource", K(ret_dev), K(ret_handle));

src/sql/resolver/ddl/ob_ddl_resolver.cpp

@@ -2001,7 +2001,7 @@ int ObDDLResolver::resolve_table_option(const ParseNode *option_node, const bool
               OZ (ObSQLUtils::split_remote_object_storage_url(url, storage_info));
             }
             OZ (tmp_location.append(url));
-            OZ (storage_info.get_storage_info_str(storage_info_buf, sizeof(storage_info_buf), true));
+            OZ (storage_info.get_storage_info_str(storage_info_buf, sizeof(storage_info_buf)));
             OZ (arg.schema_.set_external_file_location(tmp_location.string()));
             OZ (arg.schema_.set_external_file_location_access_info(storage_info_buf));
           }

tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp

@@ -510,7 +510,7 @@ ObAdminDumpBackupDataExecutor::ObAdminDumpBackupDataExecutor() : offset_(), leng
     allocator_()
 {
   MEMSET(backup_path_, 0, common::OB_MAX_URI_LENGTH);
-  MEMSET(storage_info_, 0, share::OB_MAX_BACKUP_STORAGE_INFO_LENGTH);
+  MEMSET(storage_info_, 0, common::OB_MAX_BACKUP_STORAGE_INFO_LENGTH);
 }
 
 ObAdminDumpBackupDataExecutor::~ObAdminDumpBackupDataExecutor()

tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.h

@@ -193,7 +193,7 @@ private:
       storage::ObExternTenantLocalityInfoDesc &locality_info);
 private:
   char backup_path_[common::OB_MAX_URI_LENGTH];
-  char storage_info_[share::OB_MAX_BACKUP_STORAGE_INFO_LENGTH];
+  char storage_info_[common::OB_MAX_BACKUP_STORAGE_INFO_LENGTH];
   int64_t offset_;
   int64_t length_;
   int64_t file_type_;

unittest/share/backup/CMakeLists.txt

@@ -1,4 +1,5 @@
 storage_unittest(test_backup_path)
 storage_unittest(test_backup_struct)
 storage_unittest(test_log_archive_backup_info_mgr)
-storage_unittest(test_archive_checkpoint_mgr)
+storage_unittest(test_backup_access_cos)
+storage_unittest(test_archive_checkpoint_mgr)

unittest/share/backup/test_backup_access_cos.cpp

@@ -0,0 +1,326 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include <gtest/gtest.h>
+#include "lib/utility/ob_test_util.h"
+#include "lib/restore/ob_storage.h"
+#include "lib/restore/ob_storage_cos_base.h"
+#include "lib/allocator/page_arena.h"
+#include "share/backup/ob_backup_io_adapter.h"
+#include "test_backup_access_cos.h"
+
+using namespace oceanbase::common;
+
+class TestStorageCosCommon {
+public:
+TestStorageCosCommon() {}
+~TestStorageCosCommon() {}
+
+void init()
+{
+  // ObDeviceManager will init env in first call
+  // init_cos_env();
+
+  ASSERT_EQ(OB_SUCCESS,
+            databuff_printf(account, sizeof(account),
+            "host=%s&access_id=%s&access_key=%s&appid=%s",
+            endpoint, secretid, secretkey, appid));
+  //build cos_base
+  const ObString cos_storage_info(account);
+  ASSERT_EQ(OB_SUCCESS, cos_base.set(ObStorageType::OB_STORAGE_COS, cos_storage_info.ptr()));
+}
+void destory()
+{
+  // fin_cos_env();
+}
+protected:
+  char account[OB_MAX_URI_LENGTH];
+  const char *dir_name = "test_backup_io_adapter_access_cos_dir";
+  char uri[OB_MAX_URI_LENGTH];
+  char dir_uri[OB_MAX_URI_LENGTH];
+  oceanbase::share::ObBackupStorageInfo cos_base;
+
+  int object_prefix_len = 5;
+};
+
+class TestBackupIOAdapterAccessCos : public ::testing::Test, public TestStorageCosCommon
+{
+public:
+  TestBackupIOAdapterAccessCos() : enable_test_(enable_test) {}
+  virtual ~TestBackupIOAdapterAccessCos() {}
+  virtual void SetUp()
+  {
+    init();
+  }
+  virtual void TearDown()
+  {
+    destory();
+  }
+
+  static void SetUpTestCase()
+  {
+  }
+
+  static void TearDownTestCase()
+  {
+  }
+
+private:
+  // disallow copy
+  DISALLOW_COPY_AND_ASSIGN(TestBackupIOAdapterAccessCos);
+protected:
+  bool enable_test_;
+};
+
+TEST_F(TestBackupIOAdapterAccessCos, test_basic_rw)
+{
+  int ret = OB_SUCCESS;
+  if (enable_test_) {
+    ObBackupIoAdapter adapter;
+    const char *tmp_dir = "test_basic_rw";
+    const int64_t ts = ObTimeUtility::current_time();
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld",
+        bucket, dir_name, tmp_dir, ts));
+
+    // write
+    const char *write_content = "123456789ABCDEF";
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/0", dir_uri));
+    ASSERT_EQ(OB_SUCCESS,
+        adapter.write_single_file(uri, &cos_base, write_content, strlen(write_content)));
+
+    // read
+    char read_buf[100] = {0};
+    int64_t read_size = 0;
+    ASSERT_EQ(OB_SUCCESS,
+        adapter.read_single_file(uri, &cos_base, read_buf, sizeof(read_buf), read_size));
+    ASSERT_STREQ(write_content, read_buf);
+    ASSERT_EQ(strlen(write_content), read_size);
+
+    ASSERT_EQ(OB_SUCCESS,
+        adapter.read_part_file(uri, &cos_base, read_buf, sizeof(read_buf), 0, read_size));
+    ASSERT_STREQ(write_content, read_buf);
+    ASSERT_EQ(strlen(write_content), read_size);
+
+    int64_t offset = 5;
+    ASSERT_EQ(OB_SUCCESS,
+        adapter.read_part_file(uri, &cos_base, read_buf, sizeof(read_buf), offset, read_size));
+    ASSERT_EQ('6', read_buf[0]);
+    ASSERT_EQ('F', read_buf[9]);
+    ASSERT_EQ(strlen(write_content) - offset, read_size);
+
+    offset = 6;
+    ASSERT_EQ(OB_SUCCESS,
+        adapter.read_part_file(uri, &cos_base, read_buf, 5, offset, read_size));
+    ASSERT_EQ('7', read_buf[0]);
+    ASSERT_EQ('B', read_buf[4]);
+    ASSERT_EQ(5, read_size);
+
+    offset = strlen(write_content);
+    ASSERT_EQ(OB_COS_ERROR,
+        adapter.read_part_file(uri, &cos_base, read_buf, sizeof(read_buf), offset, read_size));
+
+    ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &cos_base));
+  }
+}
+
+TEST_F(TestBackupIOAdapterAccessCos, test_util)
+{
+  int ret = OB_SUCCESS;
+  if (enable_test_) {
+    ObBackupIoAdapter adapter;
+    const char *tmp_dir = "test_util";
+    const int64_t ts = ObTimeUtility::current_time();
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld",
+        bucket, dir_name, tmp_dir, ts));
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/0", dir_uri));
+
+    bool is_obj_exist = true;
+    ASSERT_EQ(OB_SUCCESS, adapter.is_exist(uri, &cos_base, is_obj_exist));
+    ASSERT_FALSE(is_obj_exist);
+
+    const char *write_content = "123456789ABCDEF";
+    ASSERT_EQ(OB_SUCCESS,
+        adapter.write_single_file(uri, &cos_base, write_content, strlen(write_content)));
+    ASSERT_EQ(OB_SUCCESS, adapter.is_exist(uri, &cos_base, is_obj_exist));
+    ASSERT_TRUE(is_obj_exist);
+
+    int64_t file_length = -1;
+    ASSERT_EQ(OB_SUCCESS, adapter.get_file_length(uri, &cos_base, file_length));
+    ASSERT_EQ(strlen(write_content), file_length);
+
+    ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &cos_base));
+    ASSERT_EQ(OB_SUCCESS, adapter.is_exist(uri, &cos_base, is_obj_exist));
+    ASSERT_FALSE(is_obj_exist);
+  }
+}
+
+TEST_F(TestBackupIOAdapterAccessCos, test_list_files)
+{
+  int ret = OB_SUCCESS;
+  if (enable_test_) {
+    ObBackupIoAdapter adapter;
+    const char *tmp_dir = "test_list_files";
+    const int64_t ts = ObTimeUtility::current_time();
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld",
+        bucket, dir_name, tmp_dir, ts));
+
+    int64_t file_num = 1001;
+    const char *write_content = "0123456789";
+    const char *format = "%s/%0*ld_%ld";
+    for (int64_t file_idx = 0; file_idx < file_num; file_idx++) {
+      ASSERT_EQ(OB_SUCCESS,
+          databuff_printf(uri, sizeof(uri), format, dir_uri, object_prefix_len, file_idx, file_idx));
+      ASSERT_EQ(OB_SUCCESS,
+          adapter.write_single_file(uri, &cos_base, write_content, strlen(write_content)));
+    }
+
+    ObArenaAllocator allocator;
+    ObArray<ObString> name_array;
+    ObFileListArrayOp op(name_array, allocator);
+    ASSERT_EQ(OB_SUCCESS, adapter.list_files(dir_uri, &cos_base, op));
+    ASSERT_EQ(file_num, name_array.size());
+    for (int64_t file_idx = 0; file_idx < file_num; file_idx++) {
+      // listed files do not contain prefix
+      ASSERT_EQ(OB_SUCCESS,
+          databuff_printf(uri, sizeof(uri), "%0*ld_%ld", object_prefix_len, file_idx, file_idx));
+      ASSERT_STREQ(uri, name_array[file_idx].ptr());
+      ASSERT_EQ(OB_SUCCESS,
+          databuff_printf(uri, sizeof(uri), format, dir_uri, object_prefix_len, file_idx, file_idx));
+      ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &cos_base));
+    }
+  }
+}
+
+TEST_F(TestBackupIOAdapterAccessCos, test_list_directories)
+{
+  int ret = OB_SUCCESS;
+  if (enable_test_) {
+    ObBackupIoAdapter adapter;
+    const char *tmp_dir = "test_list_directories";
+    const int64_t ts = ObTimeUtility::current_time();
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld",
+        bucket, dir_name, tmp_dir, ts));
+
+    int64_t file_num = 1001;
+    const char *write_content = "0123456789";
+    const char *format = "%s/%0*ld/%ld";
+    for (int64_t file_idx = 0; file_idx < file_num; file_idx++) {
+      ASSERT_EQ(OB_SUCCESS,
+          databuff_printf(uri, sizeof(uri), format, dir_uri, object_prefix_len, file_idx, file_idx));
+      ASSERT_EQ(OB_SUCCESS,
+          adapter.write_single_file(uri, &cos_base, write_content, strlen(write_content)));
+    }
+
+    ObArenaAllocator allocator;
+    ObArray<ObString> name_array;
+    ObFileListArrayOp op(name_array, allocator);
+    ASSERT_EQ(OB_SUCCESS, adapter.list_directories(dir_uri, &cos_base, op));
+    ASSERT_EQ(file_num, name_array.size());
+    for (int64_t file_idx = 0; file_idx < file_num; file_idx++) {
+      // listed files do not contain prefix
+      ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%0*ld", object_prefix_len, file_idx));
+      ASSERT_STREQ(uri, name_array[file_idx].ptr());
+      ASSERT_EQ(OB_SUCCESS,
+          databuff_printf(uri, sizeof(uri), format, dir_uri, object_prefix_len, file_idx, file_idx));
+      ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &cos_base));
+    }
+  }
+}
+
+TEST_F(TestBackupIOAdapterAccessCos, test_is_tagging)
+{
+  int ret = OB_SUCCESS;
+  if (enable_test_) {
+    ObBackupIoAdapter adapter;
+    const char *tmp_util_dir = "test_util_is_tagging";
+    const int64_t ts = ObTimeUtility::current_time();
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld",
+      bucket, dir_name, tmp_util_dir, ts));
+
+    bool is_tagging = true;
+    char tmp_account[OB_MAX_URI_LENGTH];
+    oceanbase::share::ObBackupStorageInfo tmp_cos_base;
+    const char *write_content = "123456789ABCDEF";
+
+    // wrong tag mode
+    ASSERT_EQ(OB_SUCCESS,
+              databuff_printf(tmp_account, sizeof(tmp_account),
+              "host=%s&access_id=%s&access_key=%s&appid=%s&delete_mode=tag",
+              endpoint, secretid, secretkey, appid));
+    ASSERT_EQ(OB_INVALID_ARGUMENT, tmp_cos_base.set(ObStorageType::OB_STORAGE_COS, tmp_account));
+    tmp_cos_base.reset();
+
+    ASSERT_EQ(OB_SUCCESS,
+              databuff_printf(tmp_account, sizeof(tmp_account),
+              "host=%s&access_id=%s&access_key=%s&appid=%s&delete_mode=delete_delete",
+              endpoint, secretid, secretkey, appid));
+    ASSERT_EQ(OB_INVALID_ARGUMENT, tmp_cos_base.set(ObStorageType::OB_STORAGE_COS, tmp_account));
+    tmp_cos_base.reset();
+
+    // delete mode
+    ASSERT_EQ(OB_SUCCESS,
+              databuff_printf(tmp_account, sizeof(tmp_account),
+              "host=%s&access_id=%s&access_key=%s&appid=%s&delete_mode=delete",
+              endpoint, secretid, secretkey, appid));
+    ASSERT_EQ(OB_SUCCESS, tmp_cos_base.set(ObStorageType::OB_STORAGE_COS, tmp_account));
+
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/delete_mode", dir_uri));
+    ASSERT_EQ(OB_SUCCESS,
+        adapter.write_single_file(uri, &tmp_cos_base, write_content, strlen(write_content)));
+    ASSERT_EQ(OB_SUCCESS, adapter.is_tagging(uri, &tmp_cos_base, is_tagging));
+    ASSERT_FALSE(is_tagging);
+
+    ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &tmp_cos_base));
+    ASSERT_EQ(OB_BACKUP_FILE_NOT_EXIST, adapter.is_tagging(uri, &tmp_cos_base, is_tagging));
+    tmp_cos_base.reset();
+
+    // tagging mode
+    ASSERT_EQ(OB_SUCCESS,
+              databuff_printf(tmp_account, sizeof(tmp_account),
+              "host=%s&access_id=%s&access_key=%s&appid=%s&delete_mode=tagging",
+              endpoint, secretid, secretkey, appid));
+    ASSERT_EQ(OB_SUCCESS, tmp_cos_base.set(ObStorageType::OB_STORAGE_COS, tmp_account));
+
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/tagging_mode", dir_uri));
+    ASSERT_EQ(OB_SUCCESS,
+        adapter.write_single_file(uri, &tmp_cos_base, write_content, strlen(write_content)));
+
+    is_tagging = true;
+    ASSERT_EQ(OB_SUCCESS, adapter.is_tagging(uri, &tmp_cos_base, is_tagging));
+    ASSERT_FALSE(is_tagging);
+
+    ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &tmp_cos_base));
+    ASSERT_EQ(OB_SUCCESS, adapter.is_tagging(uri, &tmp_cos_base, is_tagging));
+    ASSERT_TRUE(is_tagging);
+    tmp_cos_base.reset();
+
+    // clean
+    ASSERT_EQ(OB_SUCCESS,
+              databuff_printf(tmp_account, sizeof(tmp_account),
+              "host=%s&access_id=%s&access_key=%s&appid=%s",
+              endpoint, secretid, secretkey, appid));
+    ASSERT_EQ(OB_SUCCESS, tmp_cos_base.set(ObStorageType::OB_STORAGE_COS, tmp_account));
+
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/tagging_mode", dir_uri));
+    ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &tmp_cos_base));
+    ASSERT_EQ(OB_BACKUP_FILE_NOT_EXIST, adapter.is_tagging(uri, &tmp_cos_base, is_tagging));
+  }
+}
+
+int main(int argc, char **argv)
+{
+  system("rm -f test_backup_access_cos.log*");
+  OB_LOGGER.set_file_name("test_backup_access_cos.log", true, true);
+  OB_LOGGER.set_log_level("INFO");
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

unittest/share/backup/test_backup_access_cos.h

@@ -0,0 +1,23 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef UNITTEST_BACKUP_ACCESS_COS_H_
+#define UNITTEST_BACKUP_ACCESS_COS_H_
+
+const bool enable_test = false;
+const char *bucket = "cos://xxx";
+const char *endpoint = "xxx";
+const char *secretid = "xxx";
+const char *secretkey = "xxx";
+const char *appid = "xxx";
+
+#endif

unittest/share/backup/test_backup_struct.cpp

@@ -52,6 +52,9 @@ TEST(ObBackupDest, nfs)
   const char *backup_test = "file:///backup_dir/?&delete_mode=tagging";
   ObBackupDest dest;
   ObBackupDest dest1;
+  ASSERT_EQ(OB_INVALID_BACKUP_DEST, dest.set(backup_test));
+  backup_test = "file:///backup_dir/";
+
   char backup_dest_str[OB_MAX_BACKUP_DEST_LENGTH] = { 0 };
   char backup_path_str[OB_MAX_BACKUP_DEST_LENGTH] = { 0 };
   ASSERT_EQ(OB_SUCCESS, dest.set(backup_test));
@@ -144,10 +147,79 @@ TEST(ObBackupDest, oss_encrypt)
   ObMasterKeyGetter::instance().wait();
   ObMasterKeyGetter::instance().reset();
 }
-#endif //
+
+TEST(ObBackupDest, cos)
+{
+  const char *backup_test = "cos://backup_dir/?host=xxx.com&access_id=111&access_key=222&delete_mode=tagging&appid=333";
+  ObBackupDest dest;
+  ObBackupDest dest1;
+  char backup_dest_str[OB_MAX_BACKUP_DEST_LENGTH] = { 0 };
+  char backup_path_str[OB_MAX_BACKUP_DEST_LENGTH] = { 0 };
+  ASSERT_EQ(OB_SUCCESS, dest.set(backup_test));
+  ASSERT_EQ(OB_SUCCESS, dest1.set(backup_test));
+  LOG_INFO("dump backup dest", K(dest), K(dest.get_root_path()), K(*(dest.get_storage_info())));
+  ASSERT_EQ(0, strcmp(dest.root_path_, "cos://backup_dir"));
+  ASSERT_TRUE(dest.storage_info_->device_type_ == 2);
+
+  EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().init(NULL));
+  EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().set_root_key(OB_SYS_TENANT_ID,
+                                                        obrpc::RootKeyType::DEFAULT, ObString()));
+  ASSERT_EQ(OB_SUCCESS, dest.get_backup_dest_str(backup_dest_str, sizeof(backup_dest_str)));
+  ASSERT_EQ(0, strcmp(backup_dest_str, "cos://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F&delete_mode=tagging&appid=333"));
+  ASSERT_EQ(OB_SUCCESS, dest.get_backup_path_str(backup_path_str, sizeof(backup_path_str)));
+  ASSERT_EQ(0, strcmp(backup_path_str, "cos://backup_dir?host=xxx.com"));
+  ASSERT_TRUE(dest.is_root_path_equal(dest1));
+  bool is_equal = false;
+  ASSERT_EQ(OB_SUCCESS, dest.is_backup_path_equal(dest1, is_equal));
+  ASSERT_TRUE(is_equal);
+  ASSERT_TRUE(dest == dest1);
+  dest1.reset();
+  ASSERT_EQ(OB_SUCCESS, dest1.set(dest.get_root_path().ptr(), dest.get_storage_info()));
+  ASSERT_TRUE(dest == dest1);
+  ObMasterKeyGetter::instance().stop();
+  ObMasterKeyGetter::instance().wait();
+  ObMasterKeyGetter::instance().reset();
+}
+
+TEST(ObBackupDest, cos_encrypt)
+{
+  const char *backup_test = "cos://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F&appid=333";
+  ObBackupDest dest;
+  EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().init(NULL));
+  EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().set_root_key(OB_SYS_TENANT_ID,
+                                                        obrpc::RootKeyType::DEFAULT, ObString()));
+  ASSERT_EQ(OB_SUCCESS, dest.set(backup_test));
+  LOG_INFO("dump backup dest", K(dest.get_root_path()), K(*(dest.get_storage_info())));
+  ASSERT_EQ(0, strcmp(dest.root_path_, "cos://backup_dir"));
+  ASSERT_TRUE(dest.storage_info_->device_type_ == 2);
+  const char *path = "cos://backup_dir/";
+  const char *endpoint = "host=xxx.com";
+  const char *authorization = "access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F";
+  const char *extension = "appid=333";
+  ObBackupDest dest1;
+  ASSERT_EQ(OB_SUCCESS, dest1.set(path, endpoint, authorization, extension));
+  ASSERT_TRUE(dest == dest1);
+
+  char backup_dest_str[OB_MAX_BACKUP_DEST_LENGTH] = { 0 };
+  char backup_path_str[OB_MAX_BACKUP_DEST_LENGTH] = { 0 };
+  ASSERT_EQ(OB_SUCCESS, dest.get_backup_dest_str(backup_dest_str, sizeof(backup_dest_str)));
+  ASSERT_EQ(0, strcmp(backup_dest_str, "cos://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F&appid=333"));
+  ASSERT_EQ(OB_SUCCESS, dest.get_backup_path_str(backup_path_str, sizeof(backup_path_str)));
+  ASSERT_EQ(0, strcmp(backup_path_str, "cos://backup_dir?host=xxx.com"));
+
+  dest1.reset();
+  ASSERT_EQ(OB_SUCCESS, dest1.set(path, endpoint, authorization, extension));
+  ASSERT_TRUE(dest == dest1);
+  ObMasterKeyGetter::instance().stop();
+  ObMasterKeyGetter::instance().wait();
+  ObMasterKeyGetter::instance().reset();
+}
+#endif
 
 int main(int argc, char **argv)
 {
+  system("rm -f test_backup_struct.log*");
+  OB_LOGGER.set_file_name("test_backup_struct.log", true, true);
   OB_LOGGER.set_log_level("INFO");
   testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();