[FEAT MERGE] Backup support AWS S3

Co-authored-by: xuhuleon <xuhuleon@qq.com>

deps/init/oceanbase.el7.aarch64.deps

@@ -28,6 +28,7 @@ devdeps-boost-1.74.0-22022110914.el7.aarch64.rpm
 devdeps-s2geometry-0.9.0-12023092021.el7.aarch64.rpm
 devdeps-icu-69.1-72022112416.el7.aarch64.rpm
 devdeps-cos-c-sdk-5.0.16-52023070517.el7.aarch64.rpm
+devdeps-s3-cpp-sdk-1.11.156-62023101011.el7.aarch64.rpm
 
 [tools]
 obdevtools-binutils-2.30-12022100413.el7.aarch64.rpm

deps/init/oceanbase.el7.x86_64.deps

@@ -30,6 +30,7 @@ devdeps-boost-1.74.0-22022110914.el7.x86_64.rpm
 devdeps-s2geometry-0.9.0-12023092021.el7.x86_64.rpm
 devdeps-icu-69.1-72022112416.el7.x86_64.rpm
 devdeps-cos-c-sdk-5.0.16-52023070517.el7.x86_64.rpm
+devdeps-s3-cpp-sdk-1.11.156-62023101011.el7.x86_64.rpm
 
 [tools]
 obdevtools-binutils-2.30-12022100413.el7.x86_64.rpm

deps/init/oceanbase.el8.aarch64.deps

@@ -28,6 +28,7 @@ devdeps-boost-1.74.0-22022110914.el8.aarch64.rpm
 devdeps-s2geometry-0.9.0-12023092021.el8.aarch64.rpm
 devdeps-icu-69.1-72022112416.el8.aarch64.rpm
 devdeps-cos-c-sdk-5.0.16-52023070517.el8.aarch64.rpm
+devdeps-s3-cpp-sdk-1.11.156-62023101011.el8.aarch64.rpm
 
 [tools]
 obdevtools-binutils-2.30-12022100413.el8.aarch64.rpm

deps/init/oceanbase.el8.x86_64.deps

@@ -29,6 +29,7 @@ devdeps-boost-1.74.0-22022110914.el8.x86_64.rpm
 devdeps-s2geometry-0.9.0-12023092021.el8.x86_64.rpm
 devdeps-icu-69.1-72022112416.el8.x86_64.rpm
 devdeps-cos-c-sdk-5.0.16-52023070517.el8.x86_64.rpm
+devdeps-s3-cpp-sdk-1.11.156-62023101011.el8.x86_64.rpm
 
 [tools]
 obdevtools-binutils-2.30-12022100413.el8.x86_64.rpm

deps/init/oceanbase.el9.aarch64.deps

@@ -32,6 +32,7 @@ devdeps-boost-1.74.0-22022110914.el8.aarch64.rpm
 devdeps-s2geometry-0.9.0-12023092021.el8.aarch64.rpm
 devdeps-icu-69.1-72022112416.el8.aarch64.rpm
 devdeps-cos-c-sdk-5.0.16-52023070517.el8.aarch64.rpm
+devdeps-s3-cpp-sdk-1.11.156-62023101011.el9.aarch64.rpm
 
 [deps-apr-el9]
 devdeps-apr-1.6.5-232023090616.el9.aarch64.rpm target=apr-el9

deps/init/oceanbase.el9.x86_64.deps

@@ -33,6 +33,7 @@ devdeps-boost-1.74.0-22022110914.el8.x86_64.rpm
 devdeps-s2geometry-0.9.0-12023092021.el8.x86_64.rpm
 devdeps-icu-69.1-72022112416.el8.x86_64.rpm
 devdeps-cos-c-sdk-5.0.16-52023070517.el8.x86_64.rpm
+devdeps-s3-cpp-sdk-1.11.156-62023101011.el9.x86_64.rpm
 
 [deps-apr-el9]
 devdeps-apr-1.6.5-232023090616.el9.x86_64.rpm target=apr-el9

deps/oblib/src/common/storage/ob_device_common.h

@@ -51,15 +51,18 @@ enum ObStorageType
   OB_STORAGE_FILE = 1,
   OB_STORAGE_COS = 2,
   OB_STORAGE_LOCAL = 3,
+  OB_STORAGE_S3 = 4,
   OB_STORAGE_MAX_TYPE
 };
 
 enum ObStorageAccessType
 {
   OB_STORAGE_ACCESS_READER = 0,
-  OB_STORAGE_ACCESS_OVERWRITER = 1,
-  OB_STORAGE_ACCESS_APPENDER = 2,
-  OB_STORAGE_ACCESS_RANDOMWRITER = 3,
+  OB_STORAGE_ACCESS_ADAPTIVE_READER = 1,
+  OB_STORAGE_ACCESS_OVERWRITER = 2,
+  OB_STORAGE_ACCESS_APPENDER = 3,
+  OB_STORAGE_ACCESS_RANDOMWRITER = 4,
+  OB_STORAGE_ACCESS_MULTIPART_WRITER = 5,
   OB_STORAGE_ACCESS_MAX_TYPE
 };
 

deps/oblib/src/common/storage/ob_io_device.h

@@ -351,6 +351,11 @@ public:
   virtual int exist(const char *pathname, bool &is_exist) = 0;
   virtual int stat(const char *pathname, ObIODFileStat &statbuf) = 0;
   virtual int fstat(const ObIOFd &fd, ObIODFileStat &statbuf) = 0;
+  virtual int del_unmerged_parts(const char *pathname) = 0;
+  virtual int adaptive_exist(const char *pathname, bool &is_exist) = 0;
+  virtual int adaptive_stat(const char *pathname, ObIODFileStat &statbuf) = 0;
+  virtual int adaptive_unlink(const char *pathname) = 0;
+  virtual int adaptive_scan_dir(const char *dir_name, ObBaseDirEntryOperator &op) = 0;
 
   //block interfaces
   virtual int mark_blocks(ObIBlockIterator &block_iter) = 0;

deps/oblib/src/lib/allocator/ob_malloc.cpp

@@ -53,36 +53,17 @@ int oceanbase::common::ObMemBuf::ensure_space(const int64_t size, const lib::ObL
 void *oceanbase::common::ob_malloc_align(const int64_t alignment, const int64_t nbyte,
                                          const lib::ObLabel &label)
 {
-  char *ptr = static_cast<char *>(oceanbase::common::ob_malloc(nbyte + alignment, label));
-  char *align_ptr = NULL;
-  if (NULL != ptr) {
-    align_ptr = reinterpret_cast<char *>(oceanbase::common::upper_align(reinterpret_cast<int64_t>(ptr),
-                                                                        alignment));
-    if (align_ptr == ptr) {
-      align_ptr = ptr + alignment;
-    }
-    int64_t padding = align_ptr - ptr;
-    if (!(padding <= alignment && padding > 0)) {
-      _OB_LOG_RET(ERROR, OB_INVALID_ARGUMENT, "invalid padding(padding=%ld, alignment=%ld", padding, alignment);
-    }
-    uint8_t *sign_ptr = reinterpret_cast<uint8_t *>(align_ptr - 1);
-    int64_t *header_ptr = reinterpret_cast<int64_t *>(align_ptr - 1 - sizeof(int64_t));
-    if (padding < (int64_t)sizeof(int64_t) + 1) {
-      *sign_ptr = static_cast<uint8_t>(padding) & 0x7f;
-    } else {
-      *sign_ptr = 0x80;
-      *header_ptr = padding;
-    }
-  } else {
-    _OB_LOG_RET(WARN, OB_ALLOCATE_MEMORY_FAILED, "ob_tc_malloc allocate memory failed, alignment[%ld], nbyte[%ld], label[%s].",
-            alignment, nbyte, (const char *)label);
-  }
-  return align_ptr;
+  ObMemAttr attr;
+  attr.label_ = label;
+  return ob_malloc_align(alignment, nbyte, attr);
 }
 
-void *oceanbase::common::ob_malloc_align(const int64_t alignment, const int64_t nbyte,
+void *oceanbase::common::ob_malloc_align(const int64_t align, const int64_t nbyte,
                                          const ObMemAttr &attr)
 {
+  const int min_align = 16;
+  const int64_t alignment =
+    align <= min_align ? min_align : align_up2(align, 16);
   char *ptr = static_cast<char *>(oceanbase::common::ob_malloc(nbyte + alignment, attr));
   char *align_ptr = NULL;
   if (NULL != ptr) {

deps/oblib/src/lib/ob_define.h

@@ -695,6 +695,11 @@ const char *const OB_LOCAL_PREFIX = "local://";
 const char *const OB_OSS_PREFIX = "oss://";
 const char *const OB_FILE_PREFIX = "file://";
 const char *const OB_COS_PREFIX = "cos://";
+const char *const OB_S3_PREFIX = "s3://";
+const char *const OB_S3_APPENDABLE_FORMAT_META = "FORMAT_META";
+const char *const OB_S3_APPENDABLE_SEAL_META = "SEAL_META";
+const char *const OB_S3_APPENDABLE_FRAGMENT_PREFIX = "@APD_PART@";
+const int64_t OB_STORAGE_LIST_MAX_NUM = 1000;
 const char *const OB_RESOURCE_UNIT_DEFINITION = "resource_unit_definition";
 const char *const OB_RESOURCE_POOL_DEFINITION = "resource_pool_definition";
 const char *const OB_CREATE_TENANT_DEFINITION = "create_tenant_definition";
@@ -2523,7 +2528,7 @@ inline bool is_x86() {
 #endif
 }
 #define __maybe_unused  __attribute__((unused))
-#define DO_PRAGMA(x) _Pragma (#x)
+#define DO_PRAGMA(x) _Pragma(#x)
 #define DISABLE_WARNING_GCC_PUSH _Pragma("GCC diagnostic push")
 #define DISABLE_WARNING_GCC(option) DO_PRAGMA(GCC diagnostic ignored option)
 #define DISABLE_WARNING_GCC_POP _Pragma("GCC diagnostic pop")

deps/oblib/src/lib/ob_errno.h

@@ -397,6 +397,7 @@ constexpr int OB_BACKUP_PWRITE_OFFSET_NOT_MATCH = -9083;
 constexpr int OB_BACKUP_PWRITE_CONTENT_NOT_MATCH = -9084;
 constexpr int OB_CLOUD_OBJECT_NOT_APPENDABLE = -9098;
 constexpr int OB_RESTORE_TENANT_FAILED = -9099;
+constexpr int OB_S3_ERROR = -9105;
 constexpr int OB_ERR_XML_PARSE = -9549;
 constexpr int OB_ERR_XSLT_PARSE = -9574;
 constexpr int OB_MAX_RAISE_APPLICATION_ERROR         = -20000;

deps/oblib/src/lib/restore/CMakeLists.txt

@@ -1,5 +1,6 @@
 oblib_add_library(restore OBJECT
   ob_i_storage.h
+  ob_i_storage.cpp
   ob_storage.cpp
   ob_storage_file.cpp
   ob_storage_file.h
@@ -34,7 +35,28 @@ target_link_libraries(oss
   ${DEP_DIR}/lib/libapr-1.a
   ${DEP_DIR}/lib/libmxml.a)
 
+oblib_add_library(s3 ob_storage_s3_base.cpp)
+target_link_directories(s3 PUBLIC ${DEP_3RD_DIR}/usr/local/oceanbase/deps/devel/lib64)
+target_link_libraries(s3
+  PUBLIC
+  libaws-cpp-sdk-s3.a
+  libaws-cpp-sdk-core.a
+  libaws-crt-cpp.a
+  libaws-c-mqtt.a
+  libaws-c-event-stream.a
+  libaws-c-s3.a
+  libaws-c-auth.a
+  libaws-c-http.a
+  libaws-c-io.a
+  libs2n.a
+  libaws-c-compression.a
+  libaws-c-cal.a
+  libaws-c-sdkutils.a
+  libaws-checksums.a
+  libaws-c-common.a
+)
+target_link_libraries(s3 PUBLIC oblib_base_base_base)
 
-target_link_libraries(restore PUBLIC oss cos_sdk oblib_base)
+target_link_libraries(restore PUBLIC oss cos_sdk oblib_base s3)
 
 add_subdirectory(cos)

deps/oblib/src/lib/restore/cos/ob_cos_wrapper.h

@@ -132,6 +132,30 @@ struct OB_PUBLIC_API CosStringBuffer
     }
     return match;
   }
+
+  int32_t get_data_size() const
+  {
+    return (!empty() && '\0' == data_[size_ - 1]) ? size_ - 1 : size_;
+  }
+
+  bool is_prefix_of(const char *str, const int32_t str_len) const
+  {
+    bool match = false;
+    const int32_t data_size = get_data_size();
+    if (str == data_) {
+      match = (data_size <= str_len) ? true : false;
+    } else if (data_size > str_len) {
+      match = false;
+    } else if (0 == memcmp(data_, str, data_size)) {
+      match = true;
+    }
+    return match;
+  }
+
+  bool is_end_with_slash_and_null() const
+  {
+    return (NULL != data_ && size_ >= 2 && data_[size_ - 1] == '\0' && data_[size_ - 2] == '/');
+  }
 };
 
 
@@ -263,29 +287,40 @@ public:
     const CosStringBuffer &object_name,
     bool &is_tagging);
 
-  struct CosListObjPara {
+  struct CosListObjPara
+  {
     enum class CosListType
     {
       COS_LIST_INVALID,
-      COS_LIST_CB_ARG
+      COS_LIST_CB_ARG,
+      COS_PART_LIST_CTX
     };
 
     CosListObjPara()
-      : arg_(NULL), cur_full_path_slice_name_(NULL),
-        full_path_size_(0), cur_object_size_(0),
-        next_flag_(false), type_(CosListType::COS_LIST_INVALID)
+      : arg_(NULL), cur_obj_full_path_(NULL),
+        full_path_size_(0), cur_object_size_str_(NULL),
+        next_flag_(false), type_(CosListType::COS_LIST_INVALID),
+        next_token_(NULL), next_token_size_(0), finish_part_list_(false)
     {
       last_container_name_.d_name[0] = '\0';
       last_container_name_.d_type = DT_REG;
     }
 
-    void* arg_;
-    char* cur_full_path_slice_name_;
+    int set_cur_obj_meta(
+        char *obj_full_path,
+        const int64_t full_path_size,
+        char *object_size_str);
+
+    void *arg_;
+    char *cur_obj_full_path_;
     struct dirent last_container_name_;
     int64_t full_path_size_;
-    int64_t cur_object_size_;
+    char *cur_object_size_str_;
     bool next_flag_;
     CosListType type_;
+    char *next_token_;
+    int64_t next_token_size_;
+    bool finish_part_list_;
   };
 
   typedef int (*handleObjectNameFunc)(CosListObjPara&);
@@ -294,6 +329,14 @@ public:
   // in the inner sub directories.
   // dir_name must be end with "/\0".
   static int list_objects(
+    Handle *h,
+    const CosStringBuffer &bucket_name,
+    const CosStringBuffer &dir_name,
+    handleObjectNameFunc handle_object_name_f,
+    void *arg);
+
+  // Only list up-to 1000 objects
+  static int list_part_objects(
     Handle *h,
     const CosStringBuffer &bucket_name,
     const CosStringBuffer &dir_name,
@@ -315,8 +358,39 @@ public:
     const CosStringBuffer &bucket_name,
     const CosStringBuffer &dir_name,
     bool &is_empty_dir);
-};
 
+  static int init_multipart_upload(
+    Handle *h,
+    const CosStringBuffer &bucket_name,
+    const CosStringBuffer &object_name,
+    char *&upload_id_str);
+
+  static int upload_part_from_buffer(
+    Handle *h,
+    const CosStringBuffer &bucket_name,
+    const CosStringBuffer &object_name,
+    const CosStringBuffer &upload_id_str,
+    const int part_num, /*the sequence number of this part, [1, 10000]*/
+    const char *buf,
+    const int64_t buf_size);
+
+  static int complete_multipart_upload(
+    Handle *h,
+    const CosStringBuffer &bucket_name,
+    const CosStringBuffer &object_name,
+    const CosStringBuffer &upload_id_str);
+
+  static int abort_multipart_upload(
+    Handle *h,
+    const CosStringBuffer &bucket_name,
+    const CosStringBuffer &object_name,
+    const CosStringBuffer &upload_id_str);
+
+  static int del_unmerged_parts(
+    Handle *h,
+    const CosStringBuffer &bucket_name,
+    const CosStringBuffer &object_name);
+};
 
 #undef OB_PUBLIC_API
 }

deps/oblib/src/lib/restore/cos/ob_cos_wrapper_handle.cpp

@@ -231,5 +231,15 @@ int ObCosWrapperHandle::set_delete_mode(const char *parameter)
   return ret;
 }
 
+void* ObCosWrapperHandle::alloc_mem(size_t size)
+{
+  return allocator_.alloc(size);
+}
+
+void ObCosWrapperHandle::free_mem(void *addr)
+{
+  allocator_.free(addr);
+}
+
 } // common
 } // oceanbase

deps/oblib/src/lib/restore/cos/ob_cos_wrapper_handle.h

@@ -61,6 +61,9 @@ public:
   int set_delete_mode(const char *parameter);
   int64_t get_delete_mode() const { return delete_mode_; }
 
+  void *alloc_mem(size_t size);
+  void free_mem(void *addr);
+
 private:
   bool is_inited_;
   qcloud_cos::ObCosWrapper::Handle *handle_;

deps/oblib/src/lib/restore/ob_i_storage.cpp

@@ -0,0 +1,600 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include "ob_i_storage.h"
+#include "lib/container/ob_se_array_iterator.h"
+
+namespace oceanbase
+{
+namespace common
+{
+
+static const char SLASH = '/';
+bool is_end_with_slash(const char *str)
+{
+  bool bret = false;
+  int64_t str_len = -1;
+  if (OB_NOT_NULL(str) && (str_len = strlen(str)) > 0) {
+    bret = (SLASH == str[str_len - 1]);
+  }
+  return bret;
+}
+
+int c_str_to_int(const char *str, int64_t &num)
+{
+  int ret = OB_SUCCESS;
+  errno = 0;
+  char *end_str = NULL;
+  if (OB_ISNULL(str) || OB_UNLIKELY(0 == strlen(str))) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "c_str_to_int str should not be null/empty", KP(str));
+  } else {
+    num = strtoll(str, &end_str, 10);
+    if (errno != 0 || (NULL != end_str && *end_str != '\0')) {
+      ret = OB_INVALID_DATA;
+      OB_LOG(WARN, "strtoll convert string to int value fail", K(str), K(num),
+          "error", strerror(errno), K(end_str));
+    }
+  }
+  return ret;
+}
+
+int handle_listed_object(ObBaseDirEntryOperator &op,
+    const char *obj_name, const int64_t obj_name_len, const int64_t obj_size)
+{
+  int ret = OB_SUCCESS;
+  dirent entry;
+  entry.d_type = DT_REG;
+  if (OB_ISNULL(obj_name)
+      || OB_UNLIKELY(sizeof(entry.d_name) <= obj_name_len || obj_name_len <= 0)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid arguments",
+        K(ret), K(obj_name), K(obj_name_len), K(sizeof(entry.d_name)));
+  } else {
+    if (op.need_get_file_size()) {
+      if (OB_UNLIKELY(obj_size < 0)) {
+        ret = OB_INVALID_ARGUMENT;
+        OB_LOG(WARN, "invalid object size", K(obj_size));
+      } else {
+        op.set_size(obj_size);
+      }
+    }
+
+    if (OB_SUCC(ret)) {
+      MEMCPY(entry.d_name, obj_name, obj_name_len);
+      entry.d_name[obj_name_len] = '\0';
+      if (OB_FAIL(op.func(&entry))) {
+        OB_LOG(WARN, "fail to exe application callback for listed object",
+            K(ret), K(obj_name), K(obj_name_len), K(obj_size));
+      }
+    }
+  }
+  return ret;
+}
+
+int handle_listed_directory(ObBaseDirEntryOperator &op,
+    const char *dir_name, const int64_t dir_name_len)
+{
+  int ret = OB_SUCCESS;
+  dirent entry;
+  entry.d_type = DT_DIR;
+  if (OB_ISNULL(dir_name)
+      || OB_UNLIKELY(sizeof(entry.d_name) <= dir_name_len || dir_name_len <= 0)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid arguments",
+        K(ret), K(dir_name), K(dir_name_len), K(sizeof(entry.d_name)));
+  } else {
+    MEMCPY(entry.d_name, dir_name, dir_name_len);
+    entry.d_name[dir_name_len] = '\0';
+    if (OB_FAIL(op.func(&entry))) {
+      OB_LOG(WARN, "fail to exe application callback for listed directory",
+          K(ret), K(dir_name), K(dir_name_len));
+    }
+  }
+  return ret;
+}
+
+static int get_storage_prefix_from_path(const common::ObString &uri, const char *&prefix)
+{
+  int ret = OB_SUCCESS;
+  if (uri.prefix_match(OB_OSS_PREFIX)) {
+    prefix = OB_OSS_PREFIX;
+  } else if (uri.prefix_match(OB_COS_PREFIX)) {
+    prefix = OB_COS_PREFIX;
+  } else if (uri.prefix_match(OB_S3_PREFIX)) {
+    prefix = OB_S3_PREFIX;
+  } else if (uri.prefix_match(OB_FILE_PREFIX)) {
+    prefix = OB_FILE_PREFIX;
+  } else {
+    ret = OB_INVALID_BACKUP_DEST;
+    STORAGE_LOG(ERROR, "invalid backup uri", K(ret), K(uri));
+  }
+  return ret;
+}
+
+int build_bucket_and_object_name(ObIAllocator &allocator,
+    const ObString &uri, ObString &bucket, ObString &object)
+{
+  int ret = OB_SUCCESS;
+  ObString::obstr_size_t bucket_start = 0;
+  ObString::obstr_size_t bucket_end = 0;
+  ObString::obstr_size_t object_start = 0;
+  char *bucket_name_buff = NULL;
+  char *object_name_buff = NULL;
+
+  const char *prefix = "UNKNOWN";
+  if (OB_FAIL(get_storage_prefix_from_path(uri, prefix))) {
+    OB_LOG(WARN, "fail to get storage type", K(ret), K(uri));
+  } else {
+    bucket_start = static_cast<ObString::obstr_size_t>(strlen(prefix));
+    if (OB_UNLIKELY(bucket_start >= uri.length())) {
+      ret = OB_INVALID_ARGUMENT;
+      OB_LOG(WARN, "bucket name is empty", K(uri), K(ret), K(bucket_start));
+    }
+    for (int64_t i = bucket_start; OB_SUCC(ret) && i < uri.length() - 1; i++) {
+      if ('/' == *(uri.ptr() + i) && '/' == *(uri.ptr() + i + 1)) {
+        ret = OB_INVALID_ARGUMENT;
+        OB_LOG(WARN, "uri has two // ", K(uri), K(ret), K(i));
+        break;
+      }
+    }
+  }
+
+  if (OB_FAIL(ret)) {
+  } else if (prefix == OB_FILE_PREFIX) {
+    // for nfs, bucket is empty
+    if (OB_FAIL(ob_write_string(allocator, uri.ptr() + bucket_start, object, true/*c_style*/))) {
+      OB_LOG(WARN, "fail to deep copy object", K(uri), K(bucket_start), K(ret));
+    }
+  } else {
+    for (bucket_end = bucket_start; OB_SUCC(ret) && bucket_end < uri.length(); ++bucket_end) {
+      if ('/' == *(uri.ptr() + bucket_end)) {
+        ObString::obstr_size_t bucket_length = bucket_end - bucket_start;
+        //must end with '\0'
+        if (OB_UNLIKELY(bucket_length <= 0)) {
+          ret = OB_INVALID_ARGUMENT;
+          OB_LOG(WARN, "bucket name is empty", K(ret), K(uri), K(bucket_start), K(bucket_length));
+        } else if (OB_UNLIKELY(bucket_end + 1 >= uri.length())) {
+          ret = OB_INVALID_ARGUMENT;
+          OB_LOG(WARN, "object name is empty", K(uri), K(ret), K(bucket_end));
+        }  else if (OB_ISNULL(bucket_name_buff =
+            static_cast<char *>(allocator.alloc(bucket_length + 1)))) {
+          OB_LOG(WARN, "failed to alloc bucket name buff", K(ret), K(uri), K(bucket_length));
+        } else if (OB_FAIL(databuff_printf(bucket_name_buff, bucket_length + 1,
+                                           "%.*s", bucket_length, uri.ptr() + bucket_start))) {
+          OB_LOG(WARN, "fail to deep copy bucket", K(uri), K(bucket_start), K(bucket_length), K(ret));
+        } else {
+          bucket.assign_ptr(bucket_name_buff, bucket_length + 1);// must include '\0'
+        }
+        break;
+      }
+    }
+
+    // parse the object name
+    if (OB_SUCC(ret)) {
+      object_start = bucket_end + 1;
+      ObString::obstr_size_t object_length = uri.length() - object_start;
+      //must end with '\0'
+      if (OB_UNLIKELY(object_length <= 0)) {
+        ret = OB_INVALID_ARGUMENT;
+        OB_LOG(WARN, "bucket name is empty", K(ret), K(uri), K(object_start), K(object_length));
+      } else if (OB_FAIL(ob_write_string(allocator, uri.ptr() + object_start, object, true/*c_style*/))) {
+        OB_LOG(WARN, "fail to deep copy object", K(uri), K(object_start), K(object_length), K(ret));
+      }
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    OB_LOG(DEBUG, "get bucket object name", K(uri), K(bucket), K(object));
+  }
+  return ret;
+}
+
+int construct_fragment_full_name(const ObString &logical_appendable_object_name,
+    const char *fragment_name, char *name_buf, const int64_t name_buf_len)
+{
+  int ret = OB_SUCCESS;
+  int64_t pos = 0;
+  const char *suffix = NULL;
+  if (OB_ISNULL(fragment_name) || OB_ISNULL(name_buf) || OB_UNLIKELY(strlen(fragment_name) <= 0)
+      || OB_UNLIKELY(logical_appendable_object_name.empty() || name_buf_len <= 0)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid arguments", K(ret),
+        K(logical_appendable_object_name), KP(fragment_name), KP(name_buf), K(name_buf_len));
+  } else if (OB_FAIL(databuff_printf(name_buf, name_buf_len, pos, "%s/%s%s",
+                                     logical_appendable_object_name.ptr(),
+                                     OB_S3_APPENDABLE_FRAGMENT_PREFIX, fragment_name))) {
+    OB_LOG(WARN, "failed to construct formatted mock append object fragment name",
+        K(ret), K(logical_appendable_object_name), K(fragment_name));
+  } else {
+    suffix = logical_appendable_object_name.reverse_find('.');
+    if (OB_NOT_NULL(suffix)) {
+      if (OB_UNLIKELY(strlen(suffix) <= 1 || strlen(suffix) >= MAX_APPENDABLE_FRAGMENT_SUFFIX_LENGTH)) {
+        ret = OB_INVALID_ARGUMENT;
+        OB_LOG(WARN, "object name has invalid suffix",
+            K(ret), K(logical_appendable_object_name), K(suffix));
+      } else if (OB_FAIL(databuff_printf(name_buf, name_buf_len, pos, "%s", suffix))) {
+        OB_LOG(WARN, "failed to set formatted mock append object fragment suffix",
+            K(ret), K(logical_appendable_object_name), K(fragment_name), K(suffix));
+      }
+    }
+  }
+  return ret;
+}
+
+int construct_fragment_full_name(const ObString &logical_appendable_object_name,
+    const int64_t start, const int64_t end, char *name_buf, const int64_t name_buf_len)
+{
+  int ret = OB_SUCCESS;
+  char fragment_name[MAX_APPENDABLE_FRAGMENT_LENGTH] = { 0 };
+  if (OB_UNLIKELY(start < 0 || end <= start)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid arguments", K(ret), K(start), K(end));
+  } else if (OB_FAIL(databuff_printf(fragment_name, sizeof(fragment_name), "%ld-%ld", start, end))) {
+    OB_LOG(WARN, "failed to construct mock append object fragment name", K(ret), K(start), K(end));
+  } else if (OB_FAIL(construct_fragment_full_name(logical_appendable_object_name,
+                                                  fragment_name, name_buf, name_buf_len))) {
+    OB_LOG(WARN, "failed to construct mock append object fragment name",
+        K(ret), K(start), K(end), K(fragment_name), K(logical_appendable_object_name));
+  }
+  return ret;
+}
+
+/*--------------------------------ObAppendableFragmentMeta--------------------------------*/
+OB_SERIALIZE_MEMBER(ObAppendableFragmentMeta, start_, end_);
+
+int ObAppendableFragmentMeta::assign(const ObAppendableFragmentMeta &other)
+{
+  int ret = OB_SUCCESS;
+  start_ = other.start_;
+  end_ = other.end_;
+  type_ = other.type_;
+  MEMCPY(suffix_, other.suffix_, sizeof(suffix_));
+  return ret;
+}
+
+int ObAppendableFragmentMeta::parse_from(ObString &fragment_name)
+{
+  int ret = OB_SUCCESS;
+  if (OB_UNLIKELY(!fragment_name.prefix_match(OB_S3_APPENDABLE_FRAGMENT_PREFIX))) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid fragment prefix", K(ret), K(fragment_name));
+  } else {
+    fragment_name += strlen(OB_S3_APPENDABLE_FRAGMENT_PREFIX);
+    const char *fragment_suffix = fragment_name.reverse_find('.');
+    fragment_name.clip(fragment_suffix);
+    if (OB_NOT_NULL(fragment_suffix)) {
+      if (strlen(fragment_suffix) <= 1 || strlen(fragment_suffix) >= MAX_APPENDABLE_FRAGMENT_SUFFIX_LENGTH) {
+        ret = OB_INVALID_ARGUMENT;
+        OB_LOG(WARN, "invalid fragment suffix", K(ret), K(fragment_suffix));
+      } else {
+        STRCPY(suffix_, fragment_suffix);
+      }
+    }
+  }
+
+  if (OB_FAIL(ret)) {
+  } else if (0 == fragment_name.compare(OB_S3_APPENDABLE_FORMAT_META)) {
+    type_ = ObAppendableFragmentType::APPENDABLE_FRAGMENT_FORMAT_META;
+  } else if (0 == fragment_name.compare(OB_S3_APPENDABLE_SEAL_META)) {
+    type_ = ObAppendableFragmentType::APPENDABLE_FRAGMENT_SEAL_META;
+  } else {
+    ObArenaAllocator allocator(ObModIds::BACKUP);
+    ObString start_part = fragment_name.split_on('-');
+    ObString start_string;
+    ObString end_string;
+    if (OB_UNLIKELY(!start_part.is_numeric() || !fragment_name.is_numeric())) {
+      ret = OB_INVALID_ARGUMENT;
+      OB_LOG(WARN, "unexpected fragment name", K(start_part), K(fragment_name));
+    } else if (OB_FAIL(ob_write_string(allocator, start_part, start_string, true))) {
+      OB_LOG(WARN, "fail to deep copy start part of fragment name",
+          K(ret), K(start_string), K(fragment_name), K_(suffix));
+    } else if (OB_FAIL(ob_write_string(allocator, fragment_name, end_string, true))) {
+      OB_LOG(WARN, "fail to deep copy end part of fragment name",
+          K(ret), K(start_string), K(fragment_name), K_(suffix));
+    } else if (OB_FAIL(c_str_to_int(start_string.ptr(), start_))) {
+      OB_LOG(WARN, "fail to parse 'start'", K(ret), K(start_string), K(fragment_name));
+    } else if (OB_FAIL(c_str_to_int(end_string.ptr(), end_))) {
+      OB_LOG(WARN, "fail to parse 'end'", K(ret), K(end_string), K(fragment_name));
+    } else {
+      type_ = ObAppendableFragmentType::APPENDABLE_FRAGMENT_DATA;
+      if (OB_UNLIKELY(!is_valid())) {
+        ret = OB_INVALID_ARGUMENT;
+        OB_LOG(WARN, "invalid fragment name", K(ret), K_(type), K(start_string), K(end_string),
+          K(fragment_name), K_(start), K_(end), K_(suffix));
+      }
+    }
+  }
+  return ret;
+}
+
+int64_t ObAppendableFragmentMeta::to_string(char *buf, const int64_t len) const
+{
+  int64_t pos = 0;
+  if (OB_NOT_NULL(buf) && OB_LIKELY(len > 0)) {
+    if (type_ == ObAppendableFragmentType::APPENDABLE_FRAGMENT_DATA) {
+      pos = snprintf(buf, len, "%ld-%ld%s", start_, end_, suffix_);
+    } else {
+      const char *meta_name = (type_ == ObAppendableFragmentType::APPENDABLE_FRAGMENT_FORMAT_META) ?
+                               OB_S3_APPENDABLE_FORMAT_META :
+                               OB_S3_APPENDABLE_SEAL_META;
+      pos = snprintf(buf, len, "%s%s%s", OB_S3_APPENDABLE_FRAGMENT_PREFIX, meta_name, suffix_);
+    }
+
+    if (pos < 0) {
+      pos = 0;
+    } else if (pos >= len) {
+      pos = len - 1;
+    }
+  }
+  return pos;
+}
+
+/*--------------------------------ObStorageObjectMeta--------------------------------*/
+OB_SERIALIZE_MEMBER(ObStorageObjectMetaBase, length_);
+OB_SERIALIZE_MEMBER((ObStorageObjectMeta, ObStorageObjectMetaBase), type_, fragment_metas_);
+
+void ObStorageObjectMeta::reset()
+{
+  // reset do not change obj type
+  ObStorageObjectMetaBase::reset();
+  fragment_metas_.reset();
+}
+
+bool ObStorageObjectMeta::is_valid() const
+{
+  bool is_valid_flag = (length_ >= 0);
+  if (is_simulate_append_type()) {
+    for (int64_t i = 0; is_valid_flag && i < fragment_metas_.count(); i++) {
+      is_valid_flag = fragment_metas_[i].is_valid();
+    }
+    if (is_valid_flag && fragment_metas_.count() > 1) {
+      for (int64_t i = 1; is_valid_flag && i < fragment_metas_.count(); i++) {
+        is_valid_flag = (fragment_metas_[i - 1].start_ < fragment_metas_[i].start_
+                && fragment_metas_[i - 1].end_ < fragment_metas_[i].end_);
+      }
+    }
+  } else {
+    // for normal objs, fragment_metas_ must be empty;
+    is_valid_flag &= fragment_metas_.empty();
+  }
+
+  return is_valid_flag;
+}
+
+bool ObStorageObjectMeta::fragment_meta_cmp_func(
+    const ObAppendableFragmentMeta &left,
+    const ObAppendableFragmentMeta &right)
+{
+  // for fragments with the same start offset, prioritize placing the largest fragment at the beginning,
+  // to facilitate subsequent cleaning of overlapping fragments
+  return left.start_ < right.start_ || (left.start_ == right.start_ && left.end_ > right.end_);
+}
+
+int ObStorageObjectMeta::get_needed_fragments(
+    const int64_t start,
+    const int64_t end,
+    ObArray<ObAppendableFragmentMeta> &fragments)
+{
+  int ret = OB_SUCCESS;
+  fragments.reset();
+  if (OB_UNLIKELY(start < 0 || end <= start)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid arguments", K(start), K(end));
+  } else if (OB_UNLIKELY(!is_simulate_append_type() || !is_valid())) {
+    ret = OB_ERR_UNEXPECTED;
+    OB_LOG(WARN, "invalid storage object meta", K(ret), K_(type), K_(fragment_metas));
+  } else if (OB_UNLIKELY(fragment_metas_.empty())) {
+    ret = OB_ERR_UNEXPECTED;
+    OB_LOG(WARN, "empty storage appendable object", K(ret));
+  } else if (fragment_metas_[fragment_metas_.count() - 1].end_ <= start) {
+    // the data to be read does not exist
+  } else {
+    int64_t cur_fragment_idx = -1;
+    ObAppendableFragmentMeta start_meta;
+    start_meta.start_ = start;
+    ObSEArray<ObAppendableFragmentMeta, 10>::iterator it =
+        std::upper_bound(fragment_metas_.begin(), fragment_metas_.end(), start_meta, fragment_meta_cmp_func);
+    if (it == fragment_metas_.begin()) {
+      ret = OB_ERR_UNEXPECTED;
+      OB_LOG(WARN, "the object data may contain holes, can't read correct data", K(ret), K(start), K(end),
+        K(fragment_metas_[0].start_));
+    } else if (FALSE_IT(cur_fragment_idx = it - fragment_metas_.begin() - 1)) {
+    } else {
+      int64_t last_fragment_end = fragment_metas_[cur_fragment_idx].start_;
+      while (OB_SUCC(ret) && cur_fragment_idx < fragment_metas_.count()
+             && fragment_metas_[cur_fragment_idx].start_ < end
+             && last_fragment_end < end) {
+        if (fragment_metas_[cur_fragment_idx].start_ > last_fragment_end) {
+          ret = OB_ERR_UNEXPECTED;
+          OB_LOG(WARN, "the object data may contain holes, can't read correct data", K(ret), K(start), K(end),
+            K(fragment_metas_[cur_fragment_idx]), K(last_fragment_end));
+        } else if (OB_FAIL(fragments.push_back(fragment_metas_[cur_fragment_idx]))) {
+          OB_LOG(WARN, "fail to push back fragement", K(ret), K(fragments));
+        } else {
+          last_fragment_end = fragment_metas_[cur_fragment_idx].end_;
+          cur_fragment_idx++;
+        }
+      }
+    }
+  }
+  return ret;
+}
+
+/*--------------------------------ObStorageListCtxBase--------------------------------*/
+int ObStorageListCtxBase::init(
+    ObArenaAllocator &allocator,
+    const int64_t max_list_num,
+    const bool need_size)
+{
+  int ret = OB_SUCCESS;
+  if (OB_UNLIKELY(max_list_num < 1)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid argument", K(ret), K(max_list_num));
+  } else {
+    max_list_num_ = max_list_num;
+    max_name_len_ = OB_MAX_URI_LENGTH;
+    need_size_ = need_size;
+    if (OB_ISNULL(name_arr_ = static_cast<char **>(allocator.alloc(sizeof(void *) * max_list_num_)))) {
+      ret = OB_ALLOCATE_MEMORY_FAILED;
+      OB_LOG(WARN, "fail to alloc name_arr buff", K(ret), K(*this));
+    } else {
+      for (int64_t i = 0; OB_SUCC(ret) && (i < max_list_num_); ++i) {
+        if (OB_ISNULL(name_arr_[i] = static_cast<char *>(allocator.alloc(max_name_len_)))) {
+          ret = OB_ALLOCATE_MEMORY_FAILED;
+          OB_LOG(WARN, "fail to alloc name buff", K(ret), K(i), K(*this));
+        } else {
+          name_arr_[i][0] = '\0';
+        }
+      }
+    }
+
+    if (OB_SUCC(ret) && need_size) {
+      if (OB_ISNULL(size_arr_ = static_cast<int64_t *>(allocator.alloc(sizeof(int64_t) * max_list_num_)))) {
+        ret = OB_ALLOCATE_MEMORY_FAILED;
+        OB_LOG(WARN, "fail to alloc size_arr buff", K(ret), K(*this));
+      }
+    }
+  }
+  return ret;
+}
+
+void ObStorageListCtxBase::reset()
+{
+  max_list_num_ = 0;
+  name_arr_ = NULL;
+  max_name_len_ = 0;
+  rsp_num_ = 0;
+  has_next_ = false;
+  need_size_ = false;
+  size_arr_ = NULL;
+}
+
+bool ObStorageListCtxBase::is_valid() const
+{
+  bool bret = (max_list_num_ > 0) && (name_arr_ != NULL) && (max_name_len_ > 0);
+  if (need_size_) {
+    bret &= (size_arr_ != NULL);
+  }
+  return bret;
+}
+
+/*--------------------------------ObStorageListObjectsCtx--------------------------------*/
+void ObStorageListObjectsCtx::reset()
+{
+  next_token_ = NULL;
+  next_token_buf_len_ = 0;
+  cur_appendable_full_obj_path_ = NULL;
+  ObStorageListCtxBase::reset();
+}
+
+int ObStorageListObjectsCtx::init(
+    ObArenaAllocator &allocator,
+    const int64_t max_list_num,
+    const bool need_size)
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(ObStorageListCtxBase::init(allocator, max_list_num, need_size))) {
+    OB_LOG(WARN, "fail to init storage_list_ctx_base", K(ret), K(max_list_num), K(need_size));
+  } else {
+    next_token_buf_len_ = OB_MAX_URI_LENGTH;
+    if (OB_ISNULL(next_token_ = static_cast<char *>(allocator.alloc(next_token_buf_len_)))) {
+      ret = OB_ALLOCATE_MEMORY_FAILED;
+      OB_LOG(WARN, "fail to alloc next_token buff", K(ret), K(*this));
+    } else {
+      next_token_[0] = '\0';
+
+      if (OB_ISNULL(cur_appendable_full_obj_path_ = static_cast<char *>(allocator.alloc(OB_MAX_URI_LENGTH)))) {
+        ret = OB_ALLOCATE_MEMORY_FAILED;
+        OB_LOG(WARN, "fail to alloc cur appendable full obj path buff", K(ret), K(*this));
+      } else {
+        cur_appendable_full_obj_path_[0] = '\0';
+      }
+    }
+  }
+  return ret;
+}
+
+int ObStorageListObjectsCtx::set_next_token(
+    const bool has_next,
+    const char *next_token,
+    const int64_t next_token_len)
+{
+  int ret = OB_SUCCESS;
+  if (OB_UNLIKELY(!is_valid())) {
+    ret = OB_NOT_INIT;
+    OB_LOG(WARN, "ObStorageListObjectsCtx not init", K(ret));
+  } else {
+    has_next_ = has_next;
+    if (has_next) {
+      if (OB_ISNULL(next_token) || OB_UNLIKELY(next_token_len <= 0)) {
+        ret = OB_INVALID_ARGUMENT;
+        OB_LOG(WARN, "invalid arguments", K(ret), K(has_next), K(next_token), K(next_token_len));
+      } else if (OB_UNLIKELY(next_token_len >= next_token_buf_len_)) {
+        ret = OB_SIZE_OVERFLOW;
+        OB_LOG(WARN, "fail to set next token, size overflow",
+            K(ret), K(has_next), K(next_token_len));
+      } else {
+        MEMCPY(next_token_, next_token, next_token_len);
+        next_token_[next_token_len] = '\0';
+      }
+    } else {
+      next_token_[0] = '\0';
+    }
+  }
+  return ret;
+}
+
+int ObStorageListObjectsCtx::handle_object(
+    const char *obj_path,
+    const int obj_path_len,
+    const int64_t obj_size)
+{
+  int ret = OB_SUCCESS;
+  if (OB_UNLIKELY(!is_valid())) {
+    ret = OB_NOT_INIT;
+    OB_LOG(WARN, "ObStorageListObjectsCtx not init", K(ret));
+  } else if (OB_UNLIKELY(obj_size < 0 || obj_path_len >= max_name_len_
+                        || obj_path_len <= 0 || rsp_num_ >= max_list_num_)
+            || OB_ISNULL(obj_path) || OB_ISNULL(name_arr_[rsp_num_])) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid arguments", K(ret), K(obj_path), K(obj_path_len), K(obj_size), K(*this));
+  } else {
+    if (need_size_) {
+      size_arr_[rsp_num_] = obj_size;
+    }
+    MEMCPY(name_arr_[rsp_num_], obj_path, obj_path_len);
+    name_arr_[rsp_num_][obj_path_len] = '\0';
+    ++rsp_num_;
+  }
+  return ret;
+}
+
+/*--------------------------------ObStorageListFilesCtx--------------------------------*/
+bool ObStorageListFilesCtx::is_valid() const
+{
+  bool bret = ObStorageListCtxBase::is_valid();
+  if (already_open_dir_) {
+    bret &= (open_dir_ != NULL);
+  }
+  return bret;
+}
+
+void ObStorageListFilesCtx::reset()
+{
+  open_dir_ = NULL;
+  already_open_dir_ = false;
+  ObStorageListCtxBase::reset();
+}
+
+}//common
+}//oceanbase

deps/oblib/src/lib/restore/ob_i_storage.h

@@ -16,6 +16,7 @@
 #include "lib/ob_define.h"
 #include "lib/string/ob_string.h"
 #include "lib/container/ob_array.h"
+#include "lib/container/ob_se_array.h"
 #include "common/storage/ob_device_common.h"
 #include "ob_storage_info.h"
 
@@ -23,6 +24,11 @@ namespace oceanbase
 {
 namespace common
 {
+
+static constexpr int64_t MAX_APPENDABLE_FRAGMENT_SUFFIX_LENGTH = 64;
+static constexpr int64_t MAX_APPENDABLE_FRAGMENT_LENGTH = 128;
+static constexpr char APPENDABLE_OBJECT_ALLOCATOR[] = "AppendableAlloc";
+
 enum StorageOpenMode
 {
   CREATE_OPEN_LOCK = 0, // default, create and open
@@ -31,6 +37,201 @@ enum StorageOpenMode
   CREATE_OPEN_NOLOCK = 3, // create and open nolock
 };
 
+enum ObStorageCRCAlgorithm
+{
+  OB_INVALID_CRC_ALGO = 0,
+  OB_CRC32_ALGO = 1,
+};
+
+enum ObStorageObjectType
+{
+  OB_OBJ_INVALID = 0,
+  OB_OBJ_NORMAL = 1,
+  OB_OBJ_SIMULATE_APPEND = 2,
+  OB_FS_DIR = 3,
+  OB_FS_FILE = 4,
+};
+
+// check the str is end with '/' or not
+bool is_end_with_slash(const char *str);
+int c_str_to_int(const char *str, int64_t &num);
+int handle_listed_object(ObBaseDirEntryOperator &op,
+    const char *obj_name, const int64_t obj_name_len, const int64_t obj_size);
+int handle_listed_directory(ObBaseDirEntryOperator &op,
+    const char *dir_name, const int64_t dir_name_len);
+int build_bucket_and_object_name(ObIAllocator &allocator,
+    const ObString &uri, ObString &bucket, ObString &object);
+int construct_fragment_full_name(const ObString &logical_appendable_object_name,
+    const char *fragment_name, char *name_buf, const int64_t name_buf_len);
+int construct_fragment_full_name(const ObString &logical_appendable_object_name,
+    const int64_t start, const int64_t end, char *name_buf, const int64_t name_buf_len);
+
+struct ObStorageObjectMetaBase
+{
+  OB_UNIS_VERSION_V(1);
+public:
+  ObStorageObjectMetaBase() : type_(ObStorageObjectType::OB_OBJ_INVALID) { reset(); }
+  ~ObStorageObjectMetaBase() { reset(); }
+
+  void reset() { is_exist_ = false; length_ = -1; }
+
+  TO_STRING_KV(K_(is_exist), K_(length));
+
+  bool is_exist_;
+  int64_t length_;
+  ObStorageObjectType type_;
+};
+
+// Each fragment meta corresponds to a normal object in a 'dir'.
+// The 'dir' name is the S3 appendable object name.
+// Fragment name format: /xxx/xxx/appendable_obj_name/prefix-start-end[-suffix]
+// 'prefix' is a special string which represents this object is a S3 appendable object fragment.
+// 'start-end' means the data range covered by this file. [start, end), include start、not include end.
+// 'suffix' may exist, mainly used by deleting file situation.
+struct ObAppendableFragmentMeta
+{
+  OB_UNIS_VERSION_V(1);
+public:
+  enum ObAppendableFragmentType
+  {
+    APPENDABLE_FRAGMENT_DATA = 0,
+    APPENDABLE_FRAGMENT_FORMAT_META = 1,
+    APPENDABLE_FRAGMENT_SEAL_META = 2,
+  };
+
+  ObAppendableFragmentMeta()
+    : start_(-1), end_(-1), type_(ObAppendableFragmentType::APPENDABLE_FRAGMENT_DATA) {
+    suffix_[0] = '\0';
+  }
+  virtual ~ObAppendableFragmentMeta() {}
+
+  bool is_format_meta() const { return type_ == ObAppendableFragmentType::APPENDABLE_FRAGMENT_FORMAT_META; }
+  bool is_seal_meta() const { return type_ == ObAppendableFragmentType::APPENDABLE_FRAGMENT_SEAL_META; }
+  bool is_data() const { return type_ == ObAppendableFragmentType::APPENDABLE_FRAGMENT_DATA; }
+  bool is_valid() const
+  {
+    return (is_format_meta()) || (is_seal_meta()) || (is_data() && start_ >= 0 && end_ > start_);
+  }
+
+  int assign(const ObAppendableFragmentMeta &other);
+  int parse_from(ObString &fragment_name);
+  int64_t to_string(char *buf, const int64_t len) const;
+
+  int64_t get_length() const { return end_ - start_; }
+
+  int64_t start_;
+  int64_t end_;
+  char suffix_[MAX_APPENDABLE_FRAGMENT_SUFFIX_LENGTH];
+  ObAppendableFragmentType type_;
+};
+
+struct ObStorageObjectMeta : public ObStorageObjectMetaBase
+{
+  OB_UNIS_VERSION_V(1);
+public:
+  ObStorageObjectMeta()
+    : ObStorageObjectMetaBase(),
+      fragment_metas_()
+  {}
+
+  ~ObStorageObjectMeta() { reset(); }
+  void reset();
+
+  bool is_valid() const;
+  // Based on the range[start, end), to choose the needed files and save these meta into @fragments.
+  int get_needed_fragments(const int64_t start, const int64_t end,
+      ObArray<ObAppendableFragmentMeta> &fragments);
+
+  bool is_object_file_type() const
+  {
+    return (type_ == ObStorageObjectType::OB_OBJ_NORMAL) ||
+           (type_ == ObStorageObjectType::OB_FS_FILE);
+  }
+  bool is_simulate_append_type() const { return type_ == ObStorageObjectType::OB_OBJ_SIMULATE_APPEND; }
+
+  static bool fragment_meta_cmp_func(const ObAppendableFragmentMeta &left, const ObAppendableFragmentMeta &right);
+
+  TO_STRING_KV(K_(is_exist), K_(length), K_(type), K_(fragment_metas));
+
+  ObSEArray<ObAppendableFragmentMeta, 10> fragment_metas_;
+};
+
+struct ObStorageListCtxBase
+{
+public:
+  int64_t max_list_num_; // each round list, can only get up-to @max_list_num_ items.
+  char **name_arr_; // for object storage, save full path; for file system, save file name.
+  int64_t max_name_len_; // no matter full path, or just object/file name, can not be longer than this value.
+  int64_t rsp_num_; // real listed-item number which is obtained from the listed result
+  bool has_next_; // list result can only return up-to 1000 objects once, thus may need to multi operation.
+  bool need_size_; // If true, that means when we list items, we also need to get each item's size
+  int64_t *size_arr_; // save all the length of each object/file (the order is the same with name_arr)
+
+  ObStorageListCtxBase()
+    : max_list_num_(0), name_arr_(NULL), max_name_len_(0), rsp_num_(0),
+      has_next_(false), need_size_(false), size_arr_(NULL)
+  {}
+
+  virtual ~ObStorageListCtxBase() { reset(); }
+
+  int init(ObArenaAllocator &allocator, const int64_t max_list_num, const bool need_size);
+
+  void reset();
+
+  bool is_valid() const;
+
+  TO_STRING_KV(K_(max_list_num), K_(max_name_len), K_(rsp_num), K_(has_next), K_(need_size),
+    KP_(name_arr), KP_(size_arr));
+};
+
+// Used for object storage
+struct ObStorageListObjectsCtx : public ObStorageListCtxBase
+{
+public:
+  char *next_token_; // save marker/continuation_token
+  int64_t next_token_buf_len_; // length of marker/continuation_token should not be longer than this value
+  char *cur_appendable_full_obj_path_;
+
+  ObStorageListObjectsCtx()
+    : next_token_(NULL), next_token_buf_len_(0), cur_appendable_full_obj_path_(NULL)
+  {}
+
+  virtual ~ObStorageListObjectsCtx() { reset(); }
+
+  void reset();
+
+  int init(ObArenaAllocator &allocator, const int64_t max_list_num, const bool need_size);
+
+  bool is_valid() const { return ObStorageListCtxBase::is_valid() && (next_token_ != NULL)
+                                 && (next_token_buf_len_ > 0); }
+  int set_next_token(const bool has_next, const char *next_token, const int64_t next_token_len);
+  int handle_object(const char *obj_path, const int obj_path_len, const int64_t obj_size);
+
+  INHERIT_TO_STRING_KV("ObStorageListCtxBase", ObStorageListCtxBase,
+      K_(next_token), K_(next_token_buf_len), K_(cur_appendable_full_obj_path));
+};
+
+// Used for file system
+struct ObStorageListFilesCtx : public ObStorageListCtxBase
+{
+public:
+  DIR *open_dir_;
+  struct dirent next_entry_; // If has_next=true, it will get the next entry based on this value.
+  bool already_open_dir_; // only during the first round, need to open dir
+
+  ObStorageListFilesCtx()
+    : open_dir_(NULL), next_entry_(), already_open_dir_(false)
+  {}
+
+  virtual ~ObStorageListFilesCtx() { reset(); }
+
+  void reset();
+
+  bool is_valid() const;
+
+  INHERIT_TO_STRING_KV("ObStorageListCtxBase", ObStorageListCtxBase, K_(already_open_dir));
+};
+
 class ObIStorageUtil
 {
 public:
@@ -44,20 +245,27 @@ public:
   virtual void close() = 0;
   virtual int is_exist(const common::ObString &uri, bool &exist) = 0;
   virtual int get_file_length(const common::ObString &uri, int64_t &file_length) = 0;
+  virtual int head_object_meta(const common::ObString &uri, ObStorageObjectMetaBase &obj_meta) = 0;
   virtual int del_file(const common::ObString &uri) = 0;
   virtual int write_single_file(const common::ObString &uri, const char *buf, const int64_t size) = 0;
   virtual int mkdir(const common::ObString &uri) = 0;
+  // list all objects which are 'prefix-matched'
   virtual int list_files(const common::ObString &dir_path, common::ObBaseDirEntryOperator &op) = 0;
+  // If we want to get the listed result and handle it in other logic, we can use this interface.
+  // @list_ctx will save the listed result(cuz obj_storage can only return up-to 1000 items at a time).
+  virtual int list_files(const common::ObString &dir_path, ObStorageListCtxBase &list_ctx) = 0;
   virtual int del_dir(const common::ObString &uri) = 0;
   virtual int list_directories(const common::ObString &dir_path, common::ObBaseDirEntryOperator &op) = 0;
   virtual int is_tagging(const common::ObString &uri, bool &is_tagging) = 0;
+  virtual int del_unmerged_parts(const common::ObString &uri) = 0;
 };
 
 class ObIStorageReader
 {
 public:
-  virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info) = 0;
-  virtual int pread(char *buf,const int64_t buf_size, int64_t offset, int64_t &read_size) = 0;
+  virtual int open(const common::ObString &uri,
+                   common::ObObjectStorageInfo *storage_info, const bool head_meta = true) = 0;
+  virtual int pread(char *buf,const int64_t buf_size, const int64_t offset, int64_t &read_size) = 0;
   virtual int close() = 0;
   virtual int64_t get_length() const = 0;
   virtual bool is_opened() const = 0;

deps/oblib/src/lib/restore/ob_object_device.cpp

@@ -17,15 +17,18 @@ namespace oceanbase
 namespace common
 {
 
-const char *OB_STORAGE_ACCESS_TYPES_STR[] = {"reader", "overwriter", "appender", "random_write"};
+const char *OB_STORAGE_ACCESS_TYPES_STR[] = {"reader", "adaptive_reader", "overwriter",
+                                             "appender", "random_write", "multipart_writer"};
 
 ObObjectDevice::ObObjectDevice()
   : storage_info_(), is_started_(false), lock_(common::ObLatchIds::OBJECT_DEVICE_LOCK)
 {
   ObMemAttr attr = SET_USE_500("ObjectDevice");
   reader_ctx_pool_.set_attr(attr);
+  adaptive_reader_ctx_pool_.set_attr(attr);
   appender_ctx_pool_.set_attr(attr);
   overwriter_ctx_pool_.set_attr(attr);
+  multipart_writer_ctx_pool_.set_attr(attr);
 }
 
 int ObObjectDevice::init(const ObIODOpts &opts)
@@ -41,7 +44,9 @@ void ObObjectDevice::destroy()
   if (is_started_) {
     appender_ctx_pool_.reset();
     reader_ctx_pool_.reset();
+    adaptive_reader_ctx_pool_.reset();
     overwriter_ctx_pool_.reset();
+    multipart_writer_ctx_pool_.reset();
     //close the util
     util_.close();
     //baseinfo will be free with allocator
@@ -131,12 +136,16 @@ int ObObjectDevice::get_access_type(ObIODOpts *opts, ObStorageAccessType& access
     OB_LOG(WARN, "can not find access type!");
   } else if (0 == STRCMP(access_type , OB_STORAGE_ACCESS_TYPES_STR[OB_STORAGE_ACCESS_READER])) {
     access_type_flag = OB_STORAGE_ACCESS_READER;
+  } else if (0 == STRCMP(access_type , OB_STORAGE_ACCESS_TYPES_STR[OB_STORAGE_ACCESS_ADAPTIVE_READER])) {
+    access_type_flag = OB_STORAGE_ACCESS_ADAPTIVE_READER;
   } else if (0 == STRCMP(access_type , OB_STORAGE_ACCESS_TYPES_STR[OB_STORAGE_ACCESS_OVERWRITER])) {
     access_type_flag = OB_STORAGE_ACCESS_OVERWRITER;
   } else if (0 == STRCMP(access_type , OB_STORAGE_ACCESS_TYPES_STR[OB_STORAGE_ACCESS_APPENDER])) {
     access_type_flag = OB_STORAGE_ACCESS_APPENDER;
   } else if (0 == STRCMP(access_type , OB_STORAGE_ACCESS_TYPES_STR[OB_STORAGE_ACCESS_RANDOMWRITER])) {
     access_type_flag = OB_STORAGE_ACCESS_RANDOMWRITER;
+  } else if (0 == STRCMP(access_type , OB_STORAGE_ACCESS_TYPES_STR[OB_STORAGE_ACCESS_MULTIPART_WRITER])) {
+    access_type_flag = OB_STORAGE_ACCESS_MULTIPART_WRITER;
   } else {
     ret = OB_INVALID_ARGUMENT;
     OB_LOG(WARN, "invaild access type!", KCSTRING(access_type));
@@ -165,6 +174,27 @@ int ObObjectDevice::open_for_reader(const char *pathname, void*& ctx)
   return ret;
 }
 
+int ObObjectDevice::open_for_adaptive_reader_(const char *pathname, void *&ctx)
+{
+  int ret = OB_SUCCESS;
+  ObStorageAdaptiveReader *adaptive_reader = adaptive_reader_ctx_pool_.alloc();
+  if (OB_ISNULL(adaptive_reader)) {
+    ret = OB_ALLOCATE_MEMORY_FAILED;
+    OB_LOG(WARN, "fail to alloc mem for object device adaptive_reader! ", K(ret), K(pathname));
+  } else {
+    if (OB_FAIL(adaptive_reader->open(pathname, &storage_info_))) {
+      OB_LOG(WARN, "fail to open for read!", K(ret), K(pathname), K_(storage_info));
+    } else {
+      ctx = (void*)adaptive_reader;
+    }
+  }
+  if (OB_FAIL(ret) && OB_NOT_NULL(adaptive_reader)) {
+    adaptive_reader_ctx_pool_.free(adaptive_reader);
+    adaptive_reader = nullptr;
+  }
+  return ret;
+}
+
 /*ObStorageOssMultiPartWriter is not used int the current version, if we use, later, the open func of
   overwriter maybe need to add para(just like the open func of appender)*/
 int ObObjectDevice::open_for_overwriter(const char *pathname, void*& ctx)
@@ -232,6 +262,27 @@ int ObObjectDevice::open_for_appender(const char *pathname, ObIODOpts *opts, voi
   return ret;
 }
 
+int ObObjectDevice::open_for_multipart_writer_(const char *pathname, void *&ctx)
+{
+  int ret = OB_SUCCESS;
+  ObStorageMultiPartWriter *multipart_writer = multipart_writer_ctx_pool_.alloc();
+  if (OB_ISNULL(multipart_writer)) {
+    ret = OB_ALLOCATE_MEMORY_FAILED;
+    OB_LOG(WARN, "fail to alloc multipart_writer!", K(ret), K(pathname));
+  } else {
+    if (OB_FAIL(multipart_writer->open(pathname, &storage_info_))) {
+      OB_LOG(WARN, "fail to open for multipart_writer!", K(ret), K(pathname), K_(storage_info));
+    } else {
+      ctx = (void*)multipart_writer;
+    }
+  }
+  if (OB_FAIL(ret) && OB_NOT_NULL(multipart_writer)) {
+    multipart_writer_ctx_pool_.free(multipart_writer);
+    multipart_writer = nullptr;
+  }
+  return ret;
+}
+
 int ObObjectDevice::release_res(void* ctx, const ObIOFd &fd, ObStorageAccessType access_type)
 {
   int ret = OB_SUCCESS;
@@ -254,12 +305,24 @@ int ObObjectDevice::release_res(void* ctx, const ObIOFd &fd, ObStorageAccessType
         OB_LOG(WARN, "fail to close the reader!", K(ret), K(access_type));
       }
       reader_ctx_pool_.free(reader);
+    } else if (OB_STORAGE_ACCESS_ADAPTIVE_READER == access_type) {
+      ObStorageAdaptiveReader *adaptive_reader = static_cast<ObStorageAdaptiveReader*>(ctx);
+      if (OB_FAIL(adaptive_reader->close())) {
+        OB_LOG(WARN, "fail to close the adaptive reader!", K(ret), K(access_type));
+      }
+      adaptive_reader_ctx_pool_.free(adaptive_reader);
     } else if (OB_STORAGE_ACCESS_OVERWRITER == access_type) {
       ObStorageWriter *overwriter = static_cast<ObStorageWriter*>(ctx);
       if (OB_FAIL(overwriter->close())) {
         OB_LOG(WARN, "fail to close the overwriter!", K(ret), K(access_type));
       }
       overwriter_ctx_pool_.free(overwriter);
+    } else if (OB_STORAGE_ACCESS_MULTIPART_WRITER == access_type) {
+      ObStorageMultiPartWriter *multipart_writer = static_cast<ObStorageMultiPartWriter*>(ctx);
+      if (OB_FAIL(multipart_writer->close())) {
+        OB_LOG(WARN, "fail to close the multipart writer!", K(ret), K(access_type));
+      }
+      multipart_writer_ctx_pool_.free(multipart_writer);
     } else {
       ret = OB_INVALID_ARGUMENT;
       OB_LOG(WARN, "invalid access_type!", K(access_type), K(ret));
@@ -303,11 +366,15 @@ int ObObjectDevice::open(const char *pathname, const int flags, const mode_t mod
     } else {
       if (OB_STORAGE_ACCESS_READER == access_type) {
         ret = open_for_reader(pathname, ctx);
+      } else if (OB_STORAGE_ACCESS_ADAPTIVE_READER == access_type) {
+        ret = open_for_adaptive_reader_(pathname, ctx);
       } else if (OB_STORAGE_ACCESS_APPENDER == access_type ||
                 OB_STORAGE_ACCESS_RANDOMWRITER == access_type) {
         ret = open_for_appender(pathname, opts, ctx);
       } else if (OB_STORAGE_ACCESS_OVERWRITER == access_type) {
         ret = open_for_overwriter(pathname, ctx);
+      } else if (OB_STORAGE_ACCESS_MULTIPART_WRITER == access_type) {
+        ret = open_for_multipart_writer_(pathname, ctx);
       }
 
       if (OB_FAIL(ret)) {
@@ -342,7 +409,7 @@ int ObObjectDevice::close(const ObIOFd &fd)
 {
   int ret = OB_SUCCESS;
   int flag = -1;
-  void* ctx = NULL;
+  void *ctx = NULL;
 
   fd_mng_.get_fd_flag(fd, flag);
   if (!fd_mng_.validate_fd(fd, true)) {
@@ -356,6 +423,43 @@ int ObObjectDevice::close(const ObIOFd &fd)
   return ret;
 }
 
+int ObObjectDevice::seal_for_adaptive(const ObIOFd &fd)
+{
+  int ret = OB_SUCCESS;
+  int flag = -1;
+  void *ctx = NULL;
+
+  fd_mng_.get_fd_flag(fd, flag);
+  if (!fd_mng_.validate_fd(fd, true)) {
+    ret = OB_NOT_INIT;
+    OB_LOG(WARN, "fd is not init!", K(fd.first_id_), K(fd.second_id_));
+  } else if (OB_FAIL(fd_mng_.fd_to_ctx(fd, ctx))) {
+    OB_LOG(WARN, "fail to get ctx accroding fd!", K(ret));
+  } else if (OB_ISNULL(ctx)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "fd ctx is null!", K(flag), K(ret));
+  } else if (flag == OB_STORAGE_ACCESS_RANDOMWRITER || flag == OB_STORAGE_ACCESS_APPENDER) {
+    ObStorageAppender *appender = static_cast<ObStorageAppender*>(ctx);
+    if (OB_FAIL(appender->seal_for_adaptive())) {
+      OB_LOG(WARN, "fail to seal!", K(ret), K(flag));
+    }
+  } else {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "unknow access type, not a appender fd!", K(flag), K(ret));
+  }
+  return ret;
+}
+
+int ObObjectDevice::del_unmerged_parts(const char *pathname)
+{
+  int ret = OB_SUCCESS;
+  common::ObString uri(pathname);
+  if (OB_FAIL(util_.del_unmerged_parts(uri))) {
+    OB_LOG(WARN, "fail to del unmerged parts", K(ret), K(uri));
+  }
+  return ret;
+}
+
 int ObObjectDevice::mkdir(const char *pathname, mode_t mode)
 {
   UNUSED(mode);
@@ -371,24 +475,66 @@ int ObObjectDevice::rmdir(const char *pathname)
 
 int ObObjectDevice::unlink(const char *pathname)
 {
+  return inner_unlink_(pathname, false/*is_adaptive*/);
+}
+
+int ObObjectDevice::adaptive_unlink(const char *pathname)
+{
+  const bool is_adaptive = true;
+  return inner_unlink_(pathname, is_adaptive);
+}
+
+int ObObjectDevice::inner_unlink_(const char *pathname, const bool is_adaptive)
+{
+  int ret = OB_SUCCESS;
   common::ObString uri(pathname);
-  return util_.del_file(uri);
+  if (OB_FAIL(util_.del_file(uri, is_adaptive))) {
+    OB_LOG(WARN, "fail to del file", K(ret), K(uri), K(is_adaptive));
+  }
+  return ret;
 }
 
 int ObObjectDevice::exist(const char *pathname, bool &is_exist)
 {
+  return inner_exist_(pathname, is_exist, false/*is_adaptive*/);
+}
+
+int ObObjectDevice::adaptive_exist(const char *pathname, bool &is_exist)
+{
+  const bool is_adaptive = true;
+  return inner_exist_(pathname, is_exist, is_adaptive);
+}
+
+int ObObjectDevice::inner_exist_(const char *pathname, bool &is_exist, const bool is_adaptive)
+{
+  int ret = OB_SUCCESS;
   common::ObString uri(pathname);
-  return util_.is_exist(uri, is_exist);
+  if (OB_FAIL(util_.is_exist(uri, is_adaptive, is_exist))) {
+    OB_LOG(WARN, "fail to check if the file exists", K(ret), K(uri), K(is_adaptive));
+  }
+  return ret;
 }
 
 /*notice: for backup, this interface only return size*/
 int ObObjectDevice::stat(const char *pathname, ObIODFileStat &statbuf)
+{
+  return inner_stat_(pathname, statbuf, false/*is_adaptive*/);
+}
+
+int ObObjectDevice::adaptive_stat(const char *pathname, ObIODFileStat &statbuf)
+{
+  const bool is_adaptive = true;
+  return inner_stat_(pathname, statbuf, is_adaptive);
+}
+
+int ObObjectDevice::inner_stat_(const char *pathname,
+    ObIODFileStat &statbuf, const bool is_adaptive)
 {
   int ret = OB_SUCCESS;
   int64_t length = 0;
   common::ObString uri(pathname);
-  if (OB_FAIL(util_.get_file_length(uri, length))) {
-    OB_LOG(WARN, "fail to get fail length!", K(ret));
+  if (OB_FAIL(util_.get_file_length(uri, is_adaptive, length))) {
+    OB_LOG(WARN, "fail to get file length!", K(ret), K(uri), K(is_adaptive));
   } else {
     statbuf.size_ = length;
   }
@@ -396,15 +542,27 @@ int ObObjectDevice::stat(const char *pathname, ObIODFileStat &statbuf)
 }
 
 int ObObjectDevice::scan_dir(const char *dir_name, common::ObBaseDirEntryOperator &op)
+{
+  return inner_scan_dir_(dir_name, op, false/*is_adaptive*/);
+}
+
+int ObObjectDevice::adaptive_scan_dir(const char *dir_name, ObBaseDirEntryOperator &op)
+{
+  const bool is_adaptive = true;
+  return inner_scan_dir_(dir_name, op, is_adaptive);
+}
+
+int ObObjectDevice::inner_scan_dir_(const char *dir_name,
+    ObBaseDirEntryOperator &op, const bool is_adaptive)
 {
   common::ObString uri(dir_name);
   int ret = OB_SUCCESS;
   bool is_dir_scan = false;
   if (op.is_dir_scan()) {
-    ret = util_.list_directories(uri, op);
+    ret = util_.list_directories(uri, is_adaptive, op);
     is_dir_scan = true;
   } else {
-    ret = util_.list_files(uri, op);
+    ret = util_.list_files(uri, is_adaptive, op);
   }
 
   if (OB_FAIL(ret)) {
@@ -425,25 +583,30 @@ int ObObjectDevice::pread(const ObIOFd &fd, const int64_t offset, const int64_t
   UNUSED(checker);
   int ret = OB_SUCCESS;
   int flag = -1;
-  void* ctx = NULL;
+  void *ctx = NULL;
 
   fd_mng_.get_fd_flag(fd, flag);
   if (!fd_mng_.validate_fd(fd, true)) {
     ret = OB_NOT_INIT;
     OB_LOG(WARN, "fd is not init!", K(fd.first_id_), K(fd.second_id_));
-  } else if (flag != OB_STORAGE_ACCESS_READER) {
-    ret = OB_INVALID_ARGUMENT;
-    OB_LOG(WARN, "fd is not a reader fd!", K(flag), K(ret));
   } else if (OB_FAIL(fd_mng_.fd_to_ctx(fd, ctx))) {
     OB_LOG(WARN, "fail to get ctx accroding fd!", K(ret));
-  } else {
+  } else if (OB_ISNULL(ctx)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "fd ctx is null!", K(flag), K(ret));
+  } else if (flag == OB_STORAGE_ACCESS_READER) {
     ObStorageReader *reader = static_cast<ObStorageReader*>(ctx);
-    if (OB_ISNULL(reader)) {
-      ret = OB_INVALID_ARGUMENT;
-      OB_LOG(WARN, "fd ctx is null!", K(flag), K(ret));
-    } else if (OB_FAIL(reader->pread((char*)buf, size, offset, read_size))) {
-      OB_LOG(WARN, "fail to pread!", K(ret));
+    if (OB_FAIL(reader->pread((char*)buf, size, offset, read_size))) {
+      OB_LOG(WARN, "fail to do normal pread!", K(ret), K(flag));
     }
+  } else if (flag == OB_STORAGE_ACCESS_ADAPTIVE_READER) {
+    ObStorageAdaptiveReader *adaptive_reader = static_cast<ObStorageAdaptiveReader*>(ctx);
+    if (OB_FAIL(adaptive_reader->pread((char*)buf, size, offset, read_size))) {
+      OB_LOG(WARN, "fail to do adaptive reader pread!", K(ret), K(flag));
+    }
+  } else {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "fd is not a reader fd!", K(flag), K(ret));
   }
   return ret;
 }
@@ -463,16 +626,24 @@ int ObObjectDevice::write(const ObIOFd &fd, const void *buf, const int64_t size,
     OB_LOG(WARN, "fd is not init!", K(fd.first_id_), K(fd.second_id_), K(ret));
   } else if (OB_FAIL(fd_mng_.fd_to_ctx(fd, ctx))) {
     OB_LOG(WARN, "fail to get ctx accroding fd!", K(ret));
+  } else if (OB_ISNULL(ctx)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "fd ctx is null!", K(flag), K(ret));
   } else if (flag == OB_STORAGE_ACCESS_OVERWRITER) {
-    ObStorageWriter* overwriter = static_cast<ObStorageWriter*>(ctx);
+    ObStorageWriter *overwriter = static_cast<ObStorageWriter*>(ctx);
     if (OB_FAIL(overwriter->write((char*)buf, size))) {
       OB_LOG(WARN, "fail to do overwrite write!", K(ret));
     }
   } else if (flag == OB_STORAGE_ACCESS_APPENDER) {
-    ObStorageAppender* appender = static_cast<ObStorageAppender*>(ctx);
+    ObStorageAppender *appender = static_cast<ObStorageAppender*>(ctx);
     if (OB_FAIL(appender->write((char*)buf, size))) {
       OB_LOG(WARN, "fail to do append write!", K(ret));
     }
+  } else if (flag == OB_STORAGE_ACCESS_MULTIPART_WRITER) {
+    ObStorageMultiPartWriter *multipart_writer = static_cast<ObStorageMultiPartWriter*>(ctx);
+    if (OB_FAIL(multipart_writer->write((char*)buf, size))) {
+      OB_LOG(WARN, "fail to do multipart writer write!", K(ret));
+    }
   } else {
     ret = OB_INVALID_ARGUMENT;
     OB_LOG(WARN, "unknow access type, not a writable type!", K(flag), K(ret));
@@ -505,11 +676,19 @@ int ObObjectDevice::pwrite(const ObIOFd &fd, const int64_t offset, const int64_t
     OB_LOG(WARN, "fd is not init!", K(fd.first_id_), K(fd.second_id_));
   } else if (OB_FAIL(fd_mng_.fd_to_ctx(fd, ctx))) {
     OB_LOG(WARN, "fail to get ctx accroding fd!", K(ret));
+  } else if (OB_ISNULL(ctx)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "fd ctx is null!", K(flag), K(ret));
   } else if (flag == OB_STORAGE_ACCESS_RANDOMWRITER) {
-    ObStorageAppender* appender = static_cast<ObStorageAppender*>(ctx);
+    ObStorageAppender *appender = static_cast<ObStorageAppender*>(ctx);
     if (OB_FAIL(appender->pwrite((char*)buf, size, offset))) {
       OB_LOG(WARN, "fail to do appender pwrite!", K(ret));
     }
+  } else if (flag == OB_STORAGE_ACCESS_MULTIPART_WRITER) {
+    ObStorageMultiPartWriter *multipart_writer = static_cast<ObStorageMultiPartWriter*>(ctx);
+    if (OB_FAIL(multipart_writer->pwrite((char*)buf, size, offset))) {
+      OB_LOG(WARN, "fail to do multipart writer pwrite!", K(ret));
+    }
   } else {
     ret = OB_INVALID_ARGUMENT;
     OB_LOG(WARN, "unknow access type, not a writable type!", K(flag), K(ret));
@@ -550,9 +729,7 @@ int ObObjectDevice::reconfig(const ObIODOpts &opts)
 
 int ObObjectDevice::seal_file(const ObIOFd &fd)
 {
-  UNUSED(fd);
-  OB_LOG_RET(WARN, OB_NOT_SUPPORTED, "seal file is not support in object device !", K(fd));
-  return OB_NOT_SUPPORTED;
+  return seal_for_adaptive(fd);
 }
 
 int ObObjectDevice::scan_dir(const char *dir_name, int (*func)(const dirent *entry))

deps/oblib/src/lib/restore/ob_object_device.h

@@ -73,15 +73,31 @@ public:
   //add new
   virtual int get_config(ObIODOpts &opts) override;
 
+  int del_unmerged_parts(const char *pathname);
+  int seal_for_adaptive(const ObIOFd &fd);
+  int adaptive_exist(const char *pathname, bool &is_exist);
+  int adaptive_stat(const char *pathname, ObIODFileStat &statbuf);
+  int adaptive_unlink(const char *pathname);
+  int adaptive_scan_dir(const char *dir_name, ObBaseDirEntryOperator &op);
+
 public:
   common::ObFdSimulator& get_fd_mng() {return fd_mng_;}                 
 
 private:
   int get_access_type(ObIODOpts *opts, ObStorageAccessType& access_type);
   int open_for_reader(const char *pathname, void*& ctx);
+  int open_for_adaptive_reader_(const char *pathname, void *&ctx);
   int open_for_overwriter(const char *pathname, void*& ctx);
   int open_for_appender(const char *pathname, ObIODOpts *opts, void*& ctx);
+  int open_for_multipart_writer_(const char *pathname, void *&ctx);
   int release_res(void* ctx, const ObIOFd &fd, ObStorageAccessType access_type);
+
+  int inner_exist_(const char *pathname, bool &is_exist, const bool is_adaptive = false);
+  int inner_stat_(const char *pathname, ObIODFileStat &statbuf, const bool is_adaptive = false);
+  int inner_unlink_(const char *pathname, const bool is_adaptive = false);
+  int inner_scan_dir_(const char *dir_name,
+      ObBaseDirEntryOperator &op, const bool is_adaptive = false);
+
 private:
   //maybe fd mng can be device level
   common::ObFdSimulator    fd_mng_;
@@ -89,8 +105,10 @@ private:
   ObStorageUtil            util_;
   /*obj ctx pool: use to create fd ctx(reader/writer)*/
   common::ObPooledAllocator<ObStorageReader, ObMalloc, ObSpinLock> reader_ctx_pool_;
+  common::ObPooledAllocator<ObStorageAdaptiveReader, ObMalloc, ObSpinLock> adaptive_reader_ctx_pool_;
   common::ObPooledAllocator<ObStorageAppender, ObMalloc, ObSpinLock> appender_ctx_pool_;
   common::ObPooledAllocator<ObStorageWriter, ObMalloc, ObSpinLock> overwriter_ctx_pool_;
+  common::ObPooledAllocator<ObStorageMultiPartWriter, ObMalloc, ObSpinLock> multipart_writer_ctx_pool_;
   common::ObObjectStorageInfo storage_info_;
   bool is_started_;
   char storage_info_str_[OB_MAX_URI_LENGTH];

deps/oblib/src/lib/restore/ob_storage.h

@@ -16,11 +16,21 @@
 #include "ob_storage_file.h"
 #include "ob_storage_oss_base.h"
 #include "ob_storage_cos_base.h"
+#include "ob_storage_s3_base.h"
 
 namespace oceanbase
 {
 namespace common
 {
+/* In order to uniform naming format, here we will define the name format about uri/path.
+ *   a. 'uri' represents a full path which has type prefix, like OSS/FILE.
+ *   b. 'raw_dir_path' represents a dir path which does not have suffix '/'
+ *   c. 'dir_path' represents a dir path, but we can't ensure that this path has suffix '/' or not
+ *   d. 'full_dir_path' represents a dir path which has suffix '/'
+ *   e. 'dir_name' represents a directory name, not a path
+ *   f. 'obj_path' represents a object/file path
+ *   g. 'obj_name' represents a object/file name, not a path
+ */
 
 void print_access_storage_log(const char *msg, const common::ObString &uri,
     const int64_t start_ts, const int64_t size = 0, bool *is_slow = NULL);
@@ -106,6 +116,42 @@ struct ObStorageObjectVersionParam {
   bool open_object_version_;
 };
 
+// If the object is 'SIMULATE_APPEND' type, we will use this operation to list all of its children objects.
+class ListAppendableObjectFragmentOp : public common::ObBaseDirEntryOperator
+{
+public:
+  ListAppendableObjectFragmentOp(const bool need_size = true)
+    : exist_format_meta_(false), exist_seal_meta_(false), meta_arr_(), need_size_(need_size) {}
+
+  virtual ~ListAppendableObjectFragmentOp() { meta_arr_.reset(); }
+  virtual int func(const dirent *entry) override;
+  virtual bool need_get_file_size() const { return need_size_; }
+  int gen_object_meta(ObStorageObjectMeta &obj_meta);
+
+  bool exist_format_meta() const { return exist_format_meta_; }
+  bool exist_seal_meta() const { return exist_seal_meta_; }
+
+private:
+  bool exist_format_meta_;
+  bool exist_seal_meta_;
+  ObArray<ObAppendableFragmentMeta> meta_arr_; // save all 'data fragment meta'
+  bool need_size_;
+};
+
+// If the object is 'SIMULATE_APPEND' type, we will use this operation to delete all of its children objects.
+class ObStorageUtil;
+class DelAppendableObjectFragmentOp : public ObBaseDirEntryOperator
+{
+public:
+  DelAppendableObjectFragmentOp(const common::ObString &uri, ObStorageUtil &util);
+  virtual ~DelAppendableObjectFragmentOp() {}
+  virtual int func(const dirent *entry) override;
+
+private:
+  const common::ObString &uri_;
+  ObStorageUtil &util_;
+};
+
 class ObStorageUtil
 {
 public:
@@ -115,26 +161,110 @@ public:
   virtual ~ObStorageUtil() {}
   int open(common::ObObjectStorageInfo *storage_info);
   void close();
+
+  ////////////////////// READY //// TO //// DROP ///// BELOW ////////////////////////////////
   int is_exist(const common::ObString &uri, bool &exist);
   int get_file_length(const common::ObString &uri, int64_t &file_length);
   int del_file(const common::ObString &uri);
+  int list_files(const common::ObString &dir_path, common::ObBaseDirEntryOperator &op);
+  int list_directories(const common::ObString &dir_path, common::ObBaseDirEntryOperator &op);
+  ////////////////////// READY //// TO //// DROP ///// ABOVE ////////////////////////////////
+
   int mkdir(const common::ObString &uri);
   int write_single_file(const common::ObString &uri, const char *buf, const int64_t size);
-  int list_files(const common::ObString &dir_path, common::ObBaseDirEntryOperator &op);
   int del_dir(const common::ObString &uri);
-  int list_directories(const common::ObString &dir_path, common::ObBaseDirEntryOperator &op);
   int is_tagging(const common::ObString &uri, bool &is_tagging);
+  // This func is to check the object/file/dir exists or not.
+  // If the uri is a common directory(not a 'SIMULATE_APPEND' object), please set @is_adaptive as FALSE
+  // If the uri is a normal object, please set @is_adaptive as FALSE
+  // If the uri is a 'SIMULATE_APPEND' object or we can't ensure that it is a normal object or a
+  // 'SIMULATE_APPEND' object, please set @is_adaptive as TRUE.
+  int is_exist(const common::ObString &uri, const bool is_adaptive, bool &exist);
+  int get_file_length(const common::ObString &uri, const bool is_adaptive, int64_t &file_length);
+  int list_appendable_file_fragments(const common::ObString &uri, ObStorageObjectMeta &obj_meta);
+
+  int del_file(const common::ObString &uri, const bool is_adaptive);
+  int del_unmerged_parts(const common::ObString &uri);
+
+  // For one object, if given us the uri(no matter in oss, cos or s3), we can't tell the type of this object.
+  // It may be a 'single、normal' object. Or it may be a 's3-appendable-object'(like a dir), containing several
+  // 'single、normal' objects.
+  // So, this function is for checking the object meta, to get its meta info
+  //
+  // @uri, the object full path in object storage.
+  // @is_adaptive, if FALSE, means it is a normal object absolutely.
+  //               if TRUE, means we don't know it type. We need to check its real type.
+  // @need_fragment_meta, if TRUE and the type is a 's3-appendable-object', we need to get its child objects meta.
+  //                      for example, when using adaptive reader, this param will set as TRUE; when using is_exist(),
+  //                      this param will set as FALSE
+  // @obj_meta the result, which saves the meta info of this object. If the target object not exists, we can check
+  //           obj_meta.is_exist_, not return OB_BACKUP_FILE_NOT_EXIST.
+  int detect_storage_obj_meta(const common::ObString &uri, const bool is_adaptive,
+                              const bool need_fragment_meta, ObStorageObjectMeta &obj_meta);
+
+  // Due to the 'SIMULATE_APPEND' object and 'NORMAL' object may exist together, thus we can't simply list all objects
+  // based on the prefix.
+  //
+  // For example,
+  //       dir1
+  //         --file1
+  //         --file2
+  //         --dir11
+  //            --file11
+  //            --file12
+  //            --appendable11
+  //               --@FORMAT_META
+  //         --appendable1
+  //            --@FORMAT_META
+  //            --@0-100
+  //
+  // ['appendable1' and 'appendable11' are 'SIMULATE_APPEND' type]. If we want to list 'dir1/', we supposed to get the result as flows:
+  //  dir1/file1, dir1/file2
+  //  dir1/dir11/file11, dir1/dir11/file12, dir1/dir11/appendable11
+  //  dir1/appendable1
+  //  Above 6 object paths are the final result.
+  //
+  // @is_adaptive  If we can ensure that there not exist 'SIMULATE_APPEND' type object in @uri, we can set this param
+  //               as FALSE, otherwise set it as TRUE.
+  int list_files(const common::ObString &uri, const bool is_adaptive, common::ObBaseDirEntryOperator &op);
+  int list_directories(const common::ObString &uri, const bool is_adaptive, common::ObBaseDirEntryOperator &op);
 
 private:
   // we does not use storage_info_ to judge init, since for nfs&local, storage_info_ is null
-  bool is_init() {return init_state;}
+  bool is_init() { return init_state; }
+
+  // If there only exists common type object in this uri, this function will list all the files.
+  // If there also exists 'SIMULATE_APPEND' type object in this uri, this function will just list
+  // this 'appendable-dir' name, not include its children objects' name.
+  //
+  // NOTICE: children objects of 'appendable-dir' all have the same prefix(OB_S3_APPENDABLE_FRAGMENT_PREFIX).
+  //         If there exists some children objects not have this prefix, these objects will also be listed.
+  //         Cuz we think these objects are just some common objects.
+  int list_adaptive_files(const common::ObString &uri, common::ObBaseDirEntryOperator &op);
+  // ObjectStorage and Filesystem need to handle seperately.
+  int handle_listed_objs(ObStorageListCtxBase *ctx_base, const common::ObString &uri,
+                         const common::ObString &dir_path, common::ObBaseDirEntryOperator &op);
+  int handle_listed_appendable_obj(ObStorageListObjectsCtx *list_ctx, const common::ObString &uri,
+                                   const common::ObString &dir_path, common::ObBaseDirEntryOperator &op);
+  int handle_listed_fs(ObStorageListCtxBase *ctx_base, const common::ObString &uri,
+                       const common::ObString &dir_path, common::ObBaseDirEntryOperator &op);
+
+  // For 'SIMULATE_APPEND' type file, if we want to get its file length, we can't get its length from object meta directly.
+  // <1> First, we need to check if there exists SEAL_META, if exists, read its content and get the file length
+  // <2> If not exists, we need to list all its children objects and get the file length
+  int get_adaptive_file_length(const common::ObString &uri, int64_t &file_length);
+  int read_seal_meta_if_needed(const common::ObString &uri, ObStorageObjectMeta &obj_meta);
+
+  int del_appendable_file(const common::ObString &uri);
 
   ObStorageFileUtil file_util_;
   ObStorageOssUtil oss_util_;
   ObStorageCosUtil cos_util_;
+  ObStorageS3Util s3_util_;
   ObIStorageUtil* util_;
   common::ObObjectStorageInfo* storage_info_;
   bool init_state;
+  ObStorageType device_type_;
   DISALLOW_COPY_AND_ASSIGN(ObStorageUtil);
 };
 
@@ -143,36 +273,69 @@ class ObStorageReader
 public:
   ObStorageReader();
   virtual ~ObStorageReader();
-  int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info);
-  int pread(char *buf,const int64_t buf_size, int64_t offset, int64_t &read_size);
+  virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info);
+  int pread(char *buf, const int64_t buf_size, int64_t offset, int64_t &read_size);
   int close();
   int64_t get_length() const { return file_length_; }
-private:
+
+protected:
   int64_t file_length_;
   ObIStorageReader *reader_;
   ObStorageFileReader file_reader_;
   ObStorageOssReader oss_reader_;
   ObStorageCosReader cos_reader_;
+  ObStorageS3Reader s3_reader_;
   int64_t start_ts_;
   char uri_[OB_MAX_URI_LENGTH];
+private:
   DISALLOW_COPY_AND_ASSIGN(ObStorageReader);
 };
 
+// The most important meaning of this class is to read SIMULATE_APPEND file.
+// But, if we use this class to read a normal object/file, it should also work well
+class ObStorageAdaptiveReader
+{
+public:
+  ObStorageAdaptiveReader();
+  ~ObStorageAdaptiveReader();
+  int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info);
+  int pread(char *buf, const int64_t buf_size, const int64_t offset, int64_t &read_size);
+  int close();
+  int64_t get_length() const { return meta_.length_; }
+
+private:
+  ObArenaAllocator allocator_;
+  ObStorageObjectMeta meta_;
+  ObString bucket_;
+  ObString object_;
+  ObIStorageReader *reader_;
+  ObStorageFileReader file_reader_;
+  ObStorageOssReader oss_reader_;
+  ObStorageCosReader cos_reader_;
+  ObStorageS3Reader s3_reader_;
+  int64_t start_ts_;
+  char uri_[OB_MAX_URI_LENGTH];
+  ObObjectStorageInfo *storage_info_;
+  DISALLOW_COPY_AND_ASSIGN(ObStorageAdaptiveReader);
+};
+
 class ObStorageWriter
 {
 public:
   ObStorageWriter();
   virtual ~ObStorageWriter();
-  int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info);
+  virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info);
   int write(const char *buf,const int64_t size);
   int close();
-private:
+protected:
   ObIStorageWriter *writer_;
   ObStorageFileWriter file_writer_;
   ObStorageOssWriter oss_writer_;
   ObStorageCosWriter cos_writer_;
+  ObStorageS3Writer s3_writer_;
   int64_t start_ts_;
   char uri_[OB_MAX_URI_LENGTH];
+private:
   DISALLOW_COPY_AND_ASSIGN(ObStorageWriter);
 };
 
@@ -190,28 +353,58 @@ public:
   };
 
   int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info);
-  int write(const char *buf,const int64_t size);
+  int write(const char *buf, const int64_t size);
   int pwrite(const char *buf, const int64_t size, const int64_t offset);
   int close();
   bool is_opened() const { return is_opened_; }
   int64_t get_length();
   void set_open_mode(StorageOpenMode mode) {file_appender_.set_open_mode(mode);}
-  TO_STRING_KV(KP(appender_), K_(start_ts), K_(is_opened),KCSTRING_(uri));
+  int seal_for_adaptive();
+  TO_STRING_KV(KP_(appender), K_(start_ts), K_(is_opened), KCSTRING_(uri));
+
 private:
   ObIStorageWriter *appender_;
   ObStorageFileAppender file_appender_;
   ObStorageOssAppendWriter oss_appender_;
   ObStorageCosAppendWriter cos_appender_;
+  ObStorageS3AppendWriter s3_appender_;
   int64_t start_ts_;
   bool is_opened_;
   char uri_[OB_MAX_URI_LENGTH];
   common::ObObjectStorageInfo storage_info_;
   ObArenaAllocator allocator_;
+  ObStorageType type_;
 
   int repeatable_pwrite_(const char *buf, const int64_t size, const int64_t offset);
   DISALLOW_COPY_AND_ASSIGN(ObStorageAppender);
 };
 
+class ObStorageMultiPartWriter
+{
+public:
+  ObStorageMultiPartWriter();
+  virtual ~ObStorageMultiPartWriter();
+  virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info);
+  int write(const char *buf, const int64_t size);
+  int pwrite(const char *buf, const int64_t size, const int64_t offset);
+  int close();
+  bool is_opened()  const {return is_opened_;}
+  int64_t get_length();
+  TO_STRING_KV(KP_(multipart_writer), K_(start_ts), K_(is_opened), KCSTRING_(uri));
+
+protected:
+  ObIStorageWriter *multipart_writer_;
+  ObStorageFileMultiPartWriter file_multipart_writer_;
+  ObStorageCosMultiPartWriter cos_multipart_writer_;
+  ObStorageOssMultiPartWriter oss_multipart_writer_;
+  ObStorageS3MultiPartWriter s3_multipart_writer_;
+  int64_t start_ts_;
+  bool is_opened_;
+  char uri_[OB_MAX_URI_LENGTH];
+  common::ObObjectStorageInfo storage_info_;
+	DISALLOW_COPY_AND_ASSIGN(ObStorageMultiPartWriter);
+};
+
 }//common
 }//oceanbase
 

deps/oblib/src/lib/restore/ob_storage_cos_base.cpp

@@ -52,65 +52,64 @@ void fin_cos_env()
 
 struct CosListFilesCbArg
 {
-  common::ObIAllocator &allocator;
-  ObString &dir_path;
-  ObBaseDirEntryOperator &list_op;
+  common::ObIAllocator &allocator_;
+  ObString &dir_path_;
+  ObBaseDirEntryOperator &list_op_;
 
   CosListFilesCbArg(
-    common::ObIAllocator &alloc,
+    common::ObIAllocator &allocator,
     ObString &dir,
     ObBaseDirEntryOperator &op)
-    : allocator(alloc),
-      dir_path(dir),
-      list_op(op) {}
+    : allocator_(allocator),
+      dir_path_(dir),
+      list_op_(op) {}
 
   ~CosListFilesCbArg() {}
 };
 
-static int execute_list_callback(
-  ObBaseDirEntryOperator &op,
-  qcloud_cos::ObCosWrapper::CosListObjPara &para,
-  ObString &file_name)
+struct CosListFilesCtx
 {
-  int ret = OB_SUCCESS;
-  int64_t dname_size = sizeof(para.last_container_name_.d_name);
-  if (dname_size >= (file_name.length() + 1)) {
-    if (OB_FAIL(databuff_printf(para.last_container_name_.d_name, dname_size, "%s", file_name.ptr()))) {
-      OB_LOG(WARN, "fail to copy file name to entry buf!", K(ret), K(dname_size));
-    } else if (OB_FAIL(op.func(&(para.last_container_name_)))) {
-      OB_LOG(WARN, "fail to execute list callback!", K(ret), KCSTRING(para.last_container_name_.d_name),
-            K(para.last_container_name_.d_type), K(DT_REG), K(dname_size));
-    }
-  } else {
-    ret = OB_BUF_NOT_ENOUGH;
-    OB_LOG(WARN, "file name len is too long!", K(ret), K(file_name.length()), K(sizeof(dirent)));
-  }
-  return ret;
-}
+  common::ObIAllocator &allocator_;
+  ObString &dir_path_;
+  ObStorageListObjectsCtx &list_ctx_;
+
+  CosListFilesCtx(
+    common::ObIAllocator &allocator,
+    ObString &dir,
+    ObStorageListObjectsCtx &ctx)
+    : allocator_(allocator),
+      dir_path_(dir),
+      list_ctx_(ctx) {}
+
+  ~CosListFilesCtx() {}
+
+  TO_STRING_KV(K_(dir_path), K_(list_ctx));
+};
 
 static int handle_object_name_cb(qcloud_cos::ObCosWrapper::CosListObjPara &para)
 {
   int ret = OB_SUCCESS;
-  para.next_flag_ = true;
   if (OB_ISNULL(para.arg_)) {
     ret = OB_INVALID_ARGUMENT;
     OB_LOG(WARN, "arg is empty", K(ret));
-  } else if (OB_ISNULL(para.cur_full_path_slice_name_) || 0 >= para.full_path_size_) {
+  } else if (OB_ISNULL(para.cur_obj_full_path_)) {
     ret = OB_INVALID_ARGUMENT;
-    OB_LOG(WARN, "current object_name is empty", K(ret));
+    OB_LOG(WARN, "current object full path is empty", K(ret), K(para.cur_obj_full_path_));
   } else if (qcloud_cos::ObCosWrapper::CosListObjPara::CosListType::COS_LIST_CB_ARG == para.type_) {
     CosListFilesCbArg *ctx = static_cast<CosListFilesCbArg *>(para.arg_);
 
     // Returned object name is the whole object path, but we donot need the prefix dir_path.
-    // So, we trim the dir_name from object_name path.
-    const int dir_name_str_len = strlen(ctx->dir_path.ptr());
-    const int object_name_len = para.full_path_size_ - dir_name_str_len;
-    const int object_name_start_pos = dir_name_str_len;
-    //tmp_file_name like slice name
-    ObString tmp_file_name(object_name_len, object_name_len,
-      para.cur_full_path_slice_name_ + object_name_start_pos);
-    if (OB_FAIL(execute_list_callback(ctx->list_op, para, tmp_file_name))) {
-      OB_LOG(WARN, "fail to execute list callback!", K(ret));
+    // So, we trim the object full path to get object name
+    const int dir_name_str_len = strlen(ctx->dir_path_.ptr());
+    int64_t object_size = -1;
+    if (OB_FAIL(c_str_to_int(para.cur_object_size_str_, object_size))) {
+      OB_LOG(WARN, "fail to get listed cos object size", K(ret), K(para.cur_object_size_str_));
+    } else if (OB_FAIL(handle_listed_object(ctx->list_op_,
+                                            para.cur_obj_full_path_ + dir_name_str_len,
+                                            para.full_path_size_ - dir_name_str_len,
+                                            object_size))) {
+      OB_LOG(WARN, "fail to handle listed cos object", K(ret), K(para.cur_obj_full_path_),
+          K(dir_name_str_len), K(para.full_path_size_), K(object_size));
     }
   } else {
     ret = OB_INVALID_ARGUMENT;
@@ -120,39 +119,55 @@ static int handle_object_name_cb(qcloud_cos::ObCosWrapper::CosListObjPara &para)
   return ret;
 }
 
+static int handle_list_object_ctx(qcloud_cos::ObCosWrapper::CosListObjPara &para)
+{
+  int ret = OB_SUCCESS;
+  if (OB_ISNULL(para.arg_)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "arg is empty", K(ret));
+  } else if (qcloud_cos::ObCosWrapper::CosListObjPara::CosListType::COS_PART_LIST_CTX == para.type_) {
+    CosListFilesCtx *ctx = static_cast<CosListFilesCtx *>(para.arg_);
+    if (para.finish_part_list_) {
+      if (OB_FAIL(ctx->list_ctx_.set_next_token(para.next_flag_,
+                                                para.next_token_,
+                                                para.next_token_size_))) {
+        OB_LOG(WARN, "fail to set list ctx next token",
+            K(ret), K(para.next_flag_), K(para.next_token_), K(para.next_token_size_));
+      }
+    } else {
+      int64_t object_size = -1;
+      if (OB_FAIL(c_str_to_int(para.cur_object_size_str_, object_size))) {
+        OB_LOG(WARN, "fail to get listed cos object size", K(ret), K(para.cur_object_size_str_));
+      } else if (OB_FAIL(ctx->list_ctx_.handle_object(para.cur_obj_full_path_,
+                                                      para.full_path_size_,
+                                                      object_size))) {
+        OB_LOG(WARN, "fail to add listed cos obejct meta into list ctx",
+            K(ret), K(para.cur_obj_full_path_), K(para.full_path_size_), K(object_size));
+      }
+    }
+  } else {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "not supported type", K(ret), K(para.type_));
+  }
+  return ret;
+}
+
 static int handle_directory_name_cb(
     void *arg,
     const qcloud_cos::ObCosWrapper::CosListObjPara::CosListType type,
-    const char *object_name,
-    int64_t object_size)
+    const char *dir_name,
+    int64_t dir_name_len)
 {
   int ret = OB_SUCCESS;
-  const char delimiter = '/';
   if (OB_ISNULL(arg)) {
     ret = OB_INVALID_ARGUMENT;
     OB_LOG(WARN, "arg is empty", K(ret), KP(arg));
-  } else if (OB_ISNULL(object_name) || 0 >= object_size) {
-    ret = OB_INVALID_ARGUMENT;
-    OB_LOG(WARN, "object_name is empty", K(ret), KP(object_name), K(object_size));
-  } else if (delimiter != object_name[object_size - 1]) {
-    ret = OB_ERR_UNEXPECTED;
-    OB_LOG(WARN, "object name is unexpected", K(ret), KCSTRING(object_name), K(object_size));
   } else {
-    const int64_t name_length = object_size - 1;
-    if (name_length <= 0) {
-      ret = OB_ERR_UNEXPECTED;
-      OB_LOG(WARN, "object is not exist", K(ret), KCSTRING(object_name), K(object_size));
-    } else if (qcloud_cos::ObCosWrapper::CosListObjPara::CosListType::COS_LIST_CB_ARG == type) {
+    if (qcloud_cos::ObCosWrapper::CosListObjPara::CosListType::COS_LIST_CB_ARG == type) {
       CosListFilesCbArg *ctx = static_cast<CosListFilesCbArg *>(arg);
-      ObString directory_name;
-      dirent dir_name_entry;
-      dir_name_entry.d_type = DT_DIR;
-      ObString tmp_directory_name(name_length, name_length, object_name);
-      if (OB_FAIL(databuff_printf(dir_name_entry.d_name, sizeof(dir_name_entry.d_name), "%.*s",
-                                  tmp_directory_name.length(), tmp_directory_name.ptr()))) {
-        OB_LOG(WARN, "fail to set dir name entry", K(ret), K(tmp_directory_name));
-      } else if (OB_FAIL(ctx->list_op.func(&dir_name_entry))) {
-        OB_LOG(WARN, "fail to call list directories operation callback", K(ret), K(tmp_directory_name));
+      if (OB_FAIL(handle_listed_directory(ctx->list_op_, dir_name, dir_name_len))) {
+        OB_LOG(WARN, "fail to handle cos directory name",
+            K(ret), K(dir_name), K(dir_name_len));
       }
     } else {
       ret = OB_INVALID_ARGUMENT;
@@ -198,37 +213,44 @@ void ObStorageCosUtil::close()
 int ObStorageCosUtil::is_exist(const ObString &uri, bool &is_exist)
 {
   int ret = OB_SUCCESS;
-  ObExternalIOCounterGuard io_guard;
   is_exist = false;
-  int64_t file_length = 0;
-  if (OB_UNLIKELY(!is_opened_)) {
-    ret = OB_COS_ERROR;
-    OB_LOG(WARN, "cos util not opened", K(ret));
-  } else if (OB_UNLIKELY(uri.empty())) {
-    ret = OB_INVALID_ARGUMENT;
-    OB_LOG(WARN, "invalid argument", K(ret), K(uri));
-  } else if (OB_FAIL(get_object_meta_(uri, is_exist, file_length))) {
-    OB_LOG(WARN, "fail to get object meta", K(ret), K(uri));
+  ObStorageObjectMetaBase obj_meta;
+  if (OB_FAIL(head_object_meta(uri, obj_meta))) {
+    OB_LOG(WARN, "fail to head object meta", K(ret), K(uri));
+  } else {
+    is_exist = obj_meta.is_exist_;
   }
   return ret;
 }
 
 int ObStorageCosUtil::get_file_length(const ObString &uri, int64_t &file_length)
+{
+  int ret = OB_SUCCESS;
+  file_length = 0;
+  ObStorageObjectMetaBase obj_meta;
+  if (OB_FAIL(head_object_meta(uri, obj_meta))) {
+    OB_LOG(WARN, "fail to head object meta", K(ret), K(uri));
+  } else if (!obj_meta.is_exist_) {
+    ret = OB_BACKUP_FILE_NOT_EXIST;
+    OB_LOG(WARN, "backup file is not exist", K(ret), K(uri));
+  } else {
+    file_length = obj_meta.length_;
+  }
+  return ret;
+}
+
+int ObStorageCosUtil::head_object_meta(const ObString &uri, ObStorageObjectMetaBase &obj_meta)
 {
   int ret = OB_SUCCESS;
   ObExternalIOCounterGuard io_guard;
-  bool is_file_exist = false;
   if (OB_UNLIKELY(!is_opened_)) {
     ret = OB_COS_ERROR;
     OB_LOG(WARN, "cos util not opened", K(ret));
   } else if (OB_UNLIKELY(uri.empty())) {
     ret = OB_INVALID_ARGUMENT;
     OB_LOG(WARN, "invalid argument", K(ret), K(uri));
-  } else if (OB_FAIL(get_object_meta_(uri, is_file_exist, file_length))) {
+  } else if (OB_FAIL(get_object_meta_(uri, obj_meta.is_exist_, obj_meta.length_))) {
     OB_LOG(WARN, "fail to get object meta", K(ret), K(uri));
-  } else if (!is_file_exist) {
-    ret = OB_BACKUP_FILE_NOT_EXIST;
-    OB_LOG(WARN, "backup file is not exist", K(ret), K(uri));
   }
   return ret;
 }
@@ -346,8 +368,9 @@ int ObStorageCosUtil::list_files(
     ObBaseDirEntryOperator &op)
 {
   int ret = OB_SUCCESS;
+  ObStorageCosBase cos_base;
   ObExternalIOCounterGuard io_guard;
-  ObArenaAllocator allocator;
+  ObArenaAllocator allocator(ObModIds::BACKUP);
 
   if (OB_UNLIKELY(!is_opened_)) {
     ret = OB_COS_ERROR;
@@ -355,61 +378,107 @@ int ObStorageCosUtil::list_files(
   } else if (OB_UNLIKELY(uri.empty())) {
     ret = OB_INVALID_ARGUMENT;
     OB_LOG(WARN, "invalid argument", K(ret), K(uri));
+  } else if (OB_FAIL(cos_base.open(uri, storage_info_))) {
+    OB_LOG(WARN, "fail to open cos base", K(ret), K(uri), KPC_(storage_info));
   } else {
-    char dir_path_array[OB_MAX_URI_LENGTH] = {0};
-    const char *ptr = NULL; // point to actual dir path
-    const int min_dir_path_len = 2; // cuz dir_path end with '\0'
-    const char *separator = "/";
-    const char end_marker = '\0';
+    const char *full_dir_path = cos_base.get_handle().get_object_name().ptr();
+    const int64_t full_dir_path_len = cos_base.get_handle().get_object_name().length();
+    ObString full_dir_path_str(full_dir_path_len, full_dir_path);
 
-    ObStorageCosBase cos_base;
-    if (OB_FAIL(cos_base.open(uri, storage_info_))) {
-      OB_LOG(WARN, "fail to open cos base", K(ret), K(uri));
-    } else {
-      const ObString &dir_path_str = cos_base.get_handle().get_object_name();
-      const ObString::obstr_size_t dir_path_len = dir_path_str.length();
-      int32_t actual_dir_path_len = dir_path_len;
-      if (dir_path_len < min_dir_path_len) {
-        ret = OB_ERR_UNEXPECTED;
-        OB_LOG(WARN, "dir_path_str length is unepxected", K(ret), K(dir_path_str), K(dir_path_len));
-      } else {
-        // Let dir path end with '/'
-        if (dir_path_str.ptr()[dir_path_len - 1] != end_marker) {
-          ret = OB_ERR_UNEXPECTED;
-          OB_LOG(WARN, "dir_path_str not end with '\0'");
-        } else if (dir_path_str.ptr()[dir_path_len - 2] == *separator) {
-          ptr = dir_path_str.ptr();
-        } else if (OB_FAIL(databuff_printf(dir_path_array, OB_MAX_URI_LENGTH, "%s/", dir_path_str.ptr()))) {
-          OB_LOG(WARN, "fail to deep copy dir path", K(ret), K(dir_path_len), K(OB_MAX_URI_LENGTH));
-        } else {
-          ptr = dir_path_array;
-          actual_dir_path_len += 1;
-        }
-      }
-
-      if (OB_SUCC(ret)) {
-        ObString tmp_dir;
-        tmp_dir.assign(const_cast<char *>(ptr), actual_dir_path_len);
-
-        // Construct list object callback arg
-        CosListFilesCbArg arg(allocator, tmp_dir, op);
-        if (OB_FAIL(cos_base.list_objects(uri, tmp_dir, separator, arg))) {
-          OB_LOG(WARN, "fail to list object in cos_base", K(ret), K(uri), K(tmp_dir));
-        }
-      }
+    // Construct list object callback arg
+    CosListFilesCbArg arg(allocator, full_dir_path_str, op);
+    if (OB_UNLIKELY(!is_end_with_slash(full_dir_path))) {
+      ret = OB_INVALID_ARGUMENT;
+      OB_LOG(WARN, "uri is not terminated with '/'", K(ret), K(uri), K(full_dir_path));
+    } else if (OB_FAIL(cos_base.list_objects(uri, full_dir_path_str, arg))) {
+      OB_LOG(WARN, "fail to list object in cos_base", K(ret), K(uri), K(full_dir_path_str));
     }
-    cos_base.reset();
   }
+
+  cos_base.reset();
+  return ret;
+}
+
+int ObStorageCosUtil::list_files(
+    const ObString &uri,
+    ObStorageListCtxBase &ctx_base)
+{
+  int ret = OB_SUCCESS;
+  ObStorageCosBase cos_base;
+  ObExternalIOCounterGuard io_guard;
+  ObArenaAllocator allocator(ObModIds::BACKUP);
+  ObStorageListObjectsCtx &list_ctx = static_cast<ObStorageListObjectsCtx &>(ctx_base);
+
+  if (OB_UNLIKELY(!is_opened_)) {
+    ret = OB_COS_ERROR;
+    OB_LOG(WARN, "cos util not opened", K(ret));
+  } else if (OB_UNLIKELY(uri.empty() || !list_ctx.is_valid())) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid argument", K(ret), K(uri), K(list_ctx));
+  } else if (OB_FAIL(cos_base.open(uri, storage_info_))) {
+    OB_LOG(WARN, "fail to open cos base", K(ret), K(uri), KPC_(storage_info));
+  } else {
+    const char *full_dir_path = cos_base.get_handle().get_object_name().ptr();
+    const int64_t full_dir_path_len = cos_base.get_handle().get_object_name().length();
+    ObString full_dir_path_str(full_dir_path_len, full_dir_path);
+
+    // Construct list object context
+    CosListFilesCtx arg(allocator, full_dir_path_str, list_ctx);
+    if (OB_UNLIKELY(!is_end_with_slash(full_dir_path))) {
+      ret = OB_INVALID_ARGUMENT;
+      OB_LOG(WARN, "uri is not terminated with '/'", K(ret), K(uri), K(full_dir_path));
+    } else if (OB_FAIL(cos_base.list_objects(uri, full_dir_path_str, list_ctx.next_token_, arg))) {
+      OB_LOG(WARN, "fail to list object in cos_base",
+          K(ret), K(list_ctx), K(uri), K(full_dir_path_str));
+    }
+  }
+
+  cos_base.reset();
   return ret;
 }
 
 int ObStorageCosUtil::list_directories(
     const ObString &uri,
     ObBaseDirEntryOperator &op)
+{
+  int ret = OB_SUCCESS;
+  ObStorageCosBase cos_base;
+  ObExternalIOCounterGuard io_guard;
+  ObArenaAllocator allocator(ObModIds::BACKUP);
+
+  if (OB_UNLIKELY(!is_opened_)) {
+    ret = OB_COS_ERROR;
+    OB_LOG(WARN, "cos util not opened", K(ret));
+  } else if (OB_UNLIKELY(uri.empty())) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid argument", K(ret), K(uri));
+  } else if (OB_FAIL(cos_base.open(uri, storage_info_))) {
+    OB_LOG(WARN, "fail to open cos base", K(ret), K(uri), KPC_(storage_info));
+  } else {
+    const char *delimiter_string = "/";
+    const char *next_marker_string = "";
+    const char *full_dir_path = cos_base.get_handle().get_object_name().ptr();
+    const int64_t full_dir_path_len = cos_base.get_handle().get_object_name().length();
+    ObString full_dir_path_str(full_dir_path_len, full_dir_path);
+
+    CosListFilesCbArg arg(allocator, full_dir_path_str, op);
+    if (OB_UNLIKELY(!is_end_with_slash(full_dir_path))) {
+      ret = OB_INVALID_ARGUMENT;
+      OB_LOG(WARN, "uri is not terminated with '/'", K(ret), K(uri), K(full_dir_path));
+    } else if (OB_FAIL(cos_base.list_directories(uri, full_dir_path_str,
+                                                 next_marker_string, delimiter_string, arg))) {
+      OB_LOG(WARN, "fail to list directories in cos_base", K(ret), K(uri), K(full_dir_path_str));
+    }
+  }
+
+  cos_base.reset();
+  return ret;
+}
+
+int ObStorageCosUtil::del_unmerged_parts(const ObString &uri)
 {
   int ret = OB_SUCCESS;
   ObExternalIOCounterGuard io_guard;
-  ObArenaAllocator allocator;
 
   if (OB_UNLIKELY(!is_opened_)) {
     ret = OB_COS_ERROR;
@@ -418,47 +487,13 @@ int ObStorageCosUtil::list_directories(
     ret = OB_INVALID_ARGUMENT;
     OB_LOG(WARN, "invalid argument", K(ret), K(uri));
   } else {
-    char dir_path_array[OB_MAX_URI_LENGTH] = {0};
-    const char *ptr = NULL; // point to actual dir path
-    const int min_dir_path_len = 2; // cuz dir_path end with '\0'
-    const char *delimiter_string = "/";
-    const char *next_marker_string = "";
-    const char end_marker = '\0';
-
     ObStorageCosBase cos_base;
     if (OB_FAIL(cos_base.open(uri, storage_info_))) {
       OB_LOG(WARN, "fail to open cos base", K(ret), K(uri));
+    } else if (OB_FAIL(cos_base.del_unmerged_parts(uri))) {
+      OB_LOG(WARN, "fail to del unmerged parts", K(ret), K(uri));
     } else {
-      const ObString &dir_path_str = cos_base.get_handle().get_object_name();
-      const ObString::obstr_size_t dir_path_len = dir_path_str.length();
-      int32_t actual_dir_path_len = dir_path_len;
-      if (dir_path_len < min_dir_path_len) {
-        ret = OB_ERR_UNEXPECTED;
-        OB_LOG(WARN, "dir_path_str length is unepxected", K(ret), K(dir_path_str), K(dir_path_len));
-      } else {
-        // Let dir path end with '/'
-        if (dir_path_str.ptr()[dir_path_len - 1] != end_marker) {
-          ret = OB_ERR_UNEXPECTED;
-          OB_LOG(WARN, "dir_path_str not end with '\0'");
-        } else if (dir_path_str.ptr()[dir_path_len - 2] == *delimiter_string) {
-          ptr = dir_path_str.ptr();
-        } else if (OB_FAIL(databuff_printf(dir_path_array, OB_MAX_URI_LENGTH, "%s/", dir_path_str.ptr()))) {
-          OB_LOG(WARN, "fail to deep copy dir path", K(ret), K(dir_path_len), K(OB_MAX_URI_LENGTH));
-        } else {
-          ptr = dir_path_array;
-          actual_dir_path_len += 1;
-        }
-      }
-
-      if (OB_SUCC(ret)) {
-        ObString tmp_dir;
-        tmp_dir.assign(const_cast<char *>(ptr), actual_dir_path_len);
-
-        CosListFilesCbArg arg(allocator, tmp_dir, op);
-        if (OB_FAIL(cos_base.list_directories(uri, tmp_dir, next_marker_string, delimiter_string, arg))) {
-          OB_LOG(WARN, "fail to list directories in cos_base", K(ret), K(uri), K(tmp_dir));
-        }
-      }
+      OB_LOG(DEBUG, "succ to delete object", K(uri));
     }
     cos_base.reset();
   }
@@ -479,7 +514,9 @@ ObStorageCosBase::~ObStorageCosBase()
 
 void ObStorageCosBase::reset()
 {
-  handle_.reset();
+  if (is_opened_) {
+    handle_.reset();
+  }
   is_opened_ = false;
 }
 
@@ -566,22 +603,44 @@ int ObStorageCosBase::delete_object(const ObString &uri)
 
 int ObStorageCosBase::list_objects(
     const ObString &uri,
-    const ObString &dir_name_str,
-    const char *separator,
+    const ObString &full_dir_path_str,
     CosListFilesCbArg &arg)
 {
   int ret = OB_SUCCESS;
-  if (OB_ISNULL(separator)) {
+  if (OB_LIKELY(is_valid())) {
+    qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer(
+        handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length());
+    qcloud_cos::CosStringBuffer full_dir_path(full_dir_path_str.ptr(), full_dir_path_str.length());
+    if (OB_FAIL(qcloud_cos::ObCosWrapper::list_objects(handle_.get_ptr(),
+        bucket_name, full_dir_path, handle_object_name_cb, (void *)(&arg)))) {
+      OB_LOG(WARN, "fail to list objects", K(ret), K(uri));
+    }
+  } else {
+    ret = OB_ERR_UNEXPECTED;
+    OB_LOG(WARN, "cos wrapper handle not init or create", K(ret));
+  }
+  return ret;
+}
+
+int ObStorageCosBase::list_objects(
+    const ObString &uri,
+    const ObString &full_dir_path_str,
+    const char *next_token,
+    CosListFilesCtx &ctx)
+{
+  int ret = OB_SUCCESS;
+  if (OB_ISNULL(next_token)) {
     ret = OB_INVALID_ARGUMENT;
     OB_LOG(WARN, "invalid argument", K(ret));
   } else if (OB_LIKELY(is_valid())) {
     qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer(
         handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length());
-    qcloud_cos::CosStringBuffer dir_name(dir_name_str.ptr(), dir_name_str.length());
-    qcloud_cos::CosStringBuffer next_marker(separator, strlen(separator) + 1);
-    if (OB_FAIL(qcloud_cos::ObCosWrapper::list_objects(handle_.get_ptr(), bucket_name,
-        dir_name, next_marker, handle_object_name_cb, (void *)(&arg)))) {
-      OB_LOG(WARN, "fail to list objects", K(ret), K(uri));
+    qcloud_cos::CosStringBuffer full_dir_path(full_dir_path_str.ptr(), full_dir_path_str.length());
+    qcloud_cos::CosStringBuffer next_marker(next_token, strlen(next_token));
+
+    if (OB_FAIL(qcloud_cos::ObCosWrapper::list_part_objects(handle_.get_ptr(), bucket_name,
+        full_dir_path, next_marker, handle_list_object_ctx, (void *)(&ctx)))) {
+      OB_LOG(WARN, "fail to list part objects", K(ret), K(uri), K(next_token), K(ctx));
     }
   } else {
     ret = OB_ERR_UNEXPECTED;
@@ -592,7 +651,7 @@ int ObStorageCosBase::list_objects(
 
 int ObStorageCosBase::list_directories(
     const ObString &uri,
-    const ObString &dir_name_str,
+    const ObString &full_dir_path_str,
     const char *next_marker_str,
     const char *delimiter_str,
     CosListFilesCbArg &arg)
@@ -604,12 +663,14 @@ int ObStorageCosBase::list_directories(
   } else if (OB_LIKELY(is_valid())) {
     qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer(
         handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length());
-    qcloud_cos::CosStringBuffer dir_name(dir_name_str.ptr(), dir_name_str.length());
+    qcloud_cos::CosStringBuffer full_dir_path(full_dir_path_str.ptr(), full_dir_path_str.length());
     qcloud_cos::CosStringBuffer next_marker(next_marker_str, strlen(next_marker_str) + 1);
     qcloud_cos::CosStringBuffer delimiter(delimiter_str, strlen(delimiter_str) + 1);
-    if (OB_FAIL(qcloud_cos::ObCosWrapper::list_directories(handle_.get_ptr(), bucket_name, dir_name,
-        next_marker, delimiter, handle_directory_name_cb, (void *)(&arg)))) {
-      OB_LOG(WARN, "failed to list directories", K(ret), K(uri));
+
+    if (OB_FAIL(qcloud_cos::ObCosWrapper::list_directories(handle_.get_ptr(), bucket_name,
+        full_dir_path, next_marker, delimiter, handle_directory_name_cb, (void *)(&arg)))) {
+      OB_LOG(WARN, "failed to list directories",
+          K(ret), K(uri), K(next_marker_str), K(delimiter_str));
     }
   } else {
     ret = OB_ERR_UNEXPECTED;
@@ -639,10 +700,29 @@ int ObStorageCosBase::is_object_tagging(
   return ret;
 }
 
+int ObStorageCosBase::del_unmerged_parts(const ObString &uri)
+{
+  int ret = OB_SUCCESS;
+  if (OB_LIKELY(is_valid())) {
+    qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer(
+        handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length());
+    qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer(
+        handle_.get_object_name().ptr(), handle_.get_object_name().length());
+    if (OB_FAIL(qcloud_cos::ObCosWrapper::del_unmerged_parts(handle_.get_ptr(),
+                                                             bucket_name, object_name))) {
+      OB_LOG(WARN, "fail to del unmerged parts", K(ret), K(uri));
+    }
+  } else {
+    ret = OB_ERR_UNEXPECTED;
+    OB_LOG(WARN, "cos wrapper handle not init or create", K(ret));
+  }
+  return ret;
+}
+
 /*--------------------------------ObStorageCosReader---------------------------*/
 
 ObStorageCosReader::ObStorageCosReader()
-  : ObStorageCosBase(), file_length_(-1)
+  : ObStorageCosBase(), has_meta_(false), file_length_(-1)
 {
 }
 
@@ -650,7 +730,8 @@ ObStorageCosReader::~ObStorageCosReader()
 {
 }
 
-int ObStorageCosReader::open(const ObString &uri, ObObjectStorageInfo *storage_info)
+int ObStorageCosReader::open(const ObString &uri,
+    ObObjectStorageInfo *storage_info, const bool head_meta)
 {
   int ret = OB_SUCCESS;
   ObExternalIOCounterGuard io_guard;
@@ -661,22 +742,28 @@ int ObStorageCosReader::open(const ObString &uri, ObObjectStorageInfo *storage_i
   } else if (OB_FAIL(ObStorageCosBase::open(uri, storage_info))) {
     OB_LOG(WARN, "fail to open in cos_base", K(ret), K(uri));
   } else {
-    const ObString &bucket_str = handle_.get_bucket_name();
-    const ObString &object_str = handle_.get_object_name();
-    qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer(
-        bucket_str.ptr(), bucket_str.length());
-    qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer(
-        object_str.ptr(), object_str.length());
-    bool is_file_exist = false;
-    qcloud_cos::CosObjectMeta obj_meta;
-    if (OB_FAIL(qcloud_cos::ObCosWrapper::head_object_meta(handle_.get_ptr(), bucket_name,
-        object_name, is_file_exist, obj_meta))) {
-      OB_LOG(WARN, "fail to get object meta", K(ret), K(bucket_str), K(object_str));
-    } else if (!is_file_exist) {
-      ret = OB_BACKUP_FILE_NOT_EXIST;
-      OB_LOG(WARN, "backup file is not exist", K(ret), K(bucket_str), K(object_str));
-    } else {
-      file_length_ = obj_meta.file_length_;
+    if (head_meta) {
+      const ObString &bucket_str = handle_.get_bucket_name();
+      const ObString &object_str = handle_.get_object_name();
+      qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer(
+          bucket_str.ptr(), bucket_str.length());
+      qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer(
+          object_str.ptr(), object_str.length());
+      bool is_file_exist = false;
+      qcloud_cos::CosObjectMeta obj_meta;
+      if (OB_FAIL(qcloud_cos::ObCosWrapper::head_object_meta(handle_.get_ptr(), bucket_name,
+          object_name, is_file_exist, obj_meta))) {
+        OB_LOG(WARN, "fail to get object meta", K(ret), K(bucket_str), K(object_str));
+      } else if (!is_file_exist) {
+        ret = OB_BACKUP_FILE_NOT_EXIST;
+        OB_LOG(WARN, "backup file is not exist", K(ret), K(bucket_str), K(object_str));
+      } else {
+        file_length_ = obj_meta.file_length_;
+        has_meta_ = true;
+      }
+    }
+
+    if (OB_SUCC(ret)) {
       is_opened_ = true;
     }
   }
@@ -695,7 +782,7 @@ int ObStorageCosReader::pread(
   if (!is_opened_) {
     ret = OB_COS_ERROR;
     OB_LOG(WARN, "cos reader cannot read before it is opened", K(ret));
-  } else if (NULL == buf || buf_size <= 0 || offset < 0) {
+  } else if (OB_ISNULL(buf) || OB_UNLIKELY(buf_size <= 0 || offset < 0)) {
     ret = OB_INVALID_ARGUMENT;
     OB_LOG(WARN, "invalid argument", K(ret), KP(buf), K(buf_size), K(offset));
   } else {
@@ -853,9 +940,9 @@ int ObStorageCosAppendWriter::pwrite(const char *buf, const int64_t size, const
   if(OB_UNLIKELY(!is_opened_)) {
     ret = OB_COS_ERROR;
     OB_LOG(WARN, "cos append writer cannot write before it is not opened", K(ret));
-  } else if(NULL == buf || size < 0 || offset < 0) {
+  } else if(NULL == buf || size <= 0 || offset < 0) {
     ret = OB_INVALID_ARGUMENT;
-    OB_LOG(WARN, "buf is NULL or size is invalid", KP(buf), K(size), K(ret));
+    OB_LOG(WARN, "invalid arguments", KP(buf), K(size), K(ret), K(offset));
   } else if (OB_FAIL(do_write(buf, size, offset, is_pwrite))) {
     OB_LOG(WARN, "failed to do write", K(ret), KP(buf), K(size), K(offset));
   }
@@ -930,5 +1017,219 @@ int ObStorageCosAppendWriter::do_write(
   return ret;
 }
 
+/*------------------------------ObStorageCosMultiPartWriter---------------------------*/
+ObStorageCosMultiPartWriter::ObStorageCosMultiPartWriter()
+  : ObStorageCosBase(),
+    mod_(ObModIds::BACKUP),
+    allocator_(ModuleArena::DEFAULT_PAGE_SIZE, mod_),
+    base_buf_(NULL),
+    base_buf_pos_(0),
+    upload_id_(NULL),
+    partnum_(0),
+    file_length_(-1)
+{}
+
+ObStorageCosMultiPartWriter::~ObStorageCosMultiPartWriter()
+{
+  destroy();
+}
+
+void ObStorageCosMultiPartWriter::reuse()
+{
+  if (is_opened_) {
+    if (nullptr != upload_id_) {
+      handle_.free_mem(static_cast<void *>(upload_id_));
+    }
+    if (nullptr != base_buf_) {
+      handle_.free_mem(static_cast<void *>(base_buf_));
+    }
+  }
+  upload_id_ = nullptr;
+  base_buf_ = nullptr;
+  partnum_ = 0;
+  file_length_ = -1;
+  ObStorageCosBase::reset();
+}
+
+int ObStorageCosMultiPartWriter::open(const ObString &uri, common::ObObjectStorageInfo *storage_info)
+{
+  int ret = OB_SUCCESS;
+  ObExternalIOCounterGuard io_guard;
+  if (OB_UNLIKELY(is_opened_)) {
+    ret = OB_COS_ERROR;
+    OB_LOG(WARN, "already open, cannot open again", K(ret));
+  } else if (OB_UNLIKELY(uri.empty())) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "uri is empty", K(ret), K(uri));
+  } else if (OB_FAIL(ObStorageCosBase::open(uri, storage_info))) {
+    OB_LOG(WARN, "fail to open in cos_base", K(ret), K(uri));
+  } else {
+    const ObString &bucket_name_str = handle_.get_bucket_name();
+    const ObString &object_name_str = handle_.get_object_name();
+    qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer(
+        bucket_name_str.ptr(), handle_.get_bucket_name().length());
+    qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer(
+        object_name_str.ptr(), handle_.get_object_name().length());
+
+    if (OB_FAIL(qcloud_cos::ObCosWrapper::init_multipart_upload(handle_.get_ptr(),
+        bucket_name, object_name, upload_id_))) {
+      OB_LOG(WARN, "fail to init multipartupload", K(ret), K(bucket_name_str), K(object_name_str));
+    } else if (OB_ISNULL(upload_id_)) {
+      ret = OB_ERR_UNEXPECTED;
+      OB_LOG(WARN, "upload_id should not be null", K(ret));
+    } else if (OB_ISNULL(base_buf_ = static_cast<char *>(handle_.alloc_mem(COS_MULTIPART_UPLOAD_BUF_SIZE)))) {
+      ret = OB_ALLOCATE_MEMORY_FAILED;
+      OB_LOG(WARN, "fail to alloc buffer for cos multipartupload", K(ret));
+    } else {
+      is_opened_ = true;
+      base_buf_pos_ = 0;
+      file_length_ = 0;
+    }
+  }
+  return ret;
+}
+
+int ObStorageCosMultiPartWriter::write(const char * buf, const int64_t size)
+{
+  int ret = OB_SUCCESS;
+  ObExternalIOCounterGuard io_guard;
+  int64_t fill_size = 0;
+  int64_t buf_pos = 0;
+  if (OB_UNLIKELY(!is_opened_)) {
+    ret = OB_COS_ERROR;
+    OB_LOG(WARN, "write cos should open first", K(ret));
+  } else if (OB_ISNULL(buf) || OB_UNLIKELY(size < 0)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "buf is NULL or size is invalid", K(ret), KP(buf), K(size));
+  }
+
+  while (OB_SUCC(ret) && buf_pos != size) {
+    fill_size = std::min(COS_MULTIPART_UPLOAD_BUF_SIZE - base_buf_pos_, size - buf_pos);
+    memcpy(base_buf_ + base_buf_pos_, buf + buf_pos, fill_size);
+    base_buf_pos_ += fill_size;
+    buf_pos += fill_size;
+    if (base_buf_pos_ == COS_MULTIPART_UPLOAD_BUF_SIZE) {
+      if (OB_FAIL(write_single_part())) {
+        OB_LOG(WARN, "fail to write part into cos", K(ret));
+      } else {
+        base_buf_pos_ = 0;
+      }
+    }
+  }
+
+  // actually, current file size may be smaller than @size. Cuz we may not finish
+  // the complete multipartupload.
+  if (OB_SUCC(ret)) {
+    file_length_ += size;
+  }
+  return ret;
+}
+
+int ObStorageCosMultiPartWriter::pwrite(const char *buf, const int64_t size, const int64_t offset)
+{
+  UNUSED(offset);
+  return write(buf, size);
+}
+
+int ObStorageCosMultiPartWriter::write_single_part()
+{
+  int ret = OB_SUCCESS;
+  int tmp_ret = OB_SUCCESS;
+  ObExternalIOCounterGuard io_guard;
+
+  ++partnum_;
+  if (partnum_ > COS_MAX_PART_NUM) {
+    ret = OB_OUT_OF_ELEMENT;
+    OB_LOG(WARN, "Out of cos element ", K(ret), K_(partnum));
+  } else if (OB_UNLIKELY(!is_opened_)) {
+    ret = OB_COS_ERROR;
+    OB_LOG(WARN, "write cos should open first", K(ret));
+  } else {
+    qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer(
+        handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length());
+    qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer(
+        handle_.get_object_name().ptr(), handle_.get_object_name().length());
+    qcloud_cos::CosStringBuffer upload_id_str = qcloud_cos::CosStringBuffer(
+        upload_id_, strlen(upload_id_));
+    if (OB_FAIL(qcloud_cos::ObCosWrapper::upload_part_from_buffer(handle_.get_ptr(), bucket_name,
+        object_name, upload_id_str, partnum_, base_buf_, base_buf_pos_))) {
+      OB_LOG(WARN, "fail to upload part to cos", K(ret), KP_(upload_id));
+      if (OB_TMP_FAIL(cleanup())) {
+        OB_LOG(WARN, "fail to abort multiupload", K(ret), K(tmp_ret), KP_(upload_id));
+      }
+    }
+  }
+  return ret;
+}
+
+int ObStorageCosMultiPartWriter::close()
+{
+  int ret = OB_SUCCESS;
+  int tmp_ret = OB_SUCCESS;
+  ObExternalIOCounterGuard io_guard;
+  const int64_t start_time = ObTimeUtility::current_time();
+  if (OB_UNLIKELY(!is_opened_)) {
+    ret = OB_COS_ERROR;
+    OB_LOG(WARN, "cos writer cannot close before it is opened", K(ret));
+  } else if (0 != base_buf_pos_) {
+    if(OB_SUCCESS != (ret = write_single_part())) {
+      OB_LOG(WARN, "fail to write the last size to cos", K(ret), K_(base_buf_pos));
+      if (OB_TMP_FAIL(cleanup())) {
+        OB_LOG(WARN, "fail to abort multiupload", K(ret), K(tmp_ret), KP_(upload_id));
+      }
+      ret = OB_COS_ERROR;
+    } else {
+      base_buf_pos_ = 0;
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer(
+        handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length());
+    qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer(
+        handle_.get_object_name().ptr(), handle_.get_object_name().length());
+    qcloud_cos::CosStringBuffer upload_id_str = qcloud_cos::CosStringBuffer(
+        upload_id_, strlen(upload_id_));
+
+    if (OB_FAIL(qcloud_cos::ObCosWrapper::complete_multipart_upload(handle_.get_ptr(), bucket_name,
+        object_name, upload_id_str))) {
+      OB_LOG(WARN, "fail to complete multipart upload", K(ret), KP_(upload_id));
+      if (OB_TMP_FAIL(cleanup())) {
+        OB_LOG(WARN, "fail to abort multiupload", K(ret), K(tmp_ret), KP_(upload_id));
+      }
+    }
+  }
+
+  reuse();
+
+  const int64_t total_cost_time = ObTimeUtility::current_time() - start_time;
+  if (total_cost_time > 3 * 1000 * 1000) {
+    OB_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "cos writer close cost too much time", K(total_cost_time), K(ret));
+  }
+  return ret;
+}
+
+int ObStorageCosMultiPartWriter::cleanup()
+{
+  int ret = OB_SUCCESS;
+  if (OB_UNLIKELY(!is_opened_)) {
+    ret = OB_COS_ERROR;
+    OB_LOG(WARN, "cos multipart writer cannot cleanup before it is opened", K(ret));
+  } else {
+    qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer(
+        handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length());
+    qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer(
+        handle_.get_object_name().ptr(), handle_.get_object_name().length());
+    qcloud_cos::CosStringBuffer upload_id_str = qcloud_cos::CosStringBuffer(
+        upload_id_, strlen(upload_id_));
+
+    if (OB_FAIL(qcloud_cos::ObCosWrapper::abort_multipart_upload(handle_.get_ptr(), bucket_name,
+        object_name, upload_id_str))) {
+      OB_LOG(WARN, "fail to abort multipart upload", K(ret), KP_(upload_id));
+    }
+  }
+  return ret;
+}
+
 } //common
 } //oceanbase

deps/oblib/src/lib/restore/ob_storage_cos_base.h

@@ -26,6 +26,7 @@ namespace common
 {
 class ObStorageCosBase;
 struct CosListFilesCbArg;
+struct CosListFilesCtx;
 
 // Before using cos, you need to initialize cos enviroment.
 // Thread safe guaranteed by user.
@@ -44,6 +45,7 @@ public:
   virtual void close();
   virtual int is_exist(const common::ObString &uri, bool &is_exist);
   virtual int get_file_length(const common::ObString &uri, int64_t &file_length);
+  virtual int head_object_meta(const common::ObString &uri, ObStorageObjectMetaBase &obj_meta);
   virtual int write_single_file(const common::ObString &uri, const char *buf,
                                 const int64_t size);
 
@@ -51,9 +53,11 @@ public:
   virtual int mkdir(const common::ObString &uri);
   virtual int del_file(const common::ObString &uri);
   virtual int list_files(const common::ObString &uri, common::ObBaseDirEntryOperator &op);
+  virtual int list_files(const common::ObString &uri, ObStorageListCtxBase &list_ctx);
   virtual int del_dir(const common::ObString &uri);
   virtual int list_directories(const common::ObString &uri, common::ObBaseDirEntryOperator &op);
   virtual int is_tagging(const common::ObString &uri, bool &is_tagging);
+  virtual int del_unmerged_parts(const ObString &uri) override;
 private:
   int get_object_meta_(const common::ObString &uri, bool &is_file_exist, int64_t &file_length);
 
@@ -75,11 +79,14 @@ public:
   // some cos function
   int get_cos_file_meta(bool &is_file_exist, common::qcloud_cos::CosObjectMeta &obj_meta);
   int delete_object(const common::ObString &uri);
+  int list_objects(const common::ObString &uri,
+      const common::ObString &dir_name_str, common::CosListFilesCbArg &arg);
   int list_objects(const common::ObString &uri, const common::ObString &dir_name_str,
-      const char *separator, common::CosListFilesCbArg &arg);
+      const char *next_token, common::CosListFilesCtx &ctx);
   int list_directories(const common::ObString &uri, const common::ObString &dir_name_str,
       const char *next_marker_str, const char *delimiter_str, common::CosListFilesCbArg &arg);
   int is_object_tagging(const common::ObString &uri, bool &is_tagging);
+  int del_unmerged_parts(const ObString &uri);
 
 private:
   int init_handle(const common::ObObjectStorageInfo &storage_info);
@@ -114,13 +121,16 @@ class ObStorageCosReader: public ObStorageCosBase, public ObIStorageReader
 public:
   ObStorageCosReader();
   virtual ~ObStorageCosReader();
-  virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info) override;
-  int pread(char *buf, const int64_t buf_size, int64_t offset, int64_t &read_size);
+  virtual int open(const common::ObString &uri,
+      common::ObObjectStorageInfo *storage_info, const bool head_meta = true) override;
+  virtual int pread(char *buf,
+      const int64_t buf_size, const int64_t offset, int64_t &read_size) override;
   int close();
   int64_t get_length() const { return file_length_; }
   bool is_opened() const { return is_opened_; }
 
 private:
+  bool has_meta_;
   int64_t file_length_;
 
   DISALLOW_COPY_AND_ASSIGN(ObStorageCosReader);
@@ -149,6 +159,42 @@ private:
   DISALLOW_COPY_AND_ASSIGN(ObStorageCosAppendWriter);
 };
 
+// part size is in [1MB, 5GB], exclude the last part
+// max part num 10000
+class ObStorageCosMultiPartWriter: public ObStorageCosBase, public ObIStorageWriter
+{
+public:
+  ObStorageCosMultiPartWriter();
+  virtual ~ObStorageCosMultiPartWriter();
+  int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info);
+  int write(const char *buf,const int64_t size);
+  int pwrite(const char *buf, const int64_t size, const int64_t offset);
+  int close();
+  int cleanup();
+  int64_t get_length() const { return file_length_; }
+  virtual bool is_opened() const { return is_opened_; }
+
+private:
+  void reuse();
+  void destroy() { reuse(); }
+  int write_single_part();
+
+private:
+  const static int64_t COS_MAX_PART_NUM = 10000;
+  const static int64_t COS_MULTIPART_UPLOAD_BUF_SIZE = 8 * 1024 * 1024L;
+
+private:
+  common::ModulePageAllocator mod_;
+  common::ModuleArena allocator_;
+  char *base_buf_;
+  int64_t base_buf_pos_;
+  char *upload_id_;
+  int partnum_;
+  int64_t file_length_;
+
+  DISALLOW_COPY_AND_ASSIGN(ObStorageCosMultiPartWriter);
+};
+
 } //common
 } //oceanbase
 #endif

deps/oblib/src/lib/restore/ob_storage_file.cpp

@@ -51,10 +51,8 @@ int get_file_path(const common::ObString &uri, char *buf, const int64_t buf_size
   } else if (!uri.prefix_match(OB_FILE_PREFIX)) {
     ret = OB_INVALID_ARGUMENT;
     STORAGE_LOG(WARN, "invalid uri", K(ret), K(uri));
-  } else if (OB_FAIL(databuff_printf(
-                         buf, buf_size, "%.*s",
-                         static_cast<int>(uri.length() - offset),
-                         uri.ptr() + offset))) {
+  } else if (OB_FAIL(databuff_printf(buf, buf_size, "%.*s",
+             static_cast<int>(uri.length() - offset), uri.ptr() + offset))) {
     STORAGE_LOG(WARN, "failed to fill path", K(ret), K(uri));
   } else if (strlen(buf) <= 0 && buf[0] != '/') {
     ret = OB_INVALID_ARGUMENT;
@@ -170,6 +168,39 @@ int ObStorageFileUtil::get_file_length(const common::ObString &uri, int64_t &fil
   return ret;
 }
 
+int ObStorageFileUtil::head_object_meta(const common::ObString &uri, ObStorageObjectMetaBase &obj_meta)
+{
+  int ret = OB_SUCCESS;
+  char path[OB_MAX_URI_LENGTH];
+  char errno_buf[OB_MAX_ERROR_MSG_LEN] = "";
+  struct stat64 file_info;
+  obj_meta.reset();
+
+  if (uri.empty()) {
+    ret = OB_INVALID_ARGUMENT;
+    STORAGE_LOG(WARN, "invalid args", K(ret), K(uri));
+  } else if (OB_FAIL(get_file_path(uri, path, sizeof(path)))) {
+    STORAGE_LOG(WARN, "failed to fill path", K(ret), K(uri));
+  } else if (0 != ::stat64(path, &file_info)) {
+    if (ENOENT == errno || ENOTDIR == errno) {
+    } else {
+      convert_io_error(errno, ret);
+      STORAGE_LOG(WARN, "file not exist",
+          K(ret), K(path), K(errno), "errno", strerror_r(errno, errno_buf, sizeof(errno_buf)));
+    }
+  } else if (S_ISDIR(file_info.st_mode)) {
+    obj_meta.is_exist_ = true;
+    obj_meta.length_ = -1;
+    obj_meta.type_ = ObStorageObjectType::OB_FS_DIR;
+  } else {
+    obj_meta.is_exist_ = true;
+    obj_meta.length_ = file_info.st_size;
+    obj_meta.type_ = ObStorageObjectType::OB_FS_FILE;
+  }
+
+  return ret;
+}
+
 int ObStorageFileUtil::del_file(const common::ObString &uri)
 {
   int ret = OB_SUCCESS;
@@ -226,23 +257,33 @@ int ObStorageFileUtil::mkdir(const common::ObString &uri)
     // find the first not exist dir
     int64_t pos = 0;
     bool found_exist_dir = false;
-    for (pos = strlen(path) - 1; OB_SUCC(ret) && pos >= 0 && !found_exist_dir; --pos) {
-      if (path[pos] == '/') {
-        path[pos] = '\0';
+    const int64_t end_pos = strlen(path) - 1;
+    for (pos = end_pos; OB_SUCC(ret) && pos >= 0 && !found_exist_dir; --pos) {
+      if (path[pos] == '/' || pos == end_pos) {
+        if (pos != end_pos) {
+          path[pos] = '\0';
+        }
+
         if (0 == ::access(path, F_OK)) {
           STORAGE_LOG(INFO, "path exist", K(pos), KCSTRING(path));
           found_exist_dir = true;
+        } else if (ENOTDIR == errno) {
+          ret = OB_FILE_ALREADY_EXIST;
+          STORAGE_LOG(WARN, "already exist the same name file", K(ret), KCSTRING(path),
+            K(errno), "errno", strerror_r(errno, errno_buf, sizeof(errno_buf)));
         } else if (ENOENT != errno) {
           ret = OB_IO_ERROR;
           STORAGE_LOG(WARN, "check is parent dir exist",
               K(ret), KCSTRING(path), K(errno), "errno", strerror_r(errno, errno_buf, sizeof(errno_buf)));
         }
-        path[pos] = '/';
+
+        if (pos != end_pos) {
+          path[pos] = '/';
+        }
       }
     }
 
     // create dirs
-    int64_t end_pos = strlen(path) - 1;
     for (int64_t i = pos + 1; OB_SUCC(ret) && i <= end_pos; ++i) {
       if (path[i] == '/' || end_pos == i) {
         if (end_pos != i) {
@@ -287,7 +328,6 @@ int ObStorageFileUtil::list_files(const common::ObString &uri, common::ObBaseDir
   struct dirent *result;
   DIR *open_dir = NULL;
   char errno_buf[OB_MAX_ERROR_MSG_LEN] = "";
-  ObString tmp_string;
   ObString file_name;
   char dir_path[OB_MAX_URI_LENGTH] = "";
   char sub_dir_path[OB_MAX_URI_LENGTH] = "";
@@ -308,7 +348,6 @@ int ObStorageFileUtil::list_files(const common::ObString &uri, common::ObBaseDir
   }
 
   while (OB_SUCC(ret) && NULL != open_dir) {
-    tmp_string.reset();
     file_name.reset();
     is_file = false;
     size  = 0;
@@ -360,6 +399,142 @@ int ObStorageFileUtil::list_files(const common::ObString &uri, common::ObBaseDir
   return ret;
 }
 
+int ObStorageFileUtil::list_files(const common::ObString &uri, ObStorageListCtxBase &ctx_base)
+{
+  int ret = OB_SUCCESS;
+  char dir_path[OB_MAX_URI_LENGTH] = "";
+  ObStorageListFilesCtx &list_ctx = static_cast<ObStorageListFilesCtx &>(ctx_base);
+
+  if (OB_UNLIKELY(uri.empty() || !list_ctx.is_valid())) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid argument", K(ret), K(list_ctx), KCSTRING(dir_path));
+  } else if (OB_FAIL(get_file_path(uri, dir_path, sizeof(dir_path)))) {
+    STORAGE_LOG(WARN, "failed to fill path", K(ret), K(uri));
+  } else {
+    list_ctx.rsp_num_ = 0;
+    list_ctx.has_next_ = false;
+
+    struct dirent *result;
+    char errno_buf[OB_MAX_ERROR_MSG_LEN] = "";
+    ObString file_name;
+    char sub_dir_path[OB_MAX_URI_LENGTH] = "";
+    bool is_file = false;
+    bool is_appendable_file = false;
+    int64_t size = 0;
+
+    if (!list_ctx.already_open_dir_) {
+      if (OB_ISNULL(list_ctx.open_dir_ = ::opendir(dir_path))) {
+        if (ENOENT != errno) {
+          ret = OB_IO_ERROR;
+          OB_LOG(WARN, "fail to open dir", K(ret), KCSTRING(dir_path),
+              KCSTRING(strerror_r(errno, errno_buf, sizeof(errno_buf))));
+        }
+      } else {
+        list_ctx.already_open_dir_ = true;
+      }
+    }
+
+    while (OB_SUCC(ret) && NULL != list_ctx.open_dir_) {
+      file_name.reset();
+      is_file = false;
+      is_appendable_file = false;
+      size = 0;
+      if (list_ctx.rsp_num_ >= list_ctx.max_list_num_) {
+        list_ctx.has_next_ = true;
+        break;
+      }
+
+      if (0 != ::readdir_r(list_ctx.open_dir_, &(list_ctx.next_entry_), &result)) {
+        ret = OB_IO_ERROR;
+        OB_LOG(WARN, "read dir error", K(ret),
+            KCSTRING(strerror_r(errno, errno_buf, sizeof(errno_buf))));
+      } else if (NULL != result) {
+        if (0 == strcmp(list_ctx.next_entry_.d_name, ".") || 0 == strcmp(list_ctx.next_entry_.d_name, "..")) {
+          // not a file
+        } else if (DT_DIR == list_ctx.next_entry_.d_type) {
+          if (OB_FAIL(check_is_appendable(uri, list_ctx.next_entry_, is_appendable_file))) {
+            OB_LOG(WARN, "fail to check is_appendable", K(ret), K(uri));
+          } else if (is_appendable_file) {
+            is_file = true;
+          }
+        } else if (DT_REG == list_ctx.next_entry_.d_type && !list_ctx.need_size_) {
+          is_file = true;
+        } else if ((DT_REG == list_ctx.next_entry_.d_type && list_ctx.need_size_) || DT_UNKNOWN == list_ctx.next_entry_.d_type) {
+          int pret = snprintf(sub_dir_path, OB_MAX_URI_LENGTH, "%s/%s", dir_path, list_ctx.next_entry_.d_name);
+          if (pret < 0 || pret >= OB_MAX_URI_LENGTH) {
+            ret = OB_BUF_NOT_ENOUGH;
+            OB_LOG(WARN, "format dir path fail", K(ret), KCSTRING(dir_path));
+          } else {
+            struct stat sb;
+            if (-1 == ::stat(sub_dir_path, &sb)) {
+              ret = OB_IO_ERROR;
+              OB_LOG(WARN, "stat fail", K(ret), KCSTRING(strerror_r(errno, errno_buf, sizeof(errno_buf))));
+            } else if (!S_ISREG(sb.st_mode)) {
+              // not a file
+            } else {
+              is_file = true;
+              size = static_cast<int64_t>(sb.st_size);
+            }
+          }
+        }
+
+        if (OB_SUCC(ret) && is_file) {
+          const int64_t name_len = strlen(list_ctx.next_entry_.d_name);
+          if (name_len >= list_ctx.max_name_len_) {
+            ret = OB_SIZE_OVERFLOW;
+            OB_LOG(WARN, "can't hold the dentry name", K(ret), K(list_ctx.max_name_len_), K(name_len));
+          } else {
+            MEMCPY(list_ctx.name_arr_[list_ctx.rsp_num_], list_ctx.next_entry_.d_name, name_len);
+            if (is_appendable_file) {
+              list_ctx.name_arr_[list_ctx.rsp_num_][name_len] = '/';
+              list_ctx.name_arr_[list_ctx.rsp_num_][name_len + 1] = '\0';
+            } else {
+              list_ctx.name_arr_[list_ctx.rsp_num_][name_len] = '\0';
+            }
+            if (list_ctx.need_size_) {
+              list_ctx.size_arr_[list_ctx.rsp_num_] = size;
+            }
+            ++list_ctx.rsp_num_;
+          }
+        }
+      } else {
+        list_ctx.has_next_ = false;
+        break; // end file
+      }
+    }
+    // close dir
+    if (!list_ctx.has_next_ && list_ctx.already_open_dir_ && NULL != list_ctx.open_dir_) {
+      ::closedir(list_ctx.open_dir_);
+    }
+  }
+
+  return ret;
+}
+
+int ObStorageFileUtil::check_is_appendable(
+    const common::ObString &uri,
+    struct dirent &cur_entry,
+    bool &is_appendable_file)
+{
+  int ret = OB_SUCCESS;
+  ObStorageObjectMetaBase obj_meta;
+  char tmp_uri_buf[OB_MAX_URI_LENGTH] = "";
+  int pos = snprintf(tmp_uri_buf, OB_MAX_URI_LENGTH, "%s/%s/%s%s", uri.ptr(), cur_entry.d_name,
+                     OB_S3_APPENDABLE_FRAGMENT_PREFIX, OB_S3_APPENDABLE_FORMAT_META);
+  if (pos < 0 || pos >= OB_MAX_URI_LENGTH) {
+    ret = OB_BUF_NOT_ENOUGH;
+    OB_LOG(WARN, "fail to build format meta file path", K(ret), K(pos), K(uri), KCSTRING(cur_entry.d_name));
+  } else {
+    common::ObString format_meta_uri(pos, tmp_uri_buf);
+    if (OB_FAIL(head_object_meta(format_meta_uri, obj_meta))) {
+      OB_LOG(WARN, "fail to head object meta", K(ret), K(format_meta_uri));
+    } else {
+      is_appendable_file = obj_meta.is_exist_;
+    }
+  }
+  return ret;
+}
+
 int ObStorageFileUtil::del_dir(const common::ObString &uri)
 {
   int ret = OB_SUCCESS;
@@ -518,6 +693,13 @@ int ObStorageFileUtil::is_tagging(const common::ObString &uri, bool &is_tagging)
   return ret;
 }
 
+int ObStorageFileUtil::del_unmerged_parts(const ObString &uri)
+{
+  UNUSED(uri);
+  int ret = OB_NOT_SUPPORTED;
+  return ret;
+}
+
 ObStorageFileReader::ObStorageFileReader()
   : fd_(-1),
     is_opened_(false),
@@ -537,7 +719,8 @@ ObStorageFileReader::~ObStorageFileReader()
   }
 }
 
-int ObStorageFileReader::open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info)
+int ObStorageFileReader::open(const common::ObString &uri,
+    common::ObObjectStorageInfo *storage_info, const bool head_meta)
 {
   int ret = OB_SUCCESS;
   int tmp_ret = OB_SUCCESS;
@@ -555,19 +738,22 @@ int ObStorageFileReader::open(const common::ObString &uri, common::ObObjectStora
     STORAGE_LOG(WARN, "failed to open read file",
         K(ret), KCSTRING(path_), K(errno), "errno", strerror_r(errno, errno_buf, sizeof(errno_buf)));
   } else {
-    is_opened_ = true;
-    if (0 != ::fstat64(fd_, &file_info)) {
-      ret = OB_IO_ERROR;
-      STORAGE_LOG(WARN, "file not exist",
-          K(ret), K(fd_), KCSTRING(path_), K(errno), "errno", strerror_r(errno, errno_buf, sizeof(errno_buf)));
-    } else if (S_ISDIR(file_info.st_mode)) {
-      ret = OB_IO_ERROR;
-      STORAGE_LOG(WARN, "uri is a dir", K(ret), KCSTRING(path_));
-    } else {
-      file_length_ = file_info.st_size;
+    if (head_meta) {
+      if (0 != ::fstat64(fd_, &file_info)) {
+        convert_io_error(errno, ret);
+        STORAGE_LOG(WARN, "file not exist",
+            K(ret), K(fd_), KCSTRING(path_), K(errno), "errno", strerror_r(errno, errno_buf, sizeof(errno_buf)));
+      } else if (S_ISDIR(file_info.st_mode)) {
+        ret = OB_IO_ERROR;
+        STORAGE_LOG(WARN, "uri is a dir", K(ret), KCSTRING(path_));
+      } else {
+        file_length_ = file_info.st_size;
+      }
     }
 
-    if (OB_FAIL(ret)) {
+    if (OB_SUCC(ret)) {
+      is_opened_ = true;
+    } else {
       if (OB_SUCCESS != (tmp_ret = close())) {
         STORAGE_LOG(WARN, "failed to close", K(ret), K(tmp_ret), KCSTRING(path_));
       }
@@ -577,7 +763,7 @@ int ObStorageFileReader::open(const common::ObString &uri, common::ObObjectStora
 }
 
 int ObStorageFileReader::pread(
-    char *buf,const int64_t buf_size, int64_t offset, int64_t &read_size)
+    char *buf, const int64_t buf_size, const int64_t offset, int64_t &read_size)
 {
   int ret = OB_SUCCESS;
   char errno_buf[OB_MAX_ERROR_MSG_LEN] = "";
@@ -614,11 +800,8 @@ int ObStorageFileReader::close()
   int ret = OB_SUCCESS;
   char errno_buf[OB_MAX_ERROR_MSG_LEN] = "";
 
-  if (!is_opened_) {
-    ret = OB_NOT_INIT;
-    STORAGE_LOG(WARN, "not opened", K(ret), K(fd_));
-  } else if (0 != ::close(fd_)) {
-    ret = OB_IO_ERROR;
+  if (is_opened_ && 0 != ::close(fd_)) {
+    convert_io_error(errno, ret);
     STORAGE_LOG(WARN, "failed to close read file",
         K(ret), KCSTRING(path_), K(errno), "errno", strerror_r(errno, errno_buf, sizeof(errno_buf)));
   }
@@ -851,6 +1034,13 @@ int ObStorageFileWriter::close()
 #endif
     if (has_error_) {
       STORAGE_LOG(WARN, "writer has error, skip rename file", KCSTRING(path_), KCSTRING(real_path_));
+
+      // has error, try delete file regardless of whether the temporary file exists
+      if (0 != ::remove(path_)) {
+        tmp_ret = OB_IO_ERROR;
+        STORAGE_LOG(WARN, "failed to remove file", K(tmp_ret), KCSTRING(path_),
+            K(errno), "errno", strerror_r(errno, errno_buf, sizeof(errno_buf)));
+      }
     } else if (0 != ::rename(path_, real_path_)) {
       ret = OB_IO_ERROR;
       STORAGE_LOG(WARN, "failed to rename meta file",
@@ -865,6 +1055,7 @@ int ObStorageFileWriter::close()
       STORAGE_LOG(INFO, "succeed to rename file after close", KCSTRING(path_), KCSTRING(real_path_));
     }
   }
+  has_error_ = false;
   return ret;
 }
 
@@ -976,5 +1167,11 @@ int ObStorageFileAppender::get_open_flag_and_mode_(int &flag, bool &need_lock)
   return ret;
 }
 
+int ObStorageFileMultiPartWriter::pwrite(const char *buf, const int64_t size, const int64_t offset)
+{
+  UNUSED(offset);
+  return write(buf, size);
+}
+
 }//common
 }//oceanbase

deps/oblib/src/lib/restore/ob_storage_file.h

@@ -34,15 +34,19 @@ public:
 
   virtual int is_exist(const common::ObString &uri, bool &exist);
   virtual int get_file_length(const common::ObString &uri, int64_t &file_length);
+  virtual int head_object_meta(const common::ObString &uri, ObStorageObjectMetaBase &obj_meta);
   virtual int del_file(const common::ObString &uri);
   virtual int write_single_file(const common::ObString &uri, const char *buf, const int64_t size);
   virtual int mkdir(const common::ObString &uri);
   virtual int list_files(const common::ObString &uri, common::ObBaseDirEntryOperator &op);
+  virtual int list_files(const common::ObString &uri, ObStorageListCtxBase &list_ctx);
   virtual int del_dir(const common::ObString &uri);
   virtual int list_directories(const common::ObString &uri, common::ObBaseDirEntryOperator &op);
   virtual int is_tagging(const common::ObString &uri, bool &is_tagging);
+  virtual int del_unmerged_parts(const ObString &uri) override;
 private:
   int get_tmp_file_format_timestamp(const char *file_name, bool &is_tmp_file, int64_t &timestamp);
+  int check_is_appendable(const common::ObString &uri, struct dirent &entry, bool &is_appendable_file);
 
 private:
   DISALLOW_COPY_AND_ASSIGN(ObStorageFileUtil);
@@ -53,11 +57,13 @@ class ObStorageFileReader: public ObIStorageReader
 public:
   ObStorageFileReader();
   virtual ~ObStorageFileReader();
-  virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info = NULL);
-  virtual int pread(char *buf,const int64_t buf_size, int64_t offset, int64_t &read_size);
-  virtual int close();
-  virtual int64_t get_length() const { return file_length_; }
-  virtual bool is_opened() const { return is_opened_; }
+  virtual int open(const common::ObString &uri,
+      common::ObObjectStorageInfo *storage_info = NULL, const bool head_meta = true) override;
+  virtual int pread(char *buf,
+      const int64_t buf_size, const int64_t offset, int64_t &read_size) override;
+  virtual int close() override;
+  virtual int64_t get_length() const override { return file_length_; }
+  virtual bool is_opened() const override { return is_opened_; }
 private:
   int fd_;
   bool is_opened_;
@@ -121,6 +127,17 @@ private:
   DISALLOW_COPY_AND_ASSIGN(ObStorageFileAppender);
 };
 
+class ObStorageFileMultiPartWriter : public ObStorageFileWriter
+{
+public:
+  ObStorageFileMultiPartWriter() {}
+  virtual ~ObStorageFileMultiPartWriter() {}
+  virtual int pwrite(const char *buf, const int64_t size, const int64_t offset) override;
+
+private:
+  DISALLOW_COPY_AND_ASSIGN(ObStorageFileMultiPartWriter);
+};
+
 
 }//common
 }//oceanbase

deps/oblib/src/lib/restore/ob_storage_info.cpp

@@ -87,9 +87,10 @@ ObStorageType ObObjectStorageInfo::get_type() const
 
 // oss:host=xxxx&access_id=xxx&access_key=xxx
 // cos:host=xxxx&access_id=xxx&access_key=xxxappid=xxx
+// s3:host=xxxx&access_id=xxx&access_key=xxx&s3_region=xxx
 int ObObjectStorageInfo::set(const common::ObStorageType device_type, const char *storage_info)
 {
-  bool has_appid = false;
+  bool has_needed_extension = false;
   int ret = OB_SUCCESS;
   if (is_valid()) {
     ret = OB_INIT_TWICE;
@@ -103,16 +104,19 @@ int ObObjectStorageInfo::set(const common::ObStorageType device_type, const char
       ret = OB_INVALID_BACKUP_DEST;
       LOG_WARN("storage info is empty", K(ret), K_(device_type));
     }
-  } else if (OB_FAIL(parse_storage_info_(storage_info, has_appid))) {
+  } else if (OB_FAIL(parse_storage_info_(storage_info, has_needed_extension))) {
     LOG_WARN("parse storage info failed", K(ret));
   } else if (OB_STORAGE_FILE != device_type
       && (0 == strlen(endpoint_) || 0 == strlen(access_id_) || 0 == strlen(access_key_))) {
     ret = OB_INVALID_BACKUP_DEST;
     LOG_WARN("backup device is not nfs, endpoint/access_id/access_key do not allow to be empty",
         K(ret), K_(device_type), K_(endpoint), K_(access_id));
-  } else if (OB_STORAGE_COS == device_type && !has_appid) {
+  } else if (OB_STORAGE_COS == device_type && !has_needed_extension) {
     ret = OB_INVALID_BACKUP_DEST;
     LOG_WARN("invalid cos info, appid do not allow to be empty", K(ret), K_(extension));
+  } else if (OB_STORAGE_S3 == device_type && !has_needed_extension) {
+    ret = OB_INVALID_BACKUP_DEST;
+    LOG_WARN("invalid s3 info, region do not allow to be empty", K(ret), K_(extension));
   } else if (OB_STORAGE_FILE == device_type
       && (0 != strlen(endpoint_) || 0 != strlen(access_id_) || 0 != strlen(access_key_))) {
     ret = OB_INVALID_BACKUP_DEST;
@@ -143,10 +147,10 @@ int ObObjectStorageInfo::set(const char *uri, const char *storage_info)
   return ret;
 }
 
-int ObObjectStorageInfo::parse_storage_info_(const char *storage_info, bool &has_appid)
+int ObObjectStorageInfo::parse_storage_info_(const char *storage_info, bool &has_needed_extension)
 {
   int ret = OB_SUCCESS;
-
+  has_needed_extension = false;
   if (OB_ISNULL(storage_info) || strlen(storage_info) >= OB_MAX_BACKUP_STORAGE_INFO_LENGTH) {
     ret = OB_INVALID_BACKUP_DEST;
     LOG_WARN("storage info is invalid", K(ret), K(storage_info), K(strlen(storage_info)));
@@ -163,9 +167,14 @@ int ObObjectStorageInfo::parse_storage_info_(const char *storage_info, bool &has
       token = ::strtok_r(str, "&", &saved_ptr);
       if (NULL == token) {
         break;
+      } else if (0 == strncmp(REGION, token, strlen(REGION))) {
+        has_needed_extension = (OB_STORAGE_S3 == device_type_);
+        if (OB_FAIL(set_storage_info_field_(token, extension_, sizeof(extension_)))) {
+          LOG_WARN("failed to set region", K(ret), K(token));
+        }
       } else if (0 == strncmp(HOST, token, strlen(HOST))) {
         if (OB_FAIL(set_storage_info_field_(token, endpoint_, sizeof(endpoint_)))) {
-          LOG_WARN("failed to set endpoint",K(ret), K(token));
+          LOG_WARN("failed to set endpoint", K(ret), K(token));
         }
       } else if (0 == strncmp(ACCESS_ID, token, strlen(ACCESS_ID))) {
         if (OB_FAIL(set_storage_info_field_(token, access_id_, sizeof(access_id_)))) {
@@ -176,7 +185,7 @@ int ObObjectStorageInfo::parse_storage_info_(const char *storage_info, bool &has
           LOG_WARN("failed to set access key", K(ret), K(token));
         }
       } else if (OB_STORAGE_FILE != device_type_ && 0 == strncmp(APPID, token, strlen(APPID))) {
-        has_appid = true;
+        has_needed_extension = (OB_STORAGE_COS == device_type_);
         if (OB_FAIL(set_storage_info_field_(token, extension_, sizeof(extension_)))) {
           LOG_WARN("failed to set appid", K(ret), K(token));
         }
@@ -267,8 +276,7 @@ int ObObjectStorageInfo::get_storage_info_str(char *storage_info, const int64_t
   }
 
   if (OB_SUCC(ret) && 0 != strlen(extension_) && info_len > strlen(storage_info)) {
-    // if OB_STORAGE_FILE's extension is not empty
-    // delimiter should be included
+    // if OB_STORAGE_FILE's extension is not empty, delimiter should be included
     int64_t str_len = strlen(storage_info);
     if (str_len > 0 && OB_FAIL(databuff_printf(storage_info, info_len, str_len, "&"))) {
       LOG_WARN("failed to add delimiter to storage info", K(ret), K(info_len), K(str_len));

deps/oblib/src/lib/restore/ob_storage_info.h

@@ -38,6 +38,7 @@ const char *const ACCESS_KEY = "access_key=";
 const char *const HOST = "host=";
 const char *const APPID = "appid=";
 const char *const DELETE_MODE = "delete_mode=";
+const char *const REGION = "s3_region=";
 
 class ObObjectStorageInfo
 {

deps/oblib/src/lib/restore/ob_storage_oss_base.h

@@ -51,6 +51,7 @@ const static int64_t MAX_OSS_KEY_LENGTH = 128;
 const static int64_t OSS_BASE_BUFFER_SIZE = 8 * 1024 * 1024L;//the buf size of upload data
 const static int64_t MAX_ELEMENT_COUNT = 10000;//oss limit element count
 const static int64_t MULTI_BASE_BUFFER_SIZE = 16 * 1024 * 1024L;//the buf size of upload data
+static constexpr char OB_STORAGE_OSS_ALLOCATOR[] = "StorageOSS";
 
 // Before using oss, you need to initialize oss enviroment.
 // Thread safe guaranteed by user.
@@ -60,7 +61,6 @@ int init_oss_env();
 // Thread safe guaranteed by user.
 void fin_oss_env();
 
-
 class ObStorageOssStaticVar
 {
 public:
@@ -184,7 +184,6 @@ private:
   common::ObString object_;
   aos_string_t upload_id_;
   int partnum_;
-  MD5_CTX whole_file_md5_;
   bool is_opened_;
   int64_t file_length_;
 
@@ -197,8 +196,10 @@ class ObStorageOssReader: public ObStorageOssBase, public ObIStorageReader
 public:
   ObStorageOssReader();
   virtual ~ObStorageOssReader();
-  int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info);
-  int pread(char *buf,const int64_t buf_size, int64_t offset, int64_t &read_size);
+  virtual int open(const common::ObString &uri,
+      common::ObObjectStorageInfo *storage_info, const bool head_meta = true) override;
+  virtual int pread(char *buf,
+      const int64_t buf_size, const int64_t offset, int64_t &read_size) override;
   int close();
   int64_t get_length() const { return file_length_; }
   virtual bool is_opened() const { return is_opened_; }
@@ -208,6 +209,7 @@ private:
   common::ObString object_;
   int64_t file_length_;
   bool is_opened_;
+  bool has_meta_;
   common::ObArenaAllocator allocator_;
 
   DISALLOW_COPY_AND_ASSIGN(ObStorageOssReader);
@@ -222,16 +224,19 @@ public:
   virtual void close();
   virtual int is_exist(const common::ObString &uri, bool &exist);
   virtual int get_file_length(const common::ObString &uri, int64_t &file_length);
+  virtual int head_object_meta(const common::ObString &uri, ObStorageObjectMetaBase &obj_meta);
   virtual int write_single_file(const common::ObString &uri, const char *buf,
                                 const int64_t size);
 
   //oss no dir
   virtual int mkdir(const common::ObString &uri);
   virtual int del_file(const common::ObString &uri);
-  virtual int list_files(const common::ObString &dir_path, common::ObBaseDirEntryOperator &op);
+  virtual int list_files(const common::ObString &uri, common::ObBaseDirEntryOperator &op);
+  virtual int list_files(const common::ObString &uri, ObStorageListCtxBase &list_ctx);
   virtual int del_dir(const common::ObString &uri);
   virtual int list_directories(const common::ObString &uri, common::ObBaseDirEntryOperator &op);
   virtual int is_tagging(const common::ObString &uri, bool &is_tagging);
+  virtual int del_unmerged_parts(const ObString &uri) override;
 private:
   int strtotime(const char *date_time, int64_t &time);
   int tagging_object_(
@@ -244,6 +249,10 @@ private:
       ObStorageOssBase &oss_base,
       const common::ObString &bucket_str,
       const common::ObString &object_str);
+  int do_list_(ObStorageOssBase &oss_base,
+      const ObString &bucket, const char *full_dir_path,
+      const int64_t max_ret, const char *delimiter,
+      const char *next_marker, oss_list_object_params_t *&params);
   bool is_opened_;
   common::ObObjectStorageInfo *storage_info_;
 };

deps/oblib/src/lib/restore/ob_storage_s3_base.h

@@ -0,0 +1,547 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef SRC_LIBRARY_SRC_LIB_RESTORE_OB_STORAGE_S3_BASE_H_
+#define SRC_LIBRARY_SRC_LIB_RESTORE_OB_STORAGE_S3_BASE_H_
+
+#include <openssl/md5.h>
+#include "lib/restore/ob_i_storage.h"
+#include "lib/container/ob_array.h"
+#include "lib/container/ob_se_array.h"
+#include "lib/container/ob_array_iterator.h"
+#include "lib/container/ob_se_array_iterator.h"
+#include <algorithm>
+#include <iostream>
+
+#pragma push_macro("private")
+#undef private
+#include <aws/core/Aws.h>
+#include <aws/s3/S3Client.h>
+#include <aws/core/auth/AWSCredentials.h>
+#include <aws/core/utils/Outcome.h>
+#include <aws/s3/model/HeadObjectRequest.h>
+#include <aws/s3/model/PutObjectRequest.h>
+#include <aws/s3/model/GetObjectRequest.h>
+#include <aws/s3/model/DeleteObjectRequest.h>
+#include <aws/s3/model/PutObjectTaggingRequest.h>
+#include <aws/s3/model/GetObjectTaggingRequest.h>
+#include <aws/s3/model/ListObjectsV2Request.h>
+#include <aws/s3/model/ListObjectsRequest.h>
+#include <aws/s3/model/CreateMultipartUploadRequest.h>
+#include <aws/s3/model/UploadPartRequest.h>
+#include <aws/s3/model/CompletedMultipartUpload.h>
+#include <aws/s3/model/ListPartsRequest.h>
+#include <aws/s3/model/CompleteMultipartUploadRequest.h>
+#include <aws/s3/model/CopyObjectRequest.h>
+#include <aws/s3/model/AbortMultipartUploadRequest.h>
+#include <aws/s3/model/ListMultipartUploadsRequest.h>
+#include <aws/core/utils/ratelimiter/DefaultRateLimiter.h>
+#include <aws/core/utils/HashingUtils.h>
+#include <aws/core/utils/crypto/CRC32.h>
+#pragma pop_macro("private")
+
+namespace oceanbase
+{
+namespace common
+{
+
+// Before using s3, you need to initialize s3 enviroment.
+// Thread safe guaranteed by user.
+int init_s3_env();
+
+// You need to clean s3 resource when not use cos any more.
+// Thread safe guaranteed by user.
+void fin_s3_env();
+
+// default s3 checksum algorithm
+static ObStorageCRCAlgorithm default_s3_crc_algo = ObStorageCRCAlgorithm::OB_INVALID_CRC_ALGO;
+// set checksum algorithm for writing object into s3
+void set_s3_checksum_algorithm(const ObStorageCRCAlgorithm crc_algo);
+// get current checksum algorithm
+ObStorageCRCAlgorithm get_s3_checksum_algorithm();
+
+static constexpr int64_t S3_CONNECT_TIMEOUT_MS = 10 * 1000;
+static constexpr int64_t S3_REQUEST_TIMEOUT_MS = 10 * 1000;
+static constexpr int64_t MAX_S3_CONNECTIONS_PER_CLIENT = 128;
+static constexpr int64_t STOP_S3_TIMEOUT_US = 10 * 1000L;   // 10ms
+// max allowed idle duration for a s3 client: 12h
+static constexpr int64_t MAX_S3_CLIENT_IDLE_DURATION = 12 * 3600 * 1000 * 1000L;
+static constexpr int64_t MAX_S3_CLIENT_MAP_THRESHOLD = 500;
+
+// TODO: check length
+static constexpr int MAX_S3_REGION_LENGTH = 128;
+static constexpr int MAX_S3_ENDPOINT_LENGTH = 128;
+static constexpr int MAX_S3_ACCESS_ID_LENGTH = 128;   // ak, access key id
+static constexpr int MAX_S3_SECRET_KEY_LENGTH = 128;  // sk, secret key
+static constexpr int MAX_S3_CLIENT_NUM = 97;
+static constexpr int MAX_S3_PART_NUM = 10000;
+static constexpr int64_t S3_MULTIPART_UPLOAD_BUFFER_SIZE = 8 * 1024 * 1024L;
+
+static constexpr char OB_S3_APPENDABLE_FORMAT_CONTENT_V1[] = "version=1";
+static constexpr char OB_STORAGE_S3_ALLOCATOR[] = "StorageS3";
+static constexpr char S3_SDK[] = "S3SDK";
+
+struct ObS3Account
+{
+  ObS3Account();
+  ~ObS3Account();
+  void reset();
+  bool is_valid() const { return is_valid_; }
+  int64_t hash() const;
+  TO_STRING_KV(K_(is_valid), K_(delete_mode), K_(region), K_(endpoint), K_(access_id));
+
+  int parse_from(const char *storage_info_str, const int64_t size);
+  int set_field(const char *value, char *field, const uint32_t field_length);
+
+  bool is_valid_;
+  int64_t delete_mode_;
+  char region_[MAX_S3_REGION_LENGTH];           // region of endpoint
+  char endpoint_[MAX_S3_ENDPOINT_LENGTH];
+  char access_id_[MAX_S3_ACCESS_ID_LENGTH];     // ak
+  char secret_key_[MAX_S3_SECRET_KEY_LENGTH];   // sk
+};
+
+class ObS3MemoryManager : public Aws::Utils::Memory::MemorySystemInterface
+{
+public:
+  ObS3MemoryManager() : attr_()
+  {
+    attr_.label_ = S3_SDK;
+  }
+  virtual ~ObS3MemoryManager() {}
+  // when aws init/shutdown, it will execute like this: init/shutdown_memory_system->Begin()/End()
+  virtual void Begin() override {}
+  virtual void End() override {}
+
+  virtual void *AllocateMemory(std::size_t blockSize,
+      std::size_t alignment, const char *allocationTag = NULL) override
+  {
+    UNUSED(allocationTag);
+    std::size_t real_alignment = MAX(alignment, 16); // should not be smaller than 16
+    return ob_malloc_align(real_alignment, blockSize, attr_);
+  }
+  virtual void FreeMemory(void *memoryPtr) override
+  {
+    ob_free_align(memoryPtr);
+    memoryPtr = NULL;
+  }
+
+private:
+  ObMemAttr attr_;
+};
+
+class ObS3Logger : public Aws::Utils::Logging::LogSystemInterface
+{
+public:
+  ObS3Logger() {}
+  virtual ~ObS3Logger() {}
+  // Gets the currently configured log level for this logger.
+  virtual Aws::Utils::Logging::LogLevel GetLogLevel(void) const override;
+  // Does a printf style output to the output stream. Don't use this, it's unsafe. See LogStream
+  virtual void Log(Aws::Utils::Logging::LogLevel logLevel, const char* tag, const char* formatStr, ...) override;
+  // Writes the stream to the output stream.
+  virtual void LogStream(Aws::Utils::Logging::LogLevel logLevel, const char* tag, const Aws::OStringStream &messageStream) override;
+  // Writes any buffered messages to the underlying device if the logger supports buffering.
+  virtual void Flush() override {}
+};
+
+class ObS3Client
+{
+public:
+  ObS3Client();
+  virtual ~ObS3Client();
+  int init(const ObS3Account &account);
+  void destroy();
+  bool is_stopped() const;
+  bool try_stop(const int64_t timeout = STOP_S3_TIMEOUT_US);
+  void stop();
+  void increase();
+  void release();
+  TO_STRING_KV(KP(&lock_), K_(is_inited), K_(ref_cnt), K_(last_modified_ts), KP(client_));
+
+  int head_object(const Aws::S3::Model::HeadObjectRequest &request,
+      Aws::S3::Model::HeadObjectOutcome &outcome);
+  int put_object(const Aws::S3::Model::PutObjectRequest &request,
+      Aws::S3::Model::PutObjectOutcome &outcome);
+  int get_object(const Aws::S3::Model::GetObjectRequest &request,
+      Aws::S3::Model::GetObjectOutcome &outcome);
+  int delete_object(const Aws::S3::Model::DeleteObjectRequest &request,
+      Aws::S3::Model::DeleteObjectOutcome &outcome);
+  int put_object_tagging(const Aws::S3::Model::PutObjectTaggingRequest &request,
+      Aws::S3::Model::PutObjectTaggingOutcome &outcome);
+  int list_objects_v2(const Aws::S3::Model::ListObjectsV2Request &request,
+      Aws::S3::Model::ListObjectsV2Outcome &outcome);
+  int list_objects(const Aws::S3::Model::ListObjectsRequest &request,
+      Aws::S3::Model::ListObjectsOutcome &outcome);
+  int get_object_tagging(const Aws::S3::Model::GetObjectTaggingRequest &request,
+      Aws::S3::Model::GetObjectTaggingOutcome &outcome);
+  int create_multipart_upload(const Aws::S3::Model::CreateMultipartUploadRequest &request,
+      Aws::S3::Model::CreateMultipartUploadOutcome &outcome);
+  int list_parts(const Aws::S3::Model::ListPartsRequest &request,
+      Aws::S3::Model::ListPartsOutcome &outcome);
+  int complete_multipart_upload(const Aws::S3::Model::CompleteMultipartUploadRequest &request,
+      Aws::S3::Model::CompleteMultipartUploadOutcome &outcome);
+  int abort_multipart_upload(const Aws::S3::Model::AbortMultipartUploadRequest &request,
+      Aws::S3::Model::AbortMultipartUploadOutcome &outcome);
+  int upload_part(const Aws::S3::Model::UploadPartRequest &request,
+      Aws::S3::Model::UploadPartOutcome &outcome);
+  int list_multipart_uploads(const Aws::S3::Model::ListMultipartUploadsRequest &request,
+      Aws::S3::Model::ListMultipartUploadsOutcome &outcome);
+
+private:
+  int init_s3_client_configuration_(const ObS3Account &account,
+                                    Aws::S3::S3ClientConfiguration &config);
+
+  template<typename RequestType, typename OutcomeType>
+  using S3OperationFunc = OutcomeType (Aws::S3::S3Client::*)(const RequestType &) const;
+
+  template<typename RequestType, typename OutcomeType>
+  int do_s3_operation_(S3OperationFunc<RequestType, OutcomeType> s3_op_func,
+                       const RequestType &request, OutcomeType &outcome);
+
+private:
+  SpinRWLock lock_;
+  bool is_inited_;
+  bool stopped_;
+  int64_t ref_cnt_;
+  int64_t last_modified_ts_;
+  Aws::S3::S3Client *client_;
+};
+
+class ObS3Env
+{
+public:
+  static ObS3Env &get_instance();
+
+  // global init s3 env resource, must and only can be called once
+  int init();
+  // global clean s3 resource when don't use s3 any more
+  void destroy();
+
+  int get_or_create_s3_client(const ObS3Account &account, ObS3Client *&client);
+  void stop();
+
+private:
+  ObS3Env();
+  int clean_s3_client_map_();
+
+private:
+  SpinRWLock lock_;
+  bool is_inited_;
+  ObS3MemoryManager s3_mem_manger_;
+  Aws::SDKOptions aws_options_;
+  hash::ObHashMap<int64_t, ObS3Client *> s3_client_map_;
+};
+
+struct S3ObjectMeta : public ObStorageObjectMetaBase
+{
+};
+
+class SafeExecutor
+{
+public:
+  template<typename Function, typename Obj, typename ... Args>
+  int do_safely(Function f, Obj obj, Args && ... args)
+  {
+    int ret = OB_SUCCESS;
+    try {
+      ret = std::mem_fn(f)(obj, std::forward<Args>(args)...);
+    } catch (const std::exception &e) {
+      ret = OB_S3_ERROR;
+      OB_LOG(WARN, "caught exception when doing s3 operation", K(ret), K(e.what()), KP(this));
+    } catch (...) {
+      ret = OB_S3_ERROR;
+      OB_LOG(WARN, "caught unknown exception when doing s3 operation", K(ret), KP(this));
+    }
+    return ret;
+  }
+};
+
+class ObStorageS3Util;
+
+class ObStorageS3Base : public SafeExecutor
+{
+public:
+  ObStorageS3Base();
+  virtual ~ObStorageS3Base();
+  virtual void reset();
+
+  virtual int open(const ObString &uri, ObObjectStorageInfo *storage_info);
+  virtual bool is_inited() const { return is_inited_; }
+  int build_bucket_and_object_name(const ObString &uri);
+
+  int get_s3_file_meta(S3ObjectMeta &meta)
+  {
+    return do_safely(&ObStorageS3Base::get_s3_file_meta_, this, meta);
+  }
+
+protected:
+  int get_s3_file_meta_(S3ObjectMeta &meta);
+  int do_list_(const int64_t max_list_num, const char *delimiter,
+      const Aws::String &next_marker, Aws::S3::Model::ListObjectsOutcome &outcome);
+
+protected:
+  ObArenaAllocator allocator_;
+  ObS3Client *s3_client_;
+  ObString bucket_;
+  ObString object_;
+
+private:
+  bool is_inited_;
+  ObS3Account s3_account_;
+
+  friend class ObStorageS3Util;
+  DISALLOW_COPY_AND_ASSIGN(ObStorageS3Base);
+};
+
+class ObStorageS3Writer : public ObStorageS3Base, public ObIStorageWriter
+{
+public:
+  ObStorageS3Writer();
+  virtual ~ObStorageS3Writer();
+
+  virtual int open(const ObString &uri, ObObjectStorageInfo *storage_info) override
+  {
+    return do_safely(&ObStorageS3Writer::open_, this, uri, storage_info);
+  }
+  virtual int write(const char *buf, const int64_t size) override
+  {
+    return do_safely(&ObStorageS3Writer::write_, this, buf, size);
+  }
+  virtual int pwrite(const char *buf, const int64_t size, const int64_t offset) override
+  {
+    return do_safely(&ObStorageS3Writer::pwrite_, this, buf, size, offset);
+  }
+  virtual int close() override
+  {
+    return do_safely(&ObStorageS3Writer::close_, this);
+  }
+  virtual int64_t get_length() const override { return file_length_; }
+  virtual bool is_opened() const override { return is_opened_; }
+
+protected:
+  int open_(const ObString &uri, ObObjectStorageInfo *storage_info);
+  int write_(const char *buf, const int64_t size);
+  int write_obj_(const char *obj_name, const char *buf, const int64_t size);
+  int pwrite_(const char *buf, const int64_t size, const int64_t offset);
+  int close_();
+
+protected:
+  bool is_opened_;
+  int64_t file_length_;
+
+private:
+  DISALLOW_COPY_AND_ASSIGN(ObStorageS3Writer);
+};
+
+class ObStorageS3Reader : public ObStorageS3Base, public ObIStorageReader
+{
+public:
+  ObStorageS3Reader();
+  virtual ~ObStorageS3Reader();
+  virtual void reset() override;
+
+  virtual int open(const ObString &uri,
+                   ObObjectStorageInfo *storage_info, const bool head_meta = true) override
+  {
+    return do_safely(&ObStorageS3Reader::open_, this, uri, storage_info, head_meta);
+  }
+  virtual int pread(char *buf, const int64_t buf_size, const int64_t offset, int64_t &read_size) override
+  {
+    return do_safely(&ObStorageS3Reader::pread_, this, buf, buf_size, offset, read_size);
+  }
+  virtual int close() override
+  {
+    return do_safely(&ObStorageS3Reader::close_, this);
+  }
+  virtual int64_t get_length() const override { return file_length_; }
+  virtual bool is_opened() const override { return is_opened_; }
+
+protected:
+  int open_(const ObString &uri, ObObjectStorageInfo *storage_info, const bool head_meta = true);
+  int pread_(char *buf, const int64_t buf_size, const int64_t offset, int64_t &read_size);
+  int close_();
+
+protected:
+  bool is_opened_;
+  bool has_meta_;
+  int64_t file_length_;
+
+private:
+  DISALLOW_COPY_AND_ASSIGN(ObStorageS3Reader);
+};
+
+class ObStorageS3Util : public SafeExecutor, public ObIStorageUtil
+{
+public:
+  ObStorageS3Util();
+  virtual ~ObStorageS3Util();
+
+  virtual int open(ObObjectStorageInfo *storage_info) override;
+  virtual void close() override;
+  virtual int head_object_meta(const ObString &uri, ObStorageObjectMetaBase &obj_meta) override;
+
+  virtual int is_exist(const ObString &uri, bool &exist) override
+  {
+    return do_safely(&ObStorageS3Util::is_exist_, this, uri, exist);
+  }
+  virtual int get_file_length(const ObString &uri, int64_t &file_length) override
+  {
+    return do_safely(&ObStorageS3Util::get_file_length_, this, uri, file_length);
+  }
+  virtual int del_file(const ObString &uri) override
+  {
+    return do_safely(&ObStorageS3Util::del_file_, this, uri);
+  }
+  virtual int write_single_file(const ObString &uri, const char *buf, const int64_t size) override
+  {
+    return do_safely(&ObStorageS3Util::write_single_file_, this, uri, buf, size);
+  }
+  virtual int mkdir(const ObString &uri) override
+  {
+    return do_safely(&ObStorageS3Util::mkdir_, this, uri);
+  }
+  virtual int list_files(const ObString &uri, ObBaseDirEntryOperator &op) override
+  {
+    return do_safely(&ObStorageS3Util::list_files_, this, uri, op);
+  }
+  virtual int list_files(const ObString &uri, ObStorageListCtxBase &list_ctx) override
+  {
+    return do_safely(&ObStorageS3Util::list_files2_, this, uri, list_ctx);
+  }
+  virtual int del_dir(const ObString &uri) override
+  {
+    return do_safely(&ObStorageS3Util::del_dir_, this, uri);
+  }
+  virtual int list_directories(const ObString &uri, ObBaseDirEntryOperator &op) override
+  {
+    return do_safely(&ObStorageS3Util::list_directories_, this, uri, op);
+  }
+  virtual int is_tagging(const ObString &uri, bool &is_tagging) override
+  {
+    return do_safely(&ObStorageS3Util::is_tagging_, this, uri, is_tagging);
+  }
+  virtual int del_unmerged_parts(const ObString &uri) override
+  {
+    return do_safely(&ObStorageS3Util::del_unmerged_parts_, this, uri);
+  }
+
+private:
+  int is_exist_(const ObString &uri, bool &exist);
+  int get_file_length_(const ObString &uri, int64_t &file_length);
+  int del_file_(const ObString &uri);
+  int write_single_file_(const ObString &uri, const char *buf, const int64_t size);
+  int mkdir_(const ObString &uri);
+  int list_files_(const ObString &uri, ObBaseDirEntryOperator &op);
+  int list_files2_(const ObString &uri, ObStorageListCtxBase &list_ctx);
+  int del_dir_(const ObString &uri);
+  int list_directories_(const ObString &uri, ObBaseDirEntryOperator &op);
+  int is_tagging_(const ObString &uri, bool &is_tagging);
+  int del_unmerged_parts_(const ObString &uri);
+
+  int delete_object_(ObStorageS3Base &s3_base);
+  int tagging_object_(ObStorageS3Base &s3_base);
+
+private:
+  bool is_opened_;
+  ObObjectStorageInfo *storage_info_;
+};
+
+class ObStorageS3AppendWriter : public ObStorageS3Writer
+{
+public:
+  ObStorageS3AppendWriter();
+  virtual ~ObStorageS3AppendWriter();
+
+  virtual int open(const ObString &uri, ObObjectStorageInfo *storage_info) override
+  {
+    return do_safely(&ObStorageS3AppendWriter::open_, this, uri, storage_info);
+  }
+  virtual int write(const char *buf, const int64_t size) override
+  {
+    return do_safely(&ObStorageS3AppendWriter::write_, this, buf, size);
+  }
+  virtual int pwrite(const char *buf, const int64_t size, const int64_t offset) override
+  {
+    return do_safely(&ObStorageS3AppendWriter::pwrite_, this, buf, size, offset);
+  }
+  virtual int close() override
+  {
+    return do_safely(&ObStorageS3AppendWriter::close_, this);
+  }
+  virtual int64_t get_length() const override;
+  virtual bool is_opened() const override { return is_opened_; }
+
+protected:
+  int open_(const ObString &uri, ObObjectStorageInfo *storage_info);
+  int write_(const char *buf, const int64_t size);
+  int pwrite_(const char *buf, const int64_t size, const int64_t offset);
+  int close_();
+
+private:
+  ObObjectStorageInfo *storage_info_;
+
+  DISALLOW_COPY_AND_ASSIGN(ObStorageS3AppendWriter);
+};
+
+class ObStorageS3MultiPartWriter : public ObStorageS3Base, public ObIStorageWriter
+{
+public:
+  ObStorageS3MultiPartWriter();
+  virtual ~ObStorageS3MultiPartWriter();
+  virtual void reset() override;
+
+  virtual int open(const ObString &uri, ObObjectStorageInfo *storage_info) override
+  {
+    return do_safely(&ObStorageS3MultiPartWriter::open_, this, uri, storage_info);
+  }
+  virtual int write(const char *buf, const int64_t size) override
+  {
+    return do_safely(&ObStorageS3MultiPartWriter::write_, this, buf, size);
+  }
+  virtual int pwrite(const char *buf, const int64_t size, const int64_t offset) override
+  {
+    return do_safely(&ObStorageS3MultiPartWriter::pwrite_, this, buf, size, offset);
+  }
+  virtual int close() override
+  {
+    return do_safely(&ObStorageS3MultiPartWriter::close_, this);
+  }
+  virtual int64_t get_length() const override { return file_length_; }
+  virtual bool is_opened() const override { return is_opened_; }
+
+  int cleanup();
+
+private:
+  int open_(const ObString &uri, ObObjectStorageInfo *storage_info);
+  int write_(const char *buf, const int64_t size);
+  int pwrite_(const char *buf, const int64_t size, const int64_t offset);
+  int close_();
+  int write_single_part_();
+
+protected:
+  bool is_opened_;
+  char *base_buf_;
+  int64_t base_buf_pos_;
+  char *upload_id_;
+  int partnum_;
+  int64_t file_length_;
+  Aws::Utils::Crypto::CRC32 *sum_hash_; // for calc the complete crc based on each part's crc.
+
+private:
+  DISALLOW_COPY_AND_ASSIGN(ObStorageS3MultiPartWriter);
+};
+
+} // common
+} // oceanbase
+
+#endif

deps/oblib/unittest/lib/CMakeLists.txt

@@ -88,6 +88,9 @@ oblib_addtest(resource/test_resource_mgr.cpp)
 #oblib_addtest(restore/test_storage_file.cpp)
 #oblib_addtest(restore/test_storage_oss.cpp)
 oblib_addtest(restore/test_storage_cos.cpp)
+oblib_addtest(restore/test_storage_s3.cpp)
+oblib_addtest(restore/test_object_storage.cpp)
+oblib_addtest(restore/test_common_storage.cpp)
 oblib_addtest(restore/test_storage_info.cpp)
 #oblib_addtest(restore/test_storage.cpp)
 oblib_addtest(stat/test_di_cache.cpp)

deps/oblib/unittest/lib/alloc/test_malloc_allocator.cpp

@@ -12,6 +12,7 @@
 
 #include <gtest/gtest.h>
 #include "lib/alloc/ob_malloc_allocator.h"
+#include "lib/allocator/ob_malloc.h"
 
 using namespace oceanbase::lib;
 using namespace oceanbase::common;
@@ -60,6 +61,17 @@ TEST(TestMallocAllocator, idle)
                                                               OB_MALLOC_BIG_BLOCK_SIZE));
 }
 
+TEST(TestMallocAllocator, ob_malloc_align)
+{
+  void *ptr = ob_malloc_align(1, 4, "test");
+  ASSERT_TRUE(ptr != NULL);
+  ASSERT_EQ(0, (int64_t)ptr % 16);
+
+  ptr = ob_malloc_align(4096, 4, "test");
+  ASSERT_TRUE(ptr != NULL);
+  ASSERT_EQ(0, (int64_t)ptr % 4096);
+}
+
 int main(int argc, char *argv[])
 {
   signal(49, SIG_IGN);

deps/oblib/unittest/lib/restore/test_common_storage.h

@@ -0,0 +1,149 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+#ifndef TEST_STORAGE_COMMON_STORAGE_H_
+#define TEST_STORAGE_COMMON_STORAGE_H_
+
+#include "lib/utility/ob_print_utils.h"
+#include "lib/hash/ob_hashmap.h"
+#include "gtest/gtest.h"
+
+/**
+ * USER GUIDE
+ *
+ * This test case is for testing different storage media type, like oss, s3, fs, etc.
+ * It will check reader/writer/util/appender/multipartupload primary interfaces correctness.
+ * Besides, it will generate some abnormal situations to check the relative function correctness.
+ *
+ * If you want to run this test case, you just need to execute some simple steps.
+ *
+ * 1. For object storage, for example, s3, you just need to revise S3_BUCKET、S3_REGION、S3_ENDPOINT、S3_AK、
+ *    S3_SK as correct info, so as to oss、cos
+ *
+ * 2. For NFS, you just need to revise FS_PATH as empty string(means "") or some other value, but not INVALID_STR. It will
+ *    build a directory in the same directory as this test bin file.
+ *
+ * NOTICE: consider the running time, you'd better check these media one by one, or it may be timeout.
+ */
+
+namespace oceanbase{
+const char *INVALID_STR = "xxx";
+
+// S3 CONFIG
+const char *S3_BUCKET = INVALID_STR;
+const char *S3_REGION = INVALID_STR;
+const char *S3_ENDPOINT = INVALID_STR;
+const char *S3_AK = INVALID_STR;
+const char *S3_SK = INVALID_STR;
+
+// OSS CONFIG
+const char *OSS_BUCKET = INVALID_STR;
+const char *OSS_ENDPOINT = INVALID_STR;
+const char *OSS_AK = INVALID_STR;
+const char *OSS_SK = INVALID_STR;
+
+// COS CONFIG
+const char *COS_BUCKET = INVALID_STR;
+const char *COS_ENDPOINT = INVALID_STR;
+const char *COS_AK = INVALID_STR;
+const char *COS_SK = INVALID_STR;
+const char *COS_APPID = INVALID_STR;
+
+// NFS CONFIG
+const char *FS_PATH = INVALID_STR; // if FS_PATH value not equals to invalid string, we will use current path as FS_PATH
+
+enum class ObTestStorageType : uint8_t
+{
+  TEST_STORAGE_INVALID = 0,
+  TEST_STORAGE_OSS = 1,
+  TEST_STORAGE_S3 = 2,
+  TEST_STORAGE_COS = 3,
+  TEST_STORAGE_FS = 4,
+  TEST_STORAGE_MAX = 5
+};
+
+const char test_storage_type_str_arr[5][8] = {"INVALID", "OSS", "S3", "COS", "NFS"};
+
+struct ObTestStorageInfoConfig
+{
+public:
+  const static int64_t CFG_BUF_LEN = 1024;
+
+  char bucket_[CFG_BUF_LEN];
+  char region_[CFG_BUF_LEN];
+  char endpoint_[CFG_BUF_LEN];
+  char ak_[CFG_BUF_LEN];
+  char sk_[CFG_BUF_LEN];
+  union {
+    char appid_[CFG_BUF_LEN];
+    char fs_path_[CFG_BUF_LEN];
+  };
+
+  bool is_valid_;
+
+  ObTestStorageInfoConfig() : is_valid_(false) {}
+  ~ObTestStorageInfoConfig() {}
+
+  #define SET_FIELD(field_name) \
+    void set_##field_name(const char *value) \
+    {  \
+      if (nullptr != value) { \
+        const int64_t val_len = strlen(value); \
+        MEMCPY(field_name##_, value, val_len); \
+        field_name##_[val_len] = '\0'; \
+      } \
+    } \
+
+  SET_FIELD(bucket);
+  SET_FIELD(region);
+  SET_FIELD(endpoint);
+  SET_FIELD(ak);
+  SET_FIELD(sk);
+  SET_FIELD(appid);
+  SET_FIELD(fs_path);
+
+  bool is_valid() const { return is_valid_; }
+
+  TO_STRING_KV(K_(bucket), K_(region), K_(endpoint), K_(ak), K_(sk), K_(appid), K_(fs_path));
+};
+
+struct ObTestStorageMeta
+{
+public:
+  ObTestStorageType type_;
+  ObTestStorageInfoConfig config_;
+
+  ObTestStorageMeta() {}
+
+  ~ObTestStorageMeta() {}
+
+  int build_config(const ObTestStorageType type);
+  bool is_valid() const;
+
+  bool is_file_type() const { return type_ == ObTestStorageType::TEST_STORAGE_FS; }
+  bool is_obj_type() const { return type_ == ObTestStorageType::TEST_STORAGE_S3 ||
+                                    type_ == ObTestStorageType::TEST_STORAGE_COS ||
+                                    type_ == ObTestStorageType::TEST_STORAGE_OSS;}
+
+  TO_STRING_KV(K_(type), K_(config));
+
+private:
+  bool is_valid_type(const ObTestStorageType type) const;
+  void build_s3_cfg();
+  void build_oss_cfg();
+  void build_cos_cfg();
+  void build_fs_cfg();
+
+};
+
+} // end of oceanbase
+
+#endif

deps/oblib/unittest/lib/restore/test_common_storage_util.h

@@ -0,0 +1,145 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef TEST_STORAGE_COMMON_STORAGE_UTIL_H_
+#define TEST_STORAGE_COMMON_STORAGE_UTIL_H_
+
+#include "lib/restore/ob_storage.h"
+
+namespace oceanbase {
+
+using namespace oceanbase::common;
+
+class TestCommonStorageUtil
+{
+public:
+  static int build_object_storage_info(const char *bucket, const char *endpoint,
+    const char *ak, const char *sk, const char *region, const char *appid,
+    ObObjectStorageInfo &storage_info);
+
+  static int build_fs_storage_info(ObObjectStorageInfo &storage_info);
+
+
+  // The format of obj_uri will have two types:
+  //   1. media_type://bucket/raw_dir_path/obj_name
+  //   2. media_type://bucket/raw_dir_path/
+  static int gen_object_uri(char *obj_uri, const int64_t uri_buf_len, const char *bucket, const char *raw_dir_path, const char *obj_name);
+
+  // The format of fs_uri will have two types:
+  //   1. media_type://pwd_path/raw_dir_path/file_name
+  //   2. media_type://pwd_path/raw_dir_path/
+  static int gen_fs_uri(char *fs_uri, const int64_t uri_buf_len, const char *pwd_path, const char *raw_dir_path, const char *file_name);
+};
+
+int TestCommonStorageUtil::build_object_storage_info(
+    const char *bucket,
+    const char *endpoint,
+    const char *ak,
+    const char *sk,
+    const char *region,
+    const char *appid,
+    ObObjectStorageInfo &storage_info)
+{
+  int ret = OB_SUCCESS;
+  if (OB_ISNULL(bucket) || OB_ISNULL(endpoint) ||
+      OB_ISNULL(ak) || OB_ISNULL(sk)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid argument", K(ret), KP(bucket), KP(region), KP(endpoint), KP(ak), KP(sk));
+  } else {
+    char account[OB_MAX_URI_LENGTH] = { 0 };
+    ObStorageType storage_type = ObStorageType::OB_STORAGE_MAX_TYPE;
+    if (OB_FAIL(get_storage_type_from_path(bucket, storage_type))) {
+      OB_LOG(WARN, "fail to get storage type from path", K(ret), K(bucket));
+    } else if (ObStorageType::OB_STORAGE_FILE == storage_type) {
+      if (OB_FAIL(storage_info.set(storage_type, account))) {
+        OB_LOG(WARN, "fail to set storage info", K(ret));
+      }
+    } else {
+      int64_t pos = 0;
+      if (OB_FAIL(databuff_printf(account, sizeof(account), pos,
+          "host=%s&access_id=%s&access_key=%s", endpoint, ak, sk))) {
+        OB_LOG(WARN, "fail to databuff printf", K(ret), K(endpoint), K(ak), K(sk));
+      } else if (ObStorageType::OB_STORAGE_COS == storage_type &&
+                 databuff_printf(account, sizeof(account), pos, "&appid=%s", appid)) {
+        OB_LOG(WARN, "fail to databuff printf", K(ret), K(appid));
+      } else if (ObStorageType::OB_STORAGE_S3 == storage_type &&
+                 databuff_printf(account, sizeof(account), pos, "&s3_region=%s", region)) {
+        OB_LOG(WARN, "fail to databuff printf", K(ret), K(region));
+      }
+
+      if (FAILEDx(storage_info.set(storage_type, account))) {
+        OB_LOG(WARN, "fail to set storage info", K(ret), K(storage_type), K(account));
+      }
+    }
+  }
+  return ret;
+}
+
+int TestCommonStorageUtil::build_fs_storage_info(
+    ObObjectStorageInfo &storage_info)
+{
+  int ret = OB_SUCCESS;
+  char account[OB_MAX_URI_LENGTH] = { 0 };
+  if (OB_FAIL(storage_info.set(ObStorageType::OB_STORAGE_FILE, account))) {
+    OB_LOG(WARN, "fail to set storage info", K(ret));
+  }
+  return ret;
+}
+
+int TestCommonStorageUtil::gen_object_uri(
+    char *obj_uri,
+    const int64_t uri_buf_len,
+    const char *bucket,
+    const char *raw_dir_path,
+    const char *obj_name)
+{
+  int ret = OB_SUCCESS;
+  int64_t pos = 0;
+  const bool exist_obj = (obj_name != nullptr);
+  if (OB_ISNULL(obj_uri) || OB_ISNULL(bucket) || OB_ISNULL(raw_dir_path)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid argument", K(ret), KP(obj_uri), KP(bucket), KP(raw_dir_path));
+  } else if (exist_obj && OB_FAIL(databuff_printf(obj_uri, uri_buf_len, pos, "%s/%s/%s",
+             bucket, raw_dir_path, obj_name))) {
+    OB_LOG(WARN, "fail to databuff printf", K(ret), K(bucket), K(raw_dir_path), K(obj_name), K(uri_buf_len));
+  } else if (!exist_obj && OB_FAIL(databuff_printf(obj_uri, uri_buf_len, pos, "%s/%s/", bucket, raw_dir_path))) {
+    OB_LOG(WARN, "fail to databuff printf", K(ret), K(bucket), K(raw_dir_path), K(uri_buf_len));
+  }
+  return ret;
+}
+
+int TestCommonStorageUtil::gen_fs_uri(
+    char *fs_uri,
+    const int64_t uri_buf_len,
+    const char *pwd_path,
+    const char *raw_dir_path,
+    const char *file_name)
+{
+  int ret = OB_SUCCESS;
+  int64_t pos = 0;
+  const bool exist_file = (file_name != nullptr);
+  if (OB_ISNULL(fs_uri) || OB_ISNULL(pwd_path) || OB_ISNULL(raw_dir_path)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid argument", K(ret), KP(fs_uri), KP(pwd_path), KP(raw_dir_path));
+  } else if (exist_file && OB_FAIL(databuff_printf(fs_uri, uri_buf_len, pos, "%s%s/%s/%s", "file://",
+             pwd_path, raw_dir_path, file_name))) {
+    OB_LOG(WARN, "fail to databuff printf", K(ret), K(pwd_path), K(raw_dir_path), K(file_name), K(uri_buf_len));
+  } else if (!exist_file && OB_FAIL(databuff_printf(fs_uri, uri_buf_len, pos, "%s%s/%s/", "file://",
+             pwd_path, raw_dir_path))) {
+    OB_LOG(WARN, "fail to databuff printf", K(ret), K(pwd_path), K(raw_dir_path), K(uri_buf_len));
+  }
+  return ret;
+}
+
+}
+
+#endif

deps/oblib/unittest/lib/restore/test_object_storage.h

@@ -0,0 +1,24 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef TEST_OBJECT_STORAGE_H_
+#define TEST_OBJECT_STORAGE_H_
+
+const bool enable_test = false;
+const char *bucket = "xxx";
+const char *region = "xxx";
+const char *endpoint = "xxx";
+const char *secretid = "xxx";
+const char *secretkey = "xxx";
+const char *appid = "xxx";
+
+#endif

deps/oblib/unittest/lib/restore/test_storage_cos.cpp

@@ -671,6 +671,44 @@ TEST_F(TestStorageCos, test_util_is_tagging)
   }
 }
 
+TEST_F(TestStorageCos, test_multipartupload)
+{
+  int ret = OB_SUCCESS;
+  if (enable_test_) {
+    ObStorageCosMultiPartWriter multi_upload;
+    ObStorageUtil util;
+    ASSERT_EQ(OB_SUCCESS, util.open(&cos_base));
+
+    const char *tmp_multi_dir = "test_multipartupload";
+    const int64_t content_size = 20 * 1024 * 1024L;//20MB
+    char *content = new char[content_size];
+    memset(content, 'a', content_size);
+
+    // operate before open
+    ASSERT_EQ(OB_COS_ERROR, multi_upload.write(content, content_size));
+
+    const int64_t ts = ObTimeUtility::current_time();
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld",
+        bucket, dir_name, tmp_multi_dir, ts));
+
+    // operate correctly
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/multipartupload", dir_uri));
+    ASSERT_EQ(OB_SUCCESS, multi_upload.open(uri, &cos_base));
+    ASSERT_EQ(true, multi_upload.is_opened());
+    ASSERT_EQ(OB_SUCCESS, multi_upload.write(content, content_size));
+    ASSERT_EQ(OB_SUCCESS, multi_upload.close());
+
+    // check multipartupload valid
+    ASSERT_EQ(content_size, multi_upload.get_length());
+    bool is_exist = false;
+    ASSERT_EQ(OB_SUCCESS, util.is_exist(uri, is_exist));
+    ASSERT_TRUE(is_exist);
+    ASSERT_EQ(OB_SUCCESS, util.del_file(uri));
+
+    delete[] content;
+  }
+}
+
 int main(int argc, char **argv)
 {
   system("rm -f test_storage_cos.log*");

deps/oblib/unittest/lib/restore/test_storage_info.cpp

@@ -104,6 +104,31 @@ TEST(ObObjectStorageInfo, cos)
   ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info));
 }
 
+TEST(ObObjectStorageInfo, s3)
+{
+  const char *uri = "s3://backup_dir?host=xxx.com&access_id=111&access_key=222&s3_region=333";
+  ObObjectStorageInfo info1;
+
+  const char *storage_info = "";
+  ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info));
+  storage_info = "host=xxx.com&access_id=111&access_key=222";
+  ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info));
+
+  storage_info = "host=xxx.com&access_id=111&access_key=222&s3_region=333";
+  ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info));
+  ASSERT_EQ(0, ::strcmp("s3_region=333", info1.extension_));
+
+  char buf[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 };
+  ASSERT_EQ(OB_SUCCESS, info1.get_storage_info_str(buf, sizeof(buf)));
+  ASSERT_STREQ(storage_info, buf);
+
+  storage_info = "host=xxx.com&access_id=111&access_key=222&s3_region=333&delete_mode=delete";
+  info1.reset();
+  ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info));
+  ASSERT_EQ(OB_SUCCESS, info1.get_storage_info_str(buf, sizeof(buf)));
+  ASSERT_STREQ(storage_info, buf);
+}
+
 int main(int argc, char **argv)
 {
   system("rm -f test_storage_info.log*");

deps/oblib/unittest/lib/restore/test_storage_s3.h

@@ -0,0 +1,24 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef TEST_STORAGE_S3_H_
+#define TEST_STORAGE_S3_H_
+
+// TODO @fangdan: configure the parameters uniformly
+const bool enable_test = false;
+const char *bucket = "s3://xxx";
+const char *region = "xxx";
+const char *endpoint = "xxx";
+const char *secretid = "xxx";
+const char *secretkey = "xxx";
+
+#endif

src/logservice/archiveservice/ob_archive_file_utils.cpp

@@ -135,7 +135,7 @@ int ObArchiveFileUtils::get_file_range(const ObString &prefix,
   ObBackupIoAdapter util;
   ObFileRangeOp file_range_op;
 
-  if (OB_FAIL(util.list_files(prefix, storage_info, file_range_op))) {
+  if (OB_FAIL(util.adaptively_list_files(prefix, storage_info, file_range_op))) {
     ARCHIVE_LOG(WARN, "list_files fail", K(ret), K(prefix));
   } else if (0 == file_range_op.get_file_num()) {
     ret = OB_ENTRY_NOT_EXIST;
@@ -154,7 +154,7 @@ int ObArchiveFileUtils::list_files(const ObString &prefix,
   int ret = OB_SUCCESS;
   ObBackupIoAdapter util;
   ObFileListOp file_list_op;
-  if (OB_FAIL(util.list_files(prefix, storage_info, file_list_op))) {
+  if (OB_FAIL(util.adaptively_list_files(prefix, storage_info, file_list_op))) {
     ARCHIVE_LOG(WARN, "list_files fail", K(ret), K(prefix));
   } else if (OB_FAIL(file_list_op.get_file_list(array))) {
     ARCHIVE_LOG(WARN, "get_file_list fail", K(ret), K(prefix));
@@ -171,7 +171,7 @@ int ObArchiveFileUtils::read_file(const ObString &uri,
   int ret = OB_SUCCESS;
   ObBackupIoAdapter util;
 
-  if (OB_FAIL(util.read_single_file(uri, storage_info, buf, file_length, read_size))) {
+  if (OB_FAIL(util.adaptively_read_single_file(uri, storage_info, buf, file_length, read_size))) {
     if (OB_BACKUP_FILE_NOT_EXIST != ret) {
       ARCHIVE_LOG(WARN, "read_single_file fail", K(ret), K(uri));
     } else {
@@ -196,7 +196,7 @@ int ObArchiveFileUtils::range_read(const ObString &uri,
   if (OB_UNLIKELY(NULL == buf || buf_size  < 0 || offset < 0)) {
     ret = OB_INVALID_ARGUMENT;
     ARCHIVE_LOG(WARN, "invalid argument", K(ret), K(buf_size), K(offset), K(uri));
-  } else if (OB_FAIL(util.read_part_file(uri, storage_info, buf, buf_size, offset, read_size))) {
+  } else if (OB_FAIL(util.adaptively_read_part_file(uri, storage_info, buf, buf_size, offset, read_size))) {
     ARCHIVE_LOG(WARN, "read part file failed", K(ret), K(uri), K(buf_size), K(offset));
   }
   return ret;
@@ -240,7 +240,7 @@ int ObArchiveFileUtils::get_file_length(const ObString &uri,
   int ret = OB_SUCCESS;
   ObBackupIoAdapter util;
 
-  if (OB_FAIL(util.get_file_length(uri, storage_info, file_len))) {
+  if (OB_FAIL(util.adaptively_get_file_length(uri, storage_info, file_len))) {
     if (OB_BACKUP_FILE_NOT_EXIST != ret) {
       ARCHIVE_LOG(WARN, "get_file_length fail", K(ret), K(uri), KP(storage_info));
     } else {

src/logservice/archiveservice/ob_archive_io.cpp

@@ -32,7 +32,8 @@ int ObArchiveIO::push_log(const ObString &uri,
     char *data,
     const int64_t data_len,
     const int64_t offset,
-    const bool is_full_file)
+    const bool is_full_file,
+    const bool is_can_seal)
 {
   int ret = OB_SUCCESS;
   ObBackupIoAdapter util;
@@ -71,10 +72,13 @@ int ObArchiveIO::push_log(const ObString &uri,
         ARCHIVE_LOG(ERROR, "device_handle is NULL", K(ret), K(device_handle), K(uri));
       } else if (OB_FAIL(device_handle->pwrite(fd, offset, data_len, data, write_size))) {
         ARCHIVE_LOG(WARN, "fail to write file", K(ret), K(uri), KP(storage_info), K(data), K(data_len));
+      }  else if (is_can_seal && OB_FAIL(device_handle->seal_file(fd))) {
+        ARCHIVE_LOG(WARN, "fail to seal file", K(ret), K(uri), KP(storage_info));
       }
     }
   }
 
+
   int tmp_ret = OB_SUCCESS;
   if (OB_SUCCESS != (tmp_ret = util.close_device_and_fd(device_handle, fd))) {
     ARCHIVE_LOG(WARN, "fail to close file and release device!", K(tmp_ret), K(uri), KP(storage_info));
@@ -108,7 +112,7 @@ int ObArchiveIO::check_context_match_in_normal_file_(const ObString &uri,
   ObBackupIoAdapter reader;
   ObArenaAllocator allocator;
 
-  if (OB_FAIL(reader.get_file_length(uri, storage_info, length))) {
+  if (OB_FAIL(reader.adaptively_get_file_length(uri, storage_info, length))) {
     ARCHIVE_LOG(WARN, "get file_length failed", K(uri));
   } else if (OB_UNLIKELY(length <= offset)) {
     ret = OB_BACKUP_PWRITE_CONTENT_NOT_MATCH;
@@ -119,7 +123,7 @@ int ObArchiveIO::check_context_match_in_normal_file_(const ObString &uri,
   } else if (OB_ISNULL(read_buffer = static_cast<char*>(allocator.alloc(data_len)))) {
     ret = OB_ALLOCATE_MEMORY_FAILED;
     ARCHIVE_LOG(WARN, "allocate memory failed", K(uri), K(data_len));
-  } else if (OB_FAIL(reader.read_part_file(uri, storage_info, read_buffer, data_len, offset, read_size))) {
+  } else if (OB_FAIL(reader.adaptively_read_part_file(uri, storage_info, read_buffer, data_len, offset, read_size))) {
     ARCHIVE_LOG(WARN, "pread failed", K(uri), K(read_buffer), K(data_len), K(offset));
   } else if (read_size < data_len) {
     ret = OB_BACKUP_PWRITE_CONTENT_NOT_MATCH;

src/logservice/archiveservice/ob_archive_io.h

@@ -33,7 +33,8 @@ public:
       char *data,
       const int64_t data_len,
       const int64_t offset,
-      const bool is_full_file);
+      const bool is_full_file,
+      const bool is_can_seal);
 
   int mkdir(const ObString &uri,
       const share::ObBackupStorageInfo *storage_info);

src/logservice/archiveservice/ob_archive_sender.cpp

@@ -583,6 +583,7 @@ int ObArchiveSender::archive_log_(const ObBackupDest &backup_dest,
   char *filled_data = NULL;
   int64_t filled_data_len = 0;
   const bool is_full_file = (task.get_end_lsn() - task.get_start_lsn()) == MAX_ARCHIVE_FILE_SIZE;
+  const bool is_can_seal = 0 == task.get_end_lsn().val_ % MAX_ARCHIVE_FILE_SIZE;
   const int64_t start_ts = common::ObTimeUtility::current_time();
   // 1. decide archive file
   if (OB_FAIL(decide_archive_file_(task, arg.cur_file_id_, arg.cur_file_offset_,
@@ -612,8 +613,8 @@ int ObArchiveSender::archive_log_(const ObBackupDest &backup_dest,
     ARCHIVE_LOG(WARN, "fill file header if needed failed", K(ret));
   }
   // 6. push log
-  else if (OB_FAIL(push_log_(id, path.get_obstr(), backup_dest.get_storage_info(), is_full_file, new_file ?
-          file_offset : file_offset + ARCHIVE_FILE_HEADER_SIZE,
+  else if (OB_FAIL(push_log_(id, path.get_obstr(), backup_dest.get_storage_info(), is_full_file,
+          is_can_seal, new_file ? file_offset : file_offset + ARCHIVE_FILE_HEADER_SIZE,
           new_file ? filled_data : origin_data, new_file ? filled_data_len : origin_data_len))) {
     ARCHIVE_LOG(WARN, "push log failed", K(ret), K(task));
   // 7. 更新日志流归档任务archive file info
@@ -723,6 +724,7 @@ int ObArchiveSender::push_log_(const ObLSID &id,
     const ObString &uri,
     const share::ObBackupStorageInfo *storage_info,
     const bool is_full_file,
+    const bool is_can_seal,
     const int64_t offset,
     char *data,
     const int64_t data_len)
@@ -730,7 +732,7 @@ int ObArchiveSender::push_log_(const ObLSID &id,
   int ret = OB_SUCCESS;
   ObArchiveIO archive_io;
 
-  if (OB_FAIL(archive_io.push_log(uri, storage_info, data, data_len, offset, is_full_file))) {
+  if (OB_FAIL(archive_io.push_log(uri, storage_info, data, data_len, offset, is_full_file, is_can_seal))) {
     ARCHIVE_LOG(WARN, "push log failed", K(ret));
   } else {
     ARCHIVE_LOG(INFO, "push log succ", K(id));

src/logservice/archiveservice/ob_archive_sender.h

@@ -160,6 +160,7 @@ private:
       const ObString &uri,
       const share::ObBackupStorageInfo *storage_info,
       const bool is_full_file,
+      const bool is_can_seal,
       const int64_t offset,
       char *data,
       const int64_t data_len);

src/logservice/ob_log_external_storage_io_task.cpp

@@ -240,7 +240,7 @@ int convert_to_storage_access_type(const OPEN_FLAG &open_flag,
 {
   int ret = OB_SUCCESS;
   if (OPEN_FLAG::READ_FLAG == open_flag) {
-    storage_access_type = ObStorageAccessType::OB_STORAGE_ACCESS_READER;
+    storage_access_type = ObStorageAccessType::OB_STORAGE_ACCESS_ADAPTIVE_READER;
   } else {
     ret = OB_NOT_SUPPORTED;
     CLOG_LOG(WARN, "not supported flag", K(open_flag));
@@ -300,7 +300,7 @@ int ObLogExternalStorageIOTaskHandleAdapter::exist(const ObString &uri,
   ObIODevice *io_device = NULL;
   if (OB_FAIL(get_and_init_io_device(uri, storage_info, io_device))) {
     CLOG_LOG(WARN, "get_io_device failed", K(uri), KP(io_device));
-  } else if (OB_FAIL(io_device->exist(uri.ptr(), exist))) {
+  } else if (OB_FAIL(io_device->adaptive_exist(uri.ptr(), exist))) {
     CLOG_LOG(WARN, "exist failed", K(uri), KP(io_device), K(exist));
   } else {
     CLOG_LOG(TRACE, "exist success", K(uri), KP(io_device), K(exist));
@@ -319,7 +319,7 @@ int ObLogExternalStorageIOTaskHandleAdapter::get_file_size(const ObString &uri,
   ObIODevice *io_device = NULL;
   if (OB_FAIL(get_and_init_io_device(uri, storage_info, io_device))) {
     CLOG_LOG(WARN, "get_io_device failed", K(uri), KP(io_device));
-  } else if (OB_FAIL(io_device->stat(uri.ptr(), file_stat))) {
+  } else if (OB_FAIL(io_device->adaptive_stat(uri.ptr(), file_stat))) {
     CLOG_LOG(WARN, "stat io deveice failed", K(uri));
   } else {
     file_size = file_stat.size_;

src/share/backup/ob_archive_store.cpp

@@ -1128,7 +1128,7 @@ int ObArchiveStore::is_archive_log_file_exist(const int64_t dest_id, const int64
     LOG_WARN("ObArchiveStore not init", K(ret));
   } else if (OB_FAIL(ObArchivePathUtil::get_ls_archive_file_path(dest, dest_id, round_id, piece_id, ls_id, file_id, full_path))) {
     LOG_WARN("failed to get archive log file path", K(ret), K(dest), K(dest_id), K(round_id), K(piece_id), K(ls_id), K(file_id));
-  } else if (OB_FAIL(util.is_exist(full_path.get_ptr(), storage_info, is_exist))) {
+  } else if (OB_FAIL(util.adaptively_is_exist(full_path.get_ptr(), storage_info, is_exist))) {
     LOG_WARN("failed to check archive log file exist.", K(ret), K(full_path), K(storage_info));
   }
 
@@ -1439,7 +1439,7 @@ int ObArchiveStore::get_file_list_in_piece(const int64_t dest_id, const int64_t
     LOG_WARN("get piece ls dir path failed", K(ret), K(dest), K(dest_id), K(round_id), K(piece_id), K(ls_id));
   } else if (OB_FAIL(op.init(this, &filelist))) {
     LOG_WARN("ObLSFileListOp init failed", K(ret));
-  } else if (OB_FAIL(util.list_files(piece_path.get_ptr(), storage_info, op))) {
+  } else if (OB_FAIL(util.adaptively_list_files(piece_path.get_ptr(), storage_info, op))) {
     LOG_WARN("list files failed", K(ret), K(piece_path), K(dest));
   }
   return ret;

src/share/backup/ob_backup_connectivity.cpp

@@ -326,7 +326,7 @@ int ObBackupCheckFile::compare_check_file_name_(
     ObDirPrefixEntryNameFilter prefix_op(d_entrys);
     if (OB_FAIL(prefix_op.init(check_file_prefix, static_cast<int32_t>(strlen(check_file_prefix))))) {
       LOG_WARN("failed to init dir prefix", K(ret), K(check_file_prefix), K_(tenant_id));
-    } else if (OB_FAIL(util.list_files(path.get_obstr(), backup_dest.get_storage_info(), prefix_op))) {
+    } else if (OB_FAIL(util.adaptively_list_files(path.get_obstr(), backup_dest.get_storage_info(), prefix_op))) {
       LOG_WARN("failed to list files", K(ret), K_(tenant_id));
     } else if (OB_FAIL(ObBackupStorageInfoOperator::get_check_file_name(
         *sql_proxy_, tenant_id_, backup_dest, check_file_name))) {
@@ -346,7 +346,7 @@ int ObBackupCheckFile::compare_check_file_name_(
           LOG_WARN("failed to set check file path", K(ret), K(path), K_(tmp_entry.name));
         } else {
           common::ObString uri(del_file_path);
-          if(OB_FAIL(util.del_file(uri, backup_dest.get_storage_info()))) {
+          if(OB_FAIL(util.adaptively_del_file(uri, backup_dest.get_storage_info()))) {
             LOG_WARN("failed to delete check file", K(ret), K_(tenant_id));
           }
         }
@@ -451,7 +451,7 @@ int ObBackupCheckFile::delete_permission_check_file(const ObBackupDest &backup_d
     ObDirPrefixEntryNameFilter prefix_op(d_entrys);
     if (OB_FAIL(prefix_op.init(check_file_prefix, static_cast<int32_t>(strlen(check_file_prefix))))) {
       LOG_WARN("failed to init dir prefix", K(ret), K(check_file_prefix), K_(tenant_id));
-    } else if (OB_FAIL(util.list_files(path.get_obstr(), backup_dest.get_storage_info(), prefix_op))) {
+    } else if (OB_FAIL(util.adaptively_list_files(path.get_obstr(), backup_dest.get_storage_info(), prefix_op))) {
       LOG_WARN("failed to list files", K(ret), K_(tenant_id));
     } else {
       char del_file_path[OB_MAX_BACKUP_PATH_LENGTH];
@@ -466,7 +466,7 @@ int ObBackupCheckFile::delete_permission_check_file(const ObBackupDest &backup_d
           LOG_WARN("failed to set delete file path", K(ret), K(path), K_(tmp_entry.name));
         } else {
           common::ObString uri(del_file_path);
-          if(OB_FAIL(util.del_file(uri, backup_dest.get_storage_info()))) {
+          if(OB_FAIL(util.adaptively_del_file(uri, backup_dest.get_storage_info()))) {
             LOG_WARN("failed to delete permission check file", K(ret), K_(tenant_id));
           }
         }
@@ -538,7 +538,7 @@ int ObBackupCheckFile::check_appender_permission_(const ObBackupDest &backup_des
     LOG_WARN("fail to set data", K(ret), K(path.get_ptr()));
   } else if (OB_FAIL(device_handle->write(fd, data, strlen(data),  write_size))) {
     LOG_WARN("fail to write file", K(ret), K(path.get_ptr()), K(data));
-  } else if (OB_FAIL(util.del_file(path.get_obstr(), backup_dest.get_storage_info()))) {
+  } else if (OB_FAIL(util.adaptively_del_file(path.get_obstr(), backup_dest.get_storage_info()))) {
     LOG_WARN("failed to del file", K(ret));
   }
 
@@ -596,7 +596,7 @@ int ObBackupCheckFile::check_io_permission(const ObBackupDest &backup_dest)
     }
     LOG_WARN("failed to write single file", K(ret), K_(tenant_id), K(backup_dest));
   } else if (FALSE_IT(write_ok = true)
-      || OB_FAIL(util.get_file_length(path.get_obstr(), backup_dest.get_storage_info(), file_len))) {
+      || OB_FAIL(util.adaptively_get_file_length(path.get_obstr(), backup_dest.get_storage_info(), file_len))) {
     if (is_permission_error_(ret)) { 
       ROOTSERVICE_EVENT_ADD("connectivity_check", "permission check", 
           "tenant_id", tenant_id_, "error_code", ret, "comment", "get file length");
@@ -606,7 +606,7 @@ int ObBackupCheckFile::check_io_permission(const ObBackupDest &backup_dest)
   } else if (OB_ISNULL(buf = reinterpret_cast<char*>(allocator.alloc(file_len)))) {
     ret = OB_ALLOCATE_MEMORY_FAILED;
     LOG_WARN("failed to alloc buf", K(ret), K(file_len));
-  } else if (OB_FAIL(util.read_single_file(path.get_obstr(), backup_dest.get_storage_info(), buf, file_len, read_size))) {
+  } else if (OB_FAIL(util.adaptively_read_single_file(path.get_obstr(), backup_dest.get_storage_info(), buf, file_len, read_size))) {
     if (is_permission_error_(ret)) {
       ROOTSERVICE_EVENT_ADD("connectivity_check", "permission check", 
           "tenant_id", tenant_id_, "error_code", ret, "comment", "read single file");
@@ -614,7 +614,7 @@ int ObBackupCheckFile::check_io_permission(const ObBackupDest &backup_dest)
     }
     LOG_WARN("failed to read single file", K(ret));
   }
-  if (write_ok && (OB_SUCCESS != (tmp_ret = util.del_file(path.get_obstr(), backup_dest.get_storage_info())))) {
+  if (write_ok && (OB_SUCCESS != (tmp_ret = util.adaptively_del_file(path.get_obstr(), backup_dest.get_storage_info())))) {
     if (is_permission_error_(tmp_ret)) {
       ROOTSERVICE_EVENT_ADD("connectivity_check", "permission check", 
           "tenant_id", tenant_id_, "error_code", tmp_ret, "comment", "delete file");
@@ -672,7 +672,7 @@ int ObBackupDestCheck::check_check_file_exist_(
   bool need_retry = true;
   is_exist = false;
   while (retry_times--) {
-    if (OB_FAIL(util.is_exist(path.get_obstr(), backup_dest.get_storage_info(), is_exist))) {
+    if (OB_FAIL(util.adaptively_is_exist(path.get_obstr(), backup_dest.get_storage_info(), is_exist))) {
       LOG_WARN("failed to check is_exist", K(ret), K(path), K(backup_dest), K(retry_times));
       ob_usleep(1 * 1000 * 1000L); // 1s 
       continue;

src/share/backup/ob_backup_io_adapter.cpp

@@ -113,7 +113,7 @@ int ObBackupIoAdapter::get_and_init_device(ObIODevice*& dev_handle,
 int ObBackupIoAdapter::is_exist(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, bool &exist)
 {
   int ret = OB_SUCCESS;
-  ObIODevice* device_handle = NULL;
+  ObIODevice *device_handle = NULL;
   if (OB_FAIL(get_and_init_device(device_handle, storage_info, uri))) {
     OB_LOG(WARN, "fail to get device!", K(ret), K(uri), KP(storage_info));
   } else if (OB_FAIL(device_handle->exist(uri.ptr(), exist))) {
@@ -123,10 +123,23 @@ int ObBackupIoAdapter::is_exist(const common::ObString &uri, const share::ObBack
   return ret;
 }
 
+int ObBackupIoAdapter::adaptively_is_exist(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, bool &exist)
+{
+  int ret = OB_SUCCESS;
+  ObIODevice *device_handle = NULL;
+  if (OB_FAIL(get_and_init_device(device_handle, storage_info, uri))) {
+    OB_LOG(WARN, "fail to get device!", K(ret), K(uri), KP(storage_info));
+  } else if (OB_FAIL(device_handle->adaptive_exist(uri.ptr(), exist))) {
+    OB_LOG(WARN, "fail to check exist!", K(ret), K(uri), KP(storage_info));
+  }
+  release_device(device_handle);
+  return ret;
+}
+
 int ObBackupIoAdapter::get_file_length(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, int64_t &file_length)
 {
   int ret = OB_SUCCESS;
-  ObIODevice* device_handle = NULL;
+  ObIODevice *device_handle = NULL;
   ObIODFileStat statbuf;
   file_length = -1;
   if (OB_FAIL(get_and_init_device(device_handle, storage_info, uri))) {
@@ -140,10 +153,27 @@ int ObBackupIoAdapter::get_file_length(const common::ObString &uri, const share:
   return ret;
 }
 
+int ObBackupIoAdapter::adaptively_get_file_length(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, int64_t &file_length)
+{
+  int ret = OB_SUCCESS;
+  ObIODevice *device_handle = NULL;
+  ObIODFileStat statbuf;
+  file_length = -1;
+  if (OB_FAIL(get_and_init_device(device_handle, storage_info, uri))) {
+    OB_LOG(WARN, "fail to get device!", K(ret), K(uri), KP(storage_info));
+  } else if (OB_FAIL(device_handle->adaptive_stat(uri.ptr(), statbuf))) {
+    OB_LOG(WARN, "fail to get file length!", K(ret), K(uri), KP(storage_info));
+  } else {
+    file_length = statbuf.size_;
+  }
+  release_device(device_handle);
+  return ret;
+}
+
 int ObBackupIoAdapter::del_file(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info)
 {
   int ret = OB_SUCCESS;
-  ObIODevice* device_handle = NULL;
+  ObIODevice *device_handle = NULL;
   if (OB_FAIL(get_and_init_device(device_handle, storage_info, uri))) {
     OB_LOG(WARN, "fail to get device!", K(ret), K(uri), KP(storage_info));
   } else if (OB_FAIL(device_handle->unlink(uri.ptr()))) {
@@ -153,10 +183,36 @@ int ObBackupIoAdapter::del_file(const common::ObString &uri, const share::ObBack
   return ret;
 }
 
+int ObBackupIoAdapter::adaptively_del_file(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info)
+{
+  int ret = OB_SUCCESS;
+  ObIODevice *device_handle = NULL;
+  if (OB_FAIL(get_and_init_device(device_handle, storage_info, uri))) {
+    OB_LOG(WARN, "fail to get device!", K(ret), K(uri), KP(storage_info));
+  } else if (OB_FAIL(device_handle->adaptive_unlink(uri.ptr()))) {
+    OB_LOG(WARN, "fail to del file!", K(ret), K(uri), KP(storage_info));
+  }
+  release_device(device_handle);
+  return ret;
+}
+
+int ObBackupIoAdapter::del_unmerged_parts(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info)
+{
+  int ret = OB_SUCCESS;
+  ObIODevice *device_handle = NULL;
+  if (OB_FAIL(get_and_init_device(device_handle, storage_info, uri))) {
+    OB_LOG(WARN, "fail to get device!", K(ret), K(uri), KP(storage_info));
+  } else if (OB_FAIL(device_handle->del_unmerged_parts(uri.ptr()))) {
+    OB_LOG(WARN, "fail to del file!", K(ret), K(uri), KP(storage_info));
+  }
+  release_device(device_handle);
+  return ret;
+}
+
 int ObBackupIoAdapter::mkdir(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info)
 {
   int ret = OB_SUCCESS;
-  ObIODevice* device_handle = NULL;
+  ObIODevice *device_handle = NULL;
   if (OB_FAIL(get_and_init_device(device_handle, storage_info, uri))) {
     OB_LOG(WARN, "fail to get device!", K(ret), K(uri), KP(storage_info));
   } else if (OB_FAIL(device_handle->mkdir(uri.ptr(), 0))) {
@@ -171,7 +227,7 @@ int ObBackupIoAdapter::mk_parent_dir(const common::ObString &uri, const share::O
 {
   int ret = OB_SUCCESS;
   char path[OB_MAX_URI_LENGTH];
-  ObIODevice* device_handle = NULL;
+  ObIODevice *device_handle = NULL;
 
   if (uri.empty()) {
     ret = OB_INVALID_ARGUMENT;
@@ -211,7 +267,7 @@ int ObBackupIoAdapter::write_single_file(const common::ObString &uri, const shar
   int ret = OB_SUCCESS;
   int ret_tmp = OB_SUCCESS;
   ObIOFd fd; 
-  ObIODevice* device_handle = NULL;
+  ObIODevice *device_handle = NULL;
   const int64_t start_ts = ObTimeUtility::current_time();
   int64_t file_length = -1;
   int64_t write_size = -1;
@@ -254,7 +310,7 @@ int ObBackupIoAdapter::read_single_file(const common::ObString &uri, const share
   int ret = OB_SUCCESS;
   int ret_tmp = OB_SUCCESS;
   ObIOFd fd;
-  ObIODevice* device_handle = NULL;
+  ObIODevice *device_handle = NULL;
   const int64_t start_ts = ObTimeUtility::current_time();
   int64_t file_length = -1;
 
@@ -263,7 +319,38 @@ int ObBackupIoAdapter::read_single_file(const common::ObString &uri, const share
     OB_LOG(WARN, "fail to get device and open file !", K(uri), K(ret));
   } else if (OB_FAIL(device_handle->pread(fd, 0, buf_size, buf, read_size))) {
     OB_LOG(WARN, "failed to do read single file", K(ret), K(uri));
-  } else if (OB_FAIL(get_file_size(device_handle, fd, file_length))) {
+  } else if (OB_FAIL(get_file_length(uri, storage_info, file_length))) {
+    OB_LOG(WARN, "failed to get file size", K(ret), K(uri));
+  } else if (file_length != read_size) {
+    ret = OB_BUF_NOT_ENOUGH;
+    OB_LOG(WARN, "not whole file read, maybe buf not enough",
+          K(ret), K(read_size), K(file_length), K(uri));
+  }
+
+  if (OB_SUCCESS != (ret_tmp = close_device_and_fd(device_handle, fd))) {
+    ret = (OB_SUCCESS == ret) ? ret_tmp : ret;
+    STORAGE_LOG(WARN, "failed to close device and fd", K(ret), K(ret_tmp));
+  }
+
+  return ret;
+}
+
+int ObBackupIoAdapter::adaptively_read_single_file(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, char *buf,
+                                        const int64_t buf_size, int64_t &read_size)
+{
+  int ret = OB_SUCCESS;
+  int ret_tmp = OB_SUCCESS;
+  ObIOFd fd;
+  ObIODevice *device_handle = NULL;
+  const int64_t start_ts = ObTimeUtility::current_time();
+  int64_t file_length = -1;
+
+  if (OB_FAIL(open_with_access_type(device_handle, fd, storage_info,
+                      uri, OB_STORAGE_ACCESS_ADAPTIVE_READER))) {
+    OB_LOG(WARN, "fail to get device and open file !", K(uri), K(ret));
+  } else if (OB_FAIL(device_handle->pread(fd, 0, buf_size, buf, read_size))) {
+    OB_LOG(WARN, "failed to do read single file", K(ret), K(uri));
+  } else if (OB_FAIL(adaptively_get_file_length(uri, storage_info, file_length))) {
     OB_LOG(WARN, "failed to get file size", K(ret), K(uri));
   } else if (file_length != read_size) {
     ret = OB_BUF_NOT_ENOUGH;
@@ -295,11 +382,27 @@ int ObBackupIoAdapter::read_single_text_file(const common::ObString &uri, const
   return ret;
 }
 
+int ObBackupIoAdapter::adaptively_read_single_text_file(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info,
+                                             char *buf, const int64_t buf_size)
+{
+  int ret = OB_SUCCESS;
+  int64_t read_size = -1;
+  if (OB_FAIL(ObBackupIoAdapter::adaptively_read_single_file(uri, storage_info, buf, buf_size, read_size))) {
+    OB_LOG(WARN, "failed to read_single_object", K(ret), K(uri));
+  } else if (read_size < 0 || read_size >= buf_size) {
+    ret = OB_BUF_NOT_ENOUGH;
+    OB_LOG(WARN, "buf not enough", K(ret), K(read_size), K(buf_size));
+  } else {
+    buf[read_size] = '\0';
+  }
+  return ret;
+}
+
 int ObBackupIoAdapter::list_files(const common::ObString &dir_path, const share::ObBackupStorageInfo *storage_info,
         common::ObBaseDirEntryOperator &op)
 {
   int ret = OB_SUCCESS;
-  ObIODevice* device_handle = NULL;
+  ObIODevice *device_handle = NULL;
   if (OB_FAIL(get_and_init_device(device_handle, storage_info, dir_path))) {
     OB_LOG(WARN, "fail to get device!", K(ret));
   } else if (OB_FAIL(device_handle->scan_dir(dir_path.ptr(), op))) {
@@ -309,6 +412,20 @@ int ObBackupIoAdapter::list_files(const common::ObString &dir_path, const share:
   return ret;
 }
 
+int ObBackupIoAdapter::adaptively_list_files(const common::ObString &dir_path, const share::ObBackupStorageInfo *storage_info,
+        common::ObBaseDirEntryOperator &op)
+{
+  int ret = OB_SUCCESS;
+  ObIODevice *device_handle = NULL;
+  if (OB_FAIL(get_and_init_device(device_handle, storage_info, dir_path))) {
+    OB_LOG(WARN, "fail to get device!", K(ret));
+  } else if (OB_FAIL(device_handle->adaptive_scan_dir(dir_path.ptr(), op))) {
+    OB_LOG(WARN, "fail to scan dir!", K(ret), K(dir_path), KP(storage_info));
+  }
+  release_device(device_handle);
+  return ret;
+}
+
 int ObBackupIoAdapter::list_directories(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info,
                         common::ObBaseDirEntryOperator &op)
 {
@@ -323,7 +440,7 @@ int ObBackupIoAdapter::list_directories(const common::ObString &uri, const share
 int ObBackupIoAdapter::is_tagging(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, bool &is_tagging)
 {
   int ret = OB_SUCCESS;
-  ObIODevice* device_handle = NULL;
+  ObIODevice *device_handle = NULL;
   if (OB_FAIL(get_and_init_device(device_handle, storage_info, uri))) {
     OB_LOG(WARN, "fail to get device!", K(ret), K(uri), KP(storage_info));
   } else if (OB_FAIL(device_handle->is_tagging(uri.ptr(), is_tagging))) {
@@ -339,7 +456,7 @@ int ObBackupIoAdapter::read_part_file(const common::ObString &uri, const share::
   int ret = OB_SUCCESS;
   int ret_tmp = OB_SUCCESS;
   ObIOFd fd;
-  ObIODevice* device_handle = NULL;
+  ObIODevice *device_handle = NULL;
 
   if (OB_FAIL(open_with_access_type(device_handle, fd, storage_info, 
                       uri, OB_STORAGE_ACCESS_READER))) {
@@ -355,7 +472,29 @@ int ObBackupIoAdapter::read_part_file(const common::ObString &uri, const share::
   return ret;
 }
 
-int ObBackupIoAdapter::get_file_size(ObIODevice* device_handle, const ObIOFd &fd, int64_t &file_length)
+int ObBackupIoAdapter::adaptively_read_part_file(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info,
+      char *buf, const int64_t buf_size, const int64_t offset, int64_t &read_size)
+{
+  int ret = OB_SUCCESS;
+  int ret_tmp = OB_SUCCESS;
+  ObIOFd fd;
+  ObIODevice*device_handle = NULL;
+
+  if (OB_FAIL(open_with_access_type(device_handle, fd, storage_info,
+                      uri, OB_STORAGE_ACCESS_ADAPTIVE_READER))) {
+    OB_LOG(WARN, "fail to get device and open file !", K(uri), K(ret), KP(storage_info));
+  } else if (OB_FAIL(device_handle->pread(fd, offset, buf_size, buf, read_size))) {
+    OB_LOG(WARN, "fail to read file !", K(uri), K(ret), KP(storage_info));
+  }
+
+  if (OB_SUCCESS != (ret_tmp = close_device_and_fd(device_handle, fd))) {
+    ret = (OB_SUCCESS == ret) ? ret_tmp : ret;
+    STORAGE_LOG(WARN, "failed to close device and fd", K(ret), K(ret_tmp), KP(storage_info), K(uri));
+  }
+  return ret;
+}
+
+int ObBackupIoAdapter::get_file_size(ObIODevice *device_handle, const ObIOFd &fd, int64_t &file_length)
 {
   int ret = OB_SUCCESS;
   int flag = -1;
@@ -364,15 +503,20 @@ int ObBackupIoAdapter::get_file_size(ObIODevice* device_handle, const ObIOFd &fd
   file_length = -1;
   ObFdSimulator::get_fd_flag(fd, flag);
   ObObjectDevice* obj_device_handle = (ObObjectDevice*)device_handle;
-  if (flag != OB_STORAGE_ACCESS_READER && flag != OB_STORAGE_ACCESS_APPENDER) {
+  if (OB_STORAGE_ACCESS_READER != flag
+             && OB_STORAGE_ACCESS_ADAPTIVE_READER != flag
+             && OB_STORAGE_ACCESS_APPENDER != flag ) {
     ret = OB_INVALID_ARGUMENT;
     OB_LOG(WARN, "Invaild access type, object device only support reader and appender get file size!", K(flag));
   } else if (OB_FAIL(obj_device_handle->get_fd_mng().fd_to_ctx(fd, ctx))) {
     OB_LOG(WARN, "fail to get ctx accroding fd!", K(ret));
   } else {
-    if (flag == OB_STORAGE_ACCESS_READER) {
+    if (OB_STORAGE_ACCESS_READER == flag) {
       ObStorageReader *reader = static_cast<ObStorageReader*>(ctx);
       file_length = reader->get_length();
+    } else if (OB_STORAGE_ACCESS_ADAPTIVE_READER == flag) {
+      ObStorageAdaptiveReader *adaptive_reader = static_cast<ObStorageAdaptiveReader*>(ctx);
+      file_length = adaptive_reader->get_length();
     } else {
       ObStorageAppender *appender = static_cast<ObStorageAppender*>(ctx);
       file_length = appender->get_length();
@@ -384,7 +528,7 @@ int ObBackupIoAdapter::get_file_size(ObIODevice* device_handle, const ObIOFd &fd
 int ObBackupIoAdapter::del_dir(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info)
 {
   int ret = OB_SUCCESS;
-  ObIODevice* device_handle = NULL;
+  ObIODevice *device_handle = NULL;
   if (OB_FAIL(get_and_init_device(device_handle, storage_info, uri))) {
     OB_LOG(WARN, "fail to get device!", K(ret), K(uri), KP(storage_info));
   } else if (OB_FAIL(device_handle->rmdir(uri.ptr()))) {
@@ -397,7 +541,7 @@ int ObBackupIoAdapter::del_dir(const common::ObString &uri, const share::ObBacku
 class ObDelTmpFileOp : public ObBaseDirEntryOperator
 {
 public: 
-  ObDelTmpFileOp(int64_t now_ts, char* dir_path, ObIODevice* device_handle) :
+  ObDelTmpFileOp(int64_t now_ts, char* dir_path, ObIODevice *device_handle) :
       now_ts_(now_ts), dir_path_(dir_path), device_handle_(device_handle)
       {}
   ~ObDelTmpFileOp() {}
@@ -405,7 +549,7 @@ public:
 private:
   int64_t now_ts_;
   char* dir_path_;
-  ObIODevice* device_handle_;
+  ObIODevice *device_handle_;
 };
 
 int get_tmp_file_format_timestamp(const char *file_name, 
@@ -506,6 +650,8 @@ int get_real_file_path(const common::ObString &uri, char *buf, const int64_t buf
     prefix = OB_OSS_PREFIX;
   } else if (OB_STORAGE_COS == device_type) {
     prefix = OB_COS_PREFIX;
+  } else if (OB_STORAGE_S3 == device_type) {
+    prefix = OB_S3_PREFIX;
   } else if (OB_STORAGE_FILE == device_type) {
     prefix = OB_FILE_PREFIX;
   } else {
@@ -538,7 +684,7 @@ int get_real_file_path(const common::ObString &uri, char *buf, const int64_t buf
 int ObBackupIoAdapter::delete_tmp_files(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info)
 {
   int ret = OB_SUCCESS;
-  ObIODevice* device_handle = NULL;
+  ObIODevice *device_handle = NULL;
   if (OB_FAIL(get_and_init_device(device_handle, storage_info, uri))) {
     OB_LOG(WARN, "fail to get device!", K(ret), K(uri), KP(storage_info));
   } else if (OB_STORAGE_FILE == device_handle->device_type_) {
@@ -578,7 +724,7 @@ int ObBackupIoAdapter::is_empty_directory(const common::ObString &uri,
                                         bool &is_empty_directory)
 {
   int ret = OB_SUCCESS;
-  ObIODevice* device_handle = NULL;
+  ObIODevice *device_handle = NULL;
   ObCheckDirEmptOp ept_dir_op;
   is_empty_directory = true;
   if (OB_FAIL(get_and_init_device(device_handle, storage_info, uri))) {

src/share/backup/ob_backup_io_adapter.h

@@ -29,22 +29,38 @@ public:
   explicit ObBackupIoAdapter() {}
   virtual ~ObBackupIoAdapter() {}
   int is_exist(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, bool &exist);
+  //TODO (@shifangdan.sfd): refine repeated logics between normal interfaces and adaptive ones
+  int adaptively_is_exist(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, bool &exist);
   int get_file_length(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, int64_t &file_length);
+  int adaptively_get_file_length(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, int64_t &file_length);
   int del_file(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info);
+  int adaptively_del_file(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info);
+  int del_unmerged_parts(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info);
   int mkdir(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info);
   int mk_parent_dir(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info);
   int write_single_file(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, const char *buf, const int64_t size);
   int read_single_file(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, char *buf,const int64_t buf_size,
       int64_t &read_size);
+  int adaptively_read_single_file(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, char *buf,const int64_t buf_size,
+      int64_t &read_size);
   int read_single_text_file(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, char *buf, const int64_t buf_size);
+  int adaptively_read_single_text_file(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, char *buf, const int64_t buf_size);
   int list_files(
       const common::ObString &dir_path,
       const share::ObBackupStorageInfo *storage_info,
       common::ObBaseDirEntryOperator &op);
+  int adaptively_list_files(
+      const common::ObString &dir_path,
+      const share::ObBackupStorageInfo *storage_info,
+      common::ObBaseDirEntryOperator &op);
   int read_part_file(
       const common::ObString &uri, const share::ObBackupStorageInfo *storage_info,
       char *buf, const int64_t buf_size, const int64_t offset,
       int64_t &read_size);
+  int adaptively_read_part_file(
+      const common::ObString &uri, const share::ObBackupStorageInfo *storage_info,
+      char *buf, const int64_t buf_size, const int64_t offset,
+      int64_t &read_size);
   int del_dir(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info);
   /*backup logical related func*/
   int get_file_size(ObIODevice* device_handle, const ObIOFd &fd, int64_t &file_length);

src/share/backup/ob_backup_struct.cpp

@@ -1171,13 +1171,13 @@ int ObBackupStorageInfo::get_access_key_(char *key_buf, const int64_t key_buf_le
   return ret;
 }
 
-int ObBackupStorageInfo::parse_storage_info_(const char *storage_info, bool &has_appid)
+int ObBackupStorageInfo::parse_storage_info_(const char *storage_info, bool &has_needed_extension)
 {
   int ret = OB_SUCCESS;
   if (OB_ISNULL(storage_info) || strlen(storage_info) >= OB_MAX_BACKUP_STORAGE_INFO_LENGTH) {
     ret = OB_INVALID_BACKUP_DEST;
     LOG_WARN("storage info is invalid", K(ret), K(storage_info), K(strlen(storage_info)));
-  } else if (OB_FAIL(ObObjectStorageInfo::parse_storage_info_(storage_info, has_appid))) {
+  } else if (OB_FAIL(ObObjectStorageInfo::parse_storage_info_(storage_info, has_needed_extension))) {
     LOG_WARN("failed to parse storage info", K(ret), K(storage_info));
   } else {
     char tmp[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 };

src/share/backup/ob_backup_struct.h

@@ -889,7 +889,7 @@ public:
 private:
 #ifdef OB_BUILD_TDE_SECURITY
   virtual int get_access_key_(char *key_buf, const int64_t key_buf_len) const override;
-  virtual int parse_storage_info_(const char *storage_info, bool &has_appid) override;
+  virtual int parse_storage_info_(const char *storage_info, bool &has_needed_extension) override;
   int encrypt_access_key_(char *encrypt_key, const int64_t length) const;
   int decrypt_access_key_(const char *buf);
 #endif

src/share/ob_device_manager.cpp

@@ -50,6 +50,8 @@ int ObDeviceManager::init_devices_env()
       OB_LOG(WARN, "fail to init oss storage", K(ret));
     } else if (OB_FAIL(init_cos_env())) {
       OB_LOG(WARN, "fail to init cos storage", K(ret));
+    } else if (OB_FAIL(init_s3_env())) {
+      OB_LOG(WARN, "fail to init s3 storage", K(ret));
     }
   }
 
@@ -84,6 +86,7 @@ void ObDeviceManager::destroy()
     allocator_.reset();
     fin_oss_env();
     fin_cos_env();
+    fin_s3_env();
     is_init_ = false;
     device_count_ = 0;
     OB_LOG_RET(WARN, ret_dev, "release the init resource", K(ret_dev), K(ret_handle));
@@ -120,6 +123,10 @@ int parse_storage_info(common::ObString storage_type_prefix, ObIODevice*& device
     device_type = OB_STORAGE_COS;
     mem = allocator.alloc(sizeof(ObObjectDevice));
     if (NULL != mem) {new(mem)ObObjectDevice;}
+  } else if (storage_type_prefix.prefix_match(OB_S3_PREFIX)) {
+    device_type = OB_STORAGE_S3;
+    mem = allocator.alloc(sizeof(ObObjectDevice));
+    if (NULL != mem) {new(mem)ObObjectDevice;}
   } else {
     ret = OB_INVALID_BACKUP_DEST;
     OB_LOG(WARN, "invaild device name info!", K(storage_type_prefix));

src/share/ob_errno.def

@@ -2111,6 +2111,7 @@ DEFINE_ERROR(OB_FILE_OR_DIRECTORY_EXIST, -9101, -1, "HY000", "file or directory
 DEFINE_ERROR(OB_FILE_OR_DIRECTORY_PERMISSION_DENIED, -9102, -1, "HY000", "file or directory permission denied");
 DEFINE_ERROR(OB_TOO_MANY_OPEN_FILES, -9103, -1, "HY000", "too many open files");
 DEFINE_ERROR(OB_DIRECT_LOAD_COMMIT_ERROR, -9104, -1, "HY000", "fail to commit direct load");
+DEFINE_ERROR_DEP(OB_S3_ERROR, -9105, -1, "HY000", "S3 error");
 
 ////////////////////////////////////////////////////////////////
 //error code for storage auto extend file

src/share/ob_errno.h

@@ -3743,6 +3743,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219;
 #define OB_FILE_OR_DIRECTORY_PERMISSION_DENIED__USER_ERROR_MSG "file or directory permission denied"
 #define OB_TOO_MANY_OPEN_FILES__USER_ERROR_MSG "too many open files"
 #define OB_DIRECT_LOAD_COMMIT_ERROR__USER_ERROR_MSG "fail to commit direct load"
+#define OB_S3_ERROR__USER_ERROR_MSG "S3 error"
 #define OB_ERR_RESIZE_FILE_TO_SMALLER__USER_ERROR_MSG "Extend ssblock file to smaller is not allowed"
 #define OB_MARK_BLOCK_INFO_TIMEOUT__USER_ERROR_MSG "Mark blocks timeout(5s) in auto extend process when alloc block fail"
 #define OB_NOT_READY_TO_EXTEND_FILE__USER_ERROR_MSG "Auto extend param is not ready to start extending file"
@@ -5939,6 +5940,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219;
 #define OB_FILE_OR_DIRECTORY_PERMISSION_DENIED__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9102, file or directory permission denied"
 #define OB_TOO_MANY_OPEN_FILES__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9103, too many open files"
 #define OB_DIRECT_LOAD_COMMIT_ERROR__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9104, fail to commit direct load"
+#define OB_S3_ERROR__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9105, S3 error"
 #define OB_ERR_RESIZE_FILE_TO_SMALLER__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9200, Extend ssblock file to smaller is not allowed"
 #define OB_MARK_BLOCK_INFO_TIMEOUT__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9201, Mark blocks timeout(5s) in auto extend process when alloc block fail"
 #define OB_NOT_READY_TO_EXTEND_FILE__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9202, Auto extend param is not ready to start extending file"
@@ -6233,7 +6235,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219;
 #define OB_ERR_DATA_TOO_LONG_MSG_FMT_V2__ORA_USER_ERROR_MSG "ORA-12899: value too large for column %.*s (actual: %ld, maximum: %ld)"
 #define OB_ERR_INVALID_DATE_MSG_FMT_V2__ORA_USER_ERROR_MSG "ORA-01861: Incorrect datetime value for column '%.*s' at row %ld"
 
-extern int g_all_ob_errnos[2192];
+extern int g_all_ob_errnos[2193];
 
   const char *ob_error_name(const int oberr);
   const char* ob_error_cause(const int oberr);

src/share/ob_local_device.cpp

@@ -645,6 +645,39 @@ int ObLocalDevice::fstat(const ObIOFd &fd, ObIODFileStat &statbuf)
   return ret;
 }
 
+int ObLocalDevice::del_unmerged_parts(const char *pathname)
+{
+  UNUSED(pathname);
+  return OB_NOT_SUPPORTED;
+}
+
+int ObLocalDevice::adaptive_exist(const char *pathname, bool &is_exist)
+{
+  UNUSED(pathname);
+  UNUSED(is_exist);
+  return OB_NOT_SUPPORTED;
+}
+
+int ObLocalDevice::adaptive_stat(const char *pathname, ObIODFileStat &statbuf)
+{
+  UNUSED(pathname);
+  UNUSED(statbuf);
+  return OB_NOT_SUPPORTED;
+}
+
+int ObLocalDevice::adaptive_unlink(const char *pathname)
+{
+  UNUSED(pathname);
+  return OB_NOT_SUPPORTED;
+}
+
+int ObLocalDevice::adaptive_scan_dir(const char *dir_name, ObBaseDirEntryOperator &op)
+{
+  UNUSED(dir_name);
+  UNUSED(op);
+  return OB_NOT_SUPPORTED;
+}
+
 //block interfaces
 int ObLocalDevice::mark_blocks(ObIBlockIterator &block_iter)
 {

src/share/ob_local_device.h

@@ -88,6 +88,13 @@ public:
   virtual int stat(const char *pathname, common::ObIODFileStat &statbuf) override;
   virtual int fstat(const common::ObIOFd &fd, common::ObIODFileStat &statbuf) override;
 
+  //for object device, local device should not use these
+  int del_unmerged_parts(const char *pathname);
+  int adaptive_exist(const char *pathname, bool &is_exist);
+  int adaptive_stat(const char *pathname, ObIODFileStat &statbuf);
+  int adaptive_unlink(const char *pathname);
+  int adaptive_scan_dir(const char *dir_name, ObBaseDirEntryOperator &op);
+
   //block interfaces
   virtual int mark_blocks(common::ObIBlockIterator &block_iter) override;
   virtual int alloc_block(const common::ObIODOpts *opts, common::ObIOFd &block_id) override;

src/storage/backup/ob_backup_ctx.cpp

@@ -400,7 +400,7 @@ int ObBackupDataCtx::open_file_writer_(const share::ObBackupPath &backup_path)
 {
   int ret = OB_SUCCESS;
   common::ObBackupIoAdapter util;
-  const ObStorageAccessType access_type = OB_STORAGE_ACCESS_RANDOMWRITER;
+  const ObStorageAccessType access_type = OB_STORAGE_ACCESS_MULTIPART_WRITER;
   if (OB_FAIL(util.mk_parent_dir(backup_path.get_obstr(), param_.backup_dest_.get_storage_info()))) {
     LOG_WARN("failed to make parent dir", K(backup_path));
   } else if (OB_FAIL(util.open_with_access_type(

src/storage/backup/ob_backup_extern_info_mgr.cpp

@@ -371,7 +371,7 @@ int ObExternTabletMetaWriter::prepare_backup_file_(const int64_t file_id)
   int ret = OB_SUCCESS;
   share::ObBackupPath backup_path;
   common::ObBackupIoAdapter util;
-  const ObStorageAccessType access_type = OB_STORAGE_ACCESS_RANDOMWRITER;
+  const ObStorageAccessType access_type = OB_STORAGE_ACCESS_MULTIPART_WRITER;
   const int64_t data_file_size = get_data_file_size();
   if (OB_FAIL(ObBackupPathUtil::get_ls_data_tablet_info_path(
       backup_set_dest_, ls_id_, turn_id_, retry_id_, file_id, backup_path))) {

src/storage/backup/ob_backup_index_merger.cpp

@@ -572,7 +572,7 @@ int ObIBackupIndexMerger::open_file_writer_(const share::ObBackupPath &path, con
 {
   int ret = OB_SUCCESS;
   common::ObBackupIoAdapter util;
-  const ObStorageAccessType access_type = OB_STORAGE_ACCESS_RANDOMWRITER;
+  const ObStorageAccessType access_type = OB_STORAGE_ACCESS_MULTIPART_WRITER;
   if (OB_FAIL(util.mk_parent_dir(path.get_obstr(), storage_info))) {
     LOG_WARN("failed to make parent dir", K(path), K(path), KP(storage_info));
   } else if (OB_FAIL(util.open_with_access_type(dev_handle_, io_fd_, storage_info, path.get_obstr(), access_type))) {

src/storage/backup/ob_backup_task.cpp

@@ -5280,7 +5280,7 @@ int ObLSBackupComplementLogTask::inner_get_piece_file_list_(const share::ObLSID
   const share::SCN &start_scn = piece_attr.start_scn_;
   if (OB_FAIL(get_src_backup_piece_dir_(ls_id, piece_attr, src_piece_dir_path))) {
     LOG_WARN("failed to get src backup piece dir", K(ret), K(round_id), K(piece_id), K(ls_id), K(piece_attr));
-  } else if (OB_FAIL(util.list_files(src_piece_dir_path.get_obstr(), archive_dest_.get_storage_info(), op))) {
+  } else if (OB_FAIL(util.adaptively_list_files(src_piece_dir_path.get_obstr(), archive_dest_.get_storage_info(), op))) {
     LOG_WARN("failed to list files", K(ret), K(src_piece_dir_path));
   } else if (OB_FAIL(op.get_file_id_list(file_id_list))) {
     LOG_WARN("failed to get files", K(ret));
@@ -5551,7 +5551,7 @@ int ObLSBackupComplementLogTask::inner_transfer_clog_file_(
   } else if (OB_ISNULL(buf = static_cast<char *>(allocator.alloc(transfer_len)))) {
     ret = OB_ALLOCATE_MEMORY_FAILED;
     LOG_WARN("failed to allocate memory", K(ret), K(transfer_len));
-  } else if (OB_FAIL(util.read_part_file(src_path.get_obstr(), archive_dest_.get_storage_info(), buf, transfer_len, dst_len, read_len))) {
+  } else if (OB_FAIL(util.adaptively_read_part_file(src_path.get_obstr(), archive_dest_.get_storage_info(), buf, transfer_len, dst_len, read_len))) {
     LOG_WARN("failed to read part file", K(ret), K(src_path));
   } else if (read_len != transfer_len) {
     ret = OB_ERR_UNEXPECTED;
@@ -5585,7 +5585,7 @@ int ObLSBackupComplementLogTask::get_file_length_(
 {
   int ret = OB_SUCCESS;
   ObBackupIoAdapter util;
-  if (OB_FAIL(util.get_file_length(file_path, storage_info, length))) {
+  if (OB_FAIL(util.adaptively_get_file_length(file_path, storage_info, length))) {
     if (OB_BACKUP_FILE_NOT_EXIST == ret) {
       ret = OB_SUCCESS;
       length = 0;

unittest/share/backup/CMakeLists.txt

@@ -2,4 +2,5 @@ storage_unittest(test_backup_path)
 storage_unittest(test_backup_struct)
 storage_unittest(test_log_archive_backup_info_mgr)
 storage_unittest(test_backup_access_cos)
+storage_unittest(test_backup_access_s3)
 storage_unittest(test_archive_checkpoint_mgr)

unittest/share/backup/test_backup_access_s3.cpp

@@ -0,0 +1,323 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include <gtest/gtest.h>
+#include "lib/utility/ob_test_util.h"
+#include "lib/restore/ob_storage.h"
+#include "lib/restore/ob_storage_s3_base.h"
+#include "share/backup/ob_backup_io_adapter.h"
+
+#include "lib/allocator/page_arena.h"
+#include "test_backup_access_s3.h"
+
+using namespace oceanbase::common;
+
+class TestStorageS3Common {
+public:
+TestStorageS3Common() {}
+~TestStorageS3Common() {}
+
+void init()
+{
+  ASSERT_EQ(OB_SUCCESS,
+            databuff_printf(account, sizeof(account),
+            "s3_region=%s&host=%s&access_id=%s&access_key=%s",
+            region, endpoint, secretid, secretkey));
+  //build s3_base
+  const ObString s3_storage_info(account);
+  ASSERT_EQ(OB_SUCCESS, s3_base.set(ObStorageType::OB_STORAGE_S3, s3_storage_info.ptr()));
+}
+void destory()
+{
+}
+protected:
+  char account[OB_MAX_URI_LENGTH];
+  const char *dir_name = "test_backup_io_adapter_access_s3_dir";
+  char uri[OB_MAX_URI_LENGTH];
+  char dir_uri[OB_MAX_URI_LENGTH];
+  oceanbase::share::ObBackupStorageInfo s3_base;
+
+  int object_prefix_len = 5;
+};
+
+class TestBackupIOAdapterAccessS3 : public ::testing::Test, public TestStorageS3Common
+{
+public:
+  TestBackupIOAdapterAccessS3() : enable_test_(enable_test) {}
+  virtual ~TestBackupIOAdapterAccessS3() {}
+  virtual void SetUp()
+  {
+    init();
+  }
+  virtual void TearDown()
+  {
+    destory();
+  }
+
+  static void SetUpTestCase()
+  {
+  }
+
+  static void TearDownTestCase()
+  {
+  }
+
+private:
+  // disallow copy
+  DISALLOW_COPY_AND_ASSIGN(TestBackupIOAdapterAccessS3);
+protected:
+  bool enable_test_;
+};
+
+TEST_F(TestBackupIOAdapterAccessS3, test_basic_rw)
+{
+  int ret = OB_SUCCESS;
+  if (enable_test_) {
+    ObBackupIoAdapter adapter;
+    const char *tmp_dir = "test_basic_rw";
+    const int64_t ts = ObTimeUtility::current_time();
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld",
+        bucket, dir_name, tmp_dir, ts));
+
+    // write
+    const char *write_content = "123456789ABCDEF";
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/0", dir_uri));
+    ASSERT_EQ(OB_SUCCESS,
+        adapter.write_single_file(uri, &s3_base, write_content, strlen(write_content)));
+
+    // read
+    char read_buf[100] = {0};
+    int64_t read_size = 0;
+    ASSERT_EQ(OB_SUCCESS,
+        adapter.read_single_file(uri, &s3_base, read_buf, sizeof(read_buf), read_size));
+    ASSERT_STREQ(write_content, read_buf);
+    ASSERT_EQ(strlen(write_content), read_size);
+
+    ASSERT_EQ(OB_SUCCESS,
+        adapter.read_part_file(uri, &s3_base, read_buf, sizeof(read_buf), 0, read_size));
+    ASSERT_STREQ(write_content, read_buf);
+    ASSERT_EQ(strlen(write_content), read_size);
+
+    int64_t offset = 5;
+    ASSERT_EQ(OB_SUCCESS,
+        adapter.read_part_file(uri, &s3_base, read_buf, sizeof(read_buf), offset, read_size));
+    ASSERT_EQ('6', read_buf[0]);
+    ASSERT_EQ('F', read_buf[9]);
+    ASSERT_EQ(strlen(write_content) - offset, read_size);
+
+    offset = 6;
+    ASSERT_EQ(OB_SUCCESS,
+        adapter.read_part_file(uri, &s3_base, read_buf, 5, offset, read_size));
+    ASSERT_EQ('7', read_buf[0]);
+    ASSERT_EQ('B', read_buf[4]);
+    ASSERT_EQ(5, read_size);
+
+    // offset = strlen(write_content);
+    // ASSERT_EQ(OB_COS_ERROR,
+    //     adapter.read_part_file(uri, &s3_base, read_buf, sizeof(read_buf), offset, read_size));
+
+    ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &s3_base));
+  }
+}
+
+TEST_F(TestBackupIOAdapterAccessS3, test_util)
+{
+  int ret = OB_SUCCESS;
+  if (enable_test_) {
+    ObBackupIoAdapter adapter;
+    const char *tmp_dir = "test_util";
+    const int64_t ts = ObTimeUtility::current_time();
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld",
+        bucket, dir_name, tmp_dir, ts));
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/0", dir_uri));
+
+    bool is_obj_exist = true;
+    ASSERT_EQ(OB_SUCCESS, adapter.is_exist(uri, &s3_base, is_obj_exist));
+    ASSERT_FALSE(is_obj_exist);
+
+    const char *write_content = "123456789ABCDEF";
+    ASSERT_EQ(OB_SUCCESS,
+        adapter.write_single_file(uri, &s3_base, write_content, strlen(write_content)));
+    ASSERT_EQ(OB_SUCCESS, adapter.is_exist(uri, &s3_base, is_obj_exist));
+    ASSERT_TRUE(is_obj_exist);
+
+    int64_t file_length = -1;
+    ASSERT_EQ(OB_SUCCESS, adapter.get_file_length(uri, &s3_base, file_length));
+    ASSERT_EQ(strlen(write_content), file_length);
+
+    ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &s3_base));
+    ASSERT_EQ(OB_SUCCESS, adapter.is_exist(uri, &s3_base, is_obj_exist));
+    ASSERT_FALSE(is_obj_exist);
+  }
+}
+
+TEST_F(TestBackupIOAdapterAccessS3, test_list_files)
+{
+  int ret = OB_SUCCESS;
+  if (enable_test_) {
+    ObBackupIoAdapter adapter;
+    const char *tmp_dir = "test_list_files";
+    const int64_t ts = ObTimeUtility::current_time();
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld",
+        bucket, dir_name, tmp_dir, ts));
+
+    int64_t file_num = 11;
+    const char *write_content = "0123456789";
+    const char *format = "%s/%0*ld_%ld";
+    for (int64_t file_idx = 0; file_idx < file_num; file_idx++) {
+      ASSERT_EQ(OB_SUCCESS,
+          databuff_printf(uri, sizeof(uri), format, dir_uri, object_prefix_len, file_idx, file_idx));
+      ASSERT_EQ(OB_SUCCESS,
+          adapter.write_single_file(uri, &s3_base, write_content, strlen(write_content)));
+    }
+
+    ObArenaAllocator allocator;
+    ObArray<ObString> name_array;
+    ObFileListArrayOp op(name_array, allocator);
+    ASSERT_EQ(OB_SUCCESS, adapter.list_files(dir_uri, &s3_base, op));
+    ASSERT_EQ(file_num, name_array.size());
+    for (int64_t file_idx = 0; file_idx < file_num; file_idx++) {
+      // listed files do not contain prefix
+      ASSERT_EQ(OB_SUCCESS,
+          databuff_printf(uri, sizeof(uri), "%0*ld_%ld", object_prefix_len, file_idx, file_idx));
+      ASSERT_STREQ(uri, name_array[file_idx].ptr());
+      ASSERT_EQ(OB_SUCCESS,
+          databuff_printf(uri, sizeof(uri), format, dir_uri, object_prefix_len, file_idx, file_idx));
+      ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &s3_base));
+    }
+  }
+}
+
+TEST_F(TestBackupIOAdapterAccessS3, test_list_directories)
+{
+  int ret = OB_SUCCESS;
+  if (enable_test_) {
+    ObBackupIoAdapter adapter;
+    const char *tmp_dir = "test_list_directories";
+    const int64_t ts = ObTimeUtility::current_time();
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld",
+        bucket, dir_name, tmp_dir, ts));
+
+    int64_t file_num = 11;
+    const char *write_content = "0123456789";
+    const char *format = "%s/%0*ld/%ld";
+    for (int64_t file_idx = 0; file_idx < file_num; file_idx++) {
+      ASSERT_EQ(OB_SUCCESS,
+          databuff_printf(uri, sizeof(uri), format, dir_uri, object_prefix_len, file_idx, file_idx));
+      ASSERT_EQ(OB_SUCCESS,
+          adapter.write_single_file(uri, &s3_base, write_content, strlen(write_content)));
+    }
+
+    ObArenaAllocator allocator;
+    ObArray<ObString> name_array;
+    ObFileListArrayOp op(name_array, allocator);
+    ASSERT_EQ(OB_SUCCESS, adapter.list_directories(dir_uri, &s3_base, op));
+    ASSERT_EQ(file_num, name_array.size());
+    for (int64_t file_idx = 0; file_idx < file_num; file_idx++) {
+      // listed files do not contain prefix
+      ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%0*ld", object_prefix_len, file_idx));
+      ASSERT_STREQ(uri, name_array[file_idx].ptr());
+      ASSERT_EQ(OB_SUCCESS,
+          databuff_printf(uri, sizeof(uri), format, dir_uri, object_prefix_len, file_idx, file_idx));
+      ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &s3_base));
+    }
+  }
+}
+
+TEST_F(TestBackupIOAdapterAccessS3, test_is_tagging)
+{
+  int ret = OB_SUCCESS;
+  if (enable_test_) {
+    ObBackupIoAdapter adapter;
+    const char *tmp_util_dir = "test_util_is_tagging";
+    const int64_t ts = ObTimeUtility::current_time();
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld",
+      bucket, dir_name, tmp_util_dir, ts));
+
+    bool is_tagging = true;
+    char tmp_account[OB_MAX_URI_LENGTH];
+    oceanbase::share::ObBackupStorageInfo tmp_s3_base;
+    const char *write_content = "123456789ABCDEF";
+
+    // wrong tag mode
+    ASSERT_EQ(OB_SUCCESS,
+              databuff_printf(tmp_account, sizeof(tmp_account),
+              "s3_region=%s&host=%s&access_id=%s&access_key=%s&delete_mode=tag",
+              region, endpoint, secretid, secretkey));
+    ASSERT_EQ(OB_INVALID_ARGUMENT, tmp_s3_base.set(ObStorageType::OB_STORAGE_S3, tmp_account));
+    tmp_s3_base.reset();
+
+    ASSERT_EQ(OB_SUCCESS,
+              databuff_printf(tmp_account, sizeof(tmp_account),
+              "s3_region=%s&host=%s&access_id=%s&access_key=%s&delete_mode=delete_delete",
+              region, endpoint, secretid, secretkey));
+    ASSERT_EQ(OB_INVALID_ARGUMENT, tmp_s3_base.set(ObStorageType::OB_STORAGE_S3, tmp_account));
+    tmp_s3_base.reset();
+
+    // delete mode
+    ASSERT_EQ(OB_SUCCESS,
+              databuff_printf(tmp_account, sizeof(tmp_account),
+              "s3_region=%s&host=%s&access_id=%s&access_key=%s&delete_mode=delete",
+              region, endpoint, secretid, secretkey));
+    ASSERT_EQ(OB_SUCCESS, tmp_s3_base.set(ObStorageType::OB_STORAGE_S3, tmp_account));
+
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/delete_mode", dir_uri));
+    ASSERT_EQ(OB_SUCCESS,
+        adapter.write_single_file(uri, &tmp_s3_base, write_content, strlen(write_content)));
+    ASSERT_EQ(OB_SUCCESS, adapter.is_tagging(uri, &tmp_s3_base, is_tagging));
+    ASSERT_FALSE(is_tagging);
+
+    ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &tmp_s3_base));
+    ASSERT_EQ(OB_BACKUP_FILE_NOT_EXIST, adapter.is_tagging(uri, &tmp_s3_base, is_tagging));
+    tmp_s3_base.reset();
+
+    // tagging mode
+    ASSERT_EQ(OB_SUCCESS,
+              databuff_printf(tmp_account, sizeof(tmp_account),
+              "s3_region=%s&host=%s&access_id=%s&access_key=%s&delete_mode=tagging",
+              region, endpoint, secretid, secretkey));
+    ASSERT_EQ(OB_SUCCESS, tmp_s3_base.set(ObStorageType::OB_STORAGE_S3, tmp_account));
+
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/tagging_mode", dir_uri));
+    ASSERT_EQ(OB_SUCCESS,
+        adapter.write_single_file(uri, &tmp_s3_base, write_content, strlen(write_content)));
+
+    is_tagging = true;
+    ASSERT_EQ(OB_SUCCESS, adapter.is_tagging(uri, &tmp_s3_base, is_tagging));
+    ASSERT_FALSE(is_tagging);
+
+    ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &tmp_s3_base));
+    ASSERT_EQ(OB_SUCCESS, adapter.is_tagging(uri, &tmp_s3_base, is_tagging));
+    ASSERT_TRUE(is_tagging);
+    tmp_s3_base.reset();
+
+    // clean
+    ASSERT_EQ(OB_SUCCESS,
+              databuff_printf(tmp_account, sizeof(tmp_account),
+              "s3_region=%s&host=%s&access_id=%s&access_key=%s",
+              region, endpoint, secretid, secretkey));
+    ASSERT_EQ(OB_SUCCESS, tmp_s3_base.set(ObStorageType::OB_STORAGE_S3, tmp_account));
+
+    ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/tagging_mode", dir_uri));
+    ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &tmp_s3_base));
+    ASSERT_EQ(OB_BACKUP_FILE_NOT_EXIST, adapter.is_tagging(uri, &tmp_s3_base, is_tagging));
+  }
+}
+
+int main(int argc, char **argv)
+{
+  system("rm -f test_backup_access_s3.log*");
+  OB_LOGGER.set_file_name("test_backup_access_s3.log", true, true);
+  OB_LOGGER.set_log_level("DEBUG");
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

unittest/share/backup/test_backup_access_s3.h

@@ -0,0 +1,23 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef UNITTEST_BACKUP_ACCESS_S3_H_
+#define UNITTEST_BACKUP_ACCESS_S3_H_
+
+const bool enable_test = false;
+const char *bucket = "s3://xxx";
+const char *region = "xxx";
+const char *endpoint = "xxx";
+const char *secretid = "xxx";
+const char *secretkey = "xxx";
+
+#endif