From 542516e36991a1273476512b12dc409e9d3ec4f7 Mon Sep 17 00:00:00 2001 From: obdev Date: Wed, 7 Feb 2024 19:13:39 +0000 Subject: [PATCH] [FEAT MERGE] Backup support AWS S3 Co-authored-by: xuhuleon --- deps/init/oceanbase.el7.aarch64.deps | 1 + deps/init/oceanbase.el7.x86_64.deps | 1 + deps/init/oceanbase.el8.aarch64.deps | 1 + deps/init/oceanbase.el8.x86_64.deps | 1 + deps/init/oceanbase.el9.aarch64.deps | 1 + deps/init/oceanbase.el9.x86_64.deps | 1 + .../src/common/storage/ob_device_common.h | 9 +- deps/oblib/src/common/storage/ob_io_device.h | 5 + deps/oblib/src/lib/allocator/ob_malloc.cpp | 33 +- deps/oblib/src/lib/ob_define.h | 7 +- deps/oblib/src/lib/ob_errno.h | 1 + deps/oblib/src/lib/restore/CMakeLists.txt | 24 +- .../src/lib/restore/cos/ob_cos_wrapper.cpp | 889 +++++-- .../src/lib/restore/cos/ob_cos_wrapper.h | 92 +- .../lib/restore/cos/ob_cos_wrapper_handle.cpp | 10 + .../lib/restore/cos/ob_cos_wrapper_handle.h | 3 + deps/oblib/src/lib/restore/ob_i_storage.cpp | 600 +++++ deps/oblib/src/lib/restore/ob_i_storage.h | 212 +- .../src/lib/restore/ob_object_device.cpp | 225 +- deps/oblib/src/lib/restore/ob_object_device.h | 18 + deps/oblib/src/lib/restore/ob_storage.cpp | 1380 ++++++++++- deps/oblib/src/lib/restore/ob_storage.h | 213 +- .../src/lib/restore/ob_storage_cos_base.cpp | 683 ++++-- .../src/lib/restore/ob_storage_cos_base.h | 52 +- .../oblib/src/lib/restore/ob_storage_file.cpp | 255 +- deps/oblib/src/lib/restore/ob_storage_file.h | 27 +- .../oblib/src/lib/restore/ob_storage_info.cpp | 26 +- deps/oblib/src/lib/restore/ob_storage_info.h | 1 + .../src/lib/restore/ob_storage_oss_base.cpp | 931 ++++---- .../src/lib/restore/ob_storage_oss_base.h | 19 +- .../src/lib/restore/ob_storage_s3_base.cpp | 2096 +++++++++++++++++ .../src/lib/restore/ob_storage_s3_base.h | 547 +++++ deps/oblib/unittest/lib/CMakeLists.txt | 3 + .../lib/alloc/test_malloc_allocator.cpp | 12 + .../lib/restore/test_common_storage.cpp | 1439 +++++++++++ .../lib/restore/test_common_storage.h | 149 ++ .../lib/restore/test_common_storage_util.h | 145 ++ .../lib/restore/test_object_storage.cpp | 1272 ++++++++++ .../lib/restore/test_object_storage.h | 24 + .../unittest/lib/restore/test_storage_cos.cpp | 38 + .../lib/restore/test_storage_info.cpp | 25 + .../unittest/lib/restore/test_storage_s3.cpp | 1108 +++++++++ .../unittest/lib/restore/test_storage_s3.h | 24 + .../archiveservice/ob_archive_file_utils.cpp | 10 +- .../archiveservice/ob_archive_io.cpp | 10 +- src/logservice/archiveservice/ob_archive_io.h | 3 +- .../archiveservice/ob_archive_sender.cpp | 8 +- .../archiveservice/ob_archive_sender.h | 1 + .../ob_log_external_storage_io_task.cpp | 6 +- src/share/backup/ob_archive_store.cpp | 4 +- src/share/backup/ob_backup_connectivity.cpp | 18 +- src/share/backup/ob_backup_io_adapter.cpp | 184 +- src/share/backup/ob_backup_io_adapter.h | 16 + src/share/backup/ob_backup_struct.cpp | 4 +- src/share/backup/ob_backup_struct.h | 2 +- src/share/ob_device_manager.cpp | 7 + src/share/ob_errno.cpp | 15 +- src/share/ob_errno.def | 1 + src/share/ob_errno.h | 4 +- src/share/ob_local_device.cpp | 33 + src/share/ob_local_device.h | 7 + src/storage/backup/ob_backup_ctx.cpp | 2 +- .../backup/ob_backup_extern_info_mgr.cpp | 2 +- src/storage/backup/ob_backup_index_merger.cpp | 2 +- src/storage/backup/ob_backup_task.cpp | 6 +- unittest/share/backup/CMakeLists.txt | 1 + .../share/backup/test_backup_access_s3.cpp | 323 +++ unittest/share/backup/test_backup_access_s3.h | 23 + 68 files changed, 12172 insertions(+), 1123 deletions(-) create mode 100644 deps/oblib/src/lib/restore/ob_i_storage.cpp create mode 100644 deps/oblib/src/lib/restore/ob_storage_s3_base.cpp create mode 100644 deps/oblib/src/lib/restore/ob_storage_s3_base.h create mode 100644 deps/oblib/unittest/lib/restore/test_common_storage.cpp create mode 100644 deps/oblib/unittest/lib/restore/test_common_storage.h create mode 100644 deps/oblib/unittest/lib/restore/test_common_storage_util.h create mode 100644 deps/oblib/unittest/lib/restore/test_object_storage.cpp create mode 100644 deps/oblib/unittest/lib/restore/test_object_storage.h create mode 100644 deps/oblib/unittest/lib/restore/test_storage_s3.cpp create mode 100644 deps/oblib/unittest/lib/restore/test_storage_s3.h create mode 100644 unittest/share/backup/test_backup_access_s3.cpp create mode 100644 unittest/share/backup/test_backup_access_s3.h diff --git a/deps/init/oceanbase.el7.aarch64.deps b/deps/init/oceanbase.el7.aarch64.deps index eff3a095c..a4dba81c6 100644 --- a/deps/init/oceanbase.el7.aarch64.deps +++ b/deps/init/oceanbase.el7.aarch64.deps @@ -28,6 +28,7 @@ devdeps-boost-1.74.0-22022110914.el7.aarch64.rpm devdeps-s2geometry-0.9.0-12023092021.el7.aarch64.rpm devdeps-icu-69.1-72022112416.el7.aarch64.rpm devdeps-cos-c-sdk-5.0.16-52023070517.el7.aarch64.rpm +devdeps-s3-cpp-sdk-1.11.156-62023101011.el7.aarch64.rpm [tools] obdevtools-binutils-2.30-12022100413.el7.aarch64.rpm diff --git a/deps/init/oceanbase.el7.x86_64.deps b/deps/init/oceanbase.el7.x86_64.deps index f5e6bc158..8df49de46 100644 --- a/deps/init/oceanbase.el7.x86_64.deps +++ b/deps/init/oceanbase.el7.x86_64.deps @@ -30,6 +30,7 @@ devdeps-boost-1.74.0-22022110914.el7.x86_64.rpm devdeps-s2geometry-0.9.0-12023092021.el7.x86_64.rpm devdeps-icu-69.1-72022112416.el7.x86_64.rpm devdeps-cos-c-sdk-5.0.16-52023070517.el7.x86_64.rpm +devdeps-s3-cpp-sdk-1.11.156-62023101011.el7.x86_64.rpm [tools] obdevtools-binutils-2.30-12022100413.el7.x86_64.rpm diff --git a/deps/init/oceanbase.el8.aarch64.deps b/deps/init/oceanbase.el8.aarch64.deps index 0b5c1346a..ad9b9f8d0 100644 --- a/deps/init/oceanbase.el8.aarch64.deps +++ b/deps/init/oceanbase.el8.aarch64.deps @@ -28,6 +28,7 @@ devdeps-boost-1.74.0-22022110914.el8.aarch64.rpm devdeps-s2geometry-0.9.0-12023092021.el8.aarch64.rpm devdeps-icu-69.1-72022112416.el8.aarch64.rpm devdeps-cos-c-sdk-5.0.16-52023070517.el8.aarch64.rpm +devdeps-s3-cpp-sdk-1.11.156-62023101011.el8.aarch64.rpm [tools] obdevtools-binutils-2.30-12022100413.el8.aarch64.rpm diff --git a/deps/init/oceanbase.el8.x86_64.deps b/deps/init/oceanbase.el8.x86_64.deps index 72e768014..f32a40fc7 100644 --- a/deps/init/oceanbase.el8.x86_64.deps +++ b/deps/init/oceanbase.el8.x86_64.deps @@ -29,6 +29,7 @@ devdeps-boost-1.74.0-22022110914.el8.x86_64.rpm devdeps-s2geometry-0.9.0-12023092021.el8.x86_64.rpm devdeps-icu-69.1-72022112416.el8.x86_64.rpm devdeps-cos-c-sdk-5.0.16-52023070517.el8.x86_64.rpm +devdeps-s3-cpp-sdk-1.11.156-62023101011.el8.x86_64.rpm [tools] obdevtools-binutils-2.30-12022100413.el8.x86_64.rpm diff --git a/deps/init/oceanbase.el9.aarch64.deps b/deps/init/oceanbase.el9.aarch64.deps index 136767c7a..d82a2088e 100644 --- a/deps/init/oceanbase.el9.aarch64.deps +++ b/deps/init/oceanbase.el9.aarch64.deps @@ -32,6 +32,7 @@ devdeps-boost-1.74.0-22022110914.el8.aarch64.rpm devdeps-s2geometry-0.9.0-12023092021.el8.aarch64.rpm devdeps-icu-69.1-72022112416.el8.aarch64.rpm devdeps-cos-c-sdk-5.0.16-52023070517.el8.aarch64.rpm +devdeps-s3-cpp-sdk-1.11.156-62023101011.el9.aarch64.rpm [deps-apr-el9] devdeps-apr-1.6.5-232023090616.el9.aarch64.rpm target=apr-el9 diff --git a/deps/init/oceanbase.el9.x86_64.deps b/deps/init/oceanbase.el9.x86_64.deps index e9712b315..7b6965f5e 100644 --- a/deps/init/oceanbase.el9.x86_64.deps +++ b/deps/init/oceanbase.el9.x86_64.deps @@ -33,6 +33,7 @@ devdeps-boost-1.74.0-22022110914.el8.x86_64.rpm devdeps-s2geometry-0.9.0-12023092021.el8.x86_64.rpm devdeps-icu-69.1-72022112416.el8.x86_64.rpm devdeps-cos-c-sdk-5.0.16-52023070517.el8.x86_64.rpm +devdeps-s3-cpp-sdk-1.11.156-62023101011.el9.x86_64.rpm [deps-apr-el9] devdeps-apr-1.6.5-232023090616.el9.x86_64.rpm target=apr-el9 diff --git a/deps/oblib/src/common/storage/ob_device_common.h b/deps/oblib/src/common/storage/ob_device_common.h index 0fe7c2c45..fd98bb675 100644 --- a/deps/oblib/src/common/storage/ob_device_common.h +++ b/deps/oblib/src/common/storage/ob_device_common.h @@ -51,15 +51,18 @@ enum ObStorageType OB_STORAGE_FILE = 1, OB_STORAGE_COS = 2, OB_STORAGE_LOCAL = 3, + OB_STORAGE_S3 = 4, OB_STORAGE_MAX_TYPE }; enum ObStorageAccessType { OB_STORAGE_ACCESS_READER = 0, - OB_STORAGE_ACCESS_OVERWRITER = 1, - OB_STORAGE_ACCESS_APPENDER = 2, - OB_STORAGE_ACCESS_RANDOMWRITER = 3, + OB_STORAGE_ACCESS_ADAPTIVE_READER = 1, + OB_STORAGE_ACCESS_OVERWRITER = 2, + OB_STORAGE_ACCESS_APPENDER = 3, + OB_STORAGE_ACCESS_RANDOMWRITER = 4, + OB_STORAGE_ACCESS_MULTIPART_WRITER = 5, OB_STORAGE_ACCESS_MAX_TYPE }; diff --git a/deps/oblib/src/common/storage/ob_io_device.h b/deps/oblib/src/common/storage/ob_io_device.h index 3625e5108..1658ea41d 100644 --- a/deps/oblib/src/common/storage/ob_io_device.h +++ b/deps/oblib/src/common/storage/ob_io_device.h @@ -351,6 +351,11 @@ public: virtual int exist(const char *pathname, bool &is_exist) = 0; virtual int stat(const char *pathname, ObIODFileStat &statbuf) = 0; virtual int fstat(const ObIOFd &fd, ObIODFileStat &statbuf) = 0; + virtual int del_unmerged_parts(const char *pathname) = 0; + virtual int adaptive_exist(const char *pathname, bool &is_exist) = 0; + virtual int adaptive_stat(const char *pathname, ObIODFileStat &statbuf) = 0; + virtual int adaptive_unlink(const char *pathname) = 0; + virtual int adaptive_scan_dir(const char *dir_name, ObBaseDirEntryOperator &op) = 0; //block interfaces virtual int mark_blocks(ObIBlockIterator &block_iter) = 0; diff --git a/deps/oblib/src/lib/allocator/ob_malloc.cpp b/deps/oblib/src/lib/allocator/ob_malloc.cpp index 7c7c79246..7ebf8559c 100644 --- a/deps/oblib/src/lib/allocator/ob_malloc.cpp +++ b/deps/oblib/src/lib/allocator/ob_malloc.cpp @@ -53,36 +53,17 @@ int oceanbase::common::ObMemBuf::ensure_space(const int64_t size, const lib::ObL void *oceanbase::common::ob_malloc_align(const int64_t alignment, const int64_t nbyte, const lib::ObLabel &label) { - char *ptr = static_cast(oceanbase::common::ob_malloc(nbyte + alignment, label)); - char *align_ptr = NULL; - if (NULL != ptr) { - align_ptr = reinterpret_cast(oceanbase::common::upper_align(reinterpret_cast(ptr), - alignment)); - if (align_ptr == ptr) { - align_ptr = ptr + alignment; - } - int64_t padding = align_ptr - ptr; - if (!(padding <= alignment && padding > 0)) { - _OB_LOG_RET(ERROR, OB_INVALID_ARGUMENT, "invalid padding(padding=%ld, alignment=%ld", padding, alignment); - } - uint8_t *sign_ptr = reinterpret_cast(align_ptr - 1); - int64_t *header_ptr = reinterpret_cast(align_ptr - 1 - sizeof(int64_t)); - if (padding < (int64_t)sizeof(int64_t) + 1) { - *sign_ptr = static_cast(padding) & 0x7f; - } else { - *sign_ptr = 0x80; - *header_ptr = padding; - } - } else { - _OB_LOG_RET(WARN, OB_ALLOCATE_MEMORY_FAILED, "ob_tc_malloc allocate memory failed, alignment[%ld], nbyte[%ld], label[%s].", - alignment, nbyte, (const char *)label); - } - return align_ptr; + ObMemAttr attr; + attr.label_ = label; + return ob_malloc_align(alignment, nbyte, attr); } -void *oceanbase::common::ob_malloc_align(const int64_t alignment, const int64_t nbyte, +void *oceanbase::common::ob_malloc_align(const int64_t align, const int64_t nbyte, const ObMemAttr &attr) { + const int min_align = 16; + const int64_t alignment = + align <= min_align ? min_align : align_up2(align, 16); char *ptr = static_cast(oceanbase::common::ob_malloc(nbyte + alignment, attr)); char *align_ptr = NULL; if (NULL != ptr) { diff --git a/deps/oblib/src/lib/ob_define.h b/deps/oblib/src/lib/ob_define.h index fbc5c5214..0ce3f51c9 100644 --- a/deps/oblib/src/lib/ob_define.h +++ b/deps/oblib/src/lib/ob_define.h @@ -695,6 +695,11 @@ const char *const OB_LOCAL_PREFIX = "local://"; const char *const OB_OSS_PREFIX = "oss://"; const char *const OB_FILE_PREFIX = "file://"; const char *const OB_COS_PREFIX = "cos://"; +const char *const OB_S3_PREFIX = "s3://"; +const char *const OB_S3_APPENDABLE_FORMAT_META = "FORMAT_META"; +const char *const OB_S3_APPENDABLE_SEAL_META = "SEAL_META"; +const char *const OB_S3_APPENDABLE_FRAGMENT_PREFIX = "@APD_PART@"; +const int64_t OB_STORAGE_LIST_MAX_NUM = 1000; const char *const OB_RESOURCE_UNIT_DEFINITION = "resource_unit_definition"; const char *const OB_RESOURCE_POOL_DEFINITION = "resource_pool_definition"; const char *const OB_CREATE_TENANT_DEFINITION = "create_tenant_definition"; @@ -2523,7 +2528,7 @@ inline bool is_x86() { #endif } #define __maybe_unused __attribute__((unused)) -#define DO_PRAGMA(x) _Pragma (#x) +#define DO_PRAGMA(x) _Pragma(#x) #define DISABLE_WARNING_GCC_PUSH _Pragma("GCC diagnostic push") #define DISABLE_WARNING_GCC(option) DO_PRAGMA(GCC diagnostic ignored option) #define DISABLE_WARNING_GCC_POP _Pragma("GCC diagnostic pop") diff --git a/deps/oblib/src/lib/ob_errno.h b/deps/oblib/src/lib/ob_errno.h index 9d5860d73..d04531743 100644 --- a/deps/oblib/src/lib/ob_errno.h +++ b/deps/oblib/src/lib/ob_errno.h @@ -397,6 +397,7 @@ constexpr int OB_BACKUP_PWRITE_OFFSET_NOT_MATCH = -9083; constexpr int OB_BACKUP_PWRITE_CONTENT_NOT_MATCH = -9084; constexpr int OB_CLOUD_OBJECT_NOT_APPENDABLE = -9098; constexpr int OB_RESTORE_TENANT_FAILED = -9099; +constexpr int OB_S3_ERROR = -9105; constexpr int OB_ERR_XML_PARSE = -9549; constexpr int OB_ERR_XSLT_PARSE = -9574; constexpr int OB_MAX_RAISE_APPLICATION_ERROR = -20000; diff --git a/deps/oblib/src/lib/restore/CMakeLists.txt b/deps/oblib/src/lib/restore/CMakeLists.txt index 595cdf56d..40691479f 100644 --- a/deps/oblib/src/lib/restore/CMakeLists.txt +++ b/deps/oblib/src/lib/restore/CMakeLists.txt @@ -1,5 +1,6 @@ oblib_add_library(restore OBJECT ob_i_storage.h + ob_i_storage.cpp ob_storage.cpp ob_storage_file.cpp ob_storage_file.h @@ -34,7 +35,28 @@ target_link_libraries(oss ${DEP_DIR}/lib/libapr-1.a ${DEP_DIR}/lib/libmxml.a) +oblib_add_library(s3 ob_storage_s3_base.cpp) +target_link_directories(s3 PUBLIC ${DEP_3RD_DIR}/usr/local/oceanbase/deps/devel/lib64) +target_link_libraries(s3 + PUBLIC + libaws-cpp-sdk-s3.a + libaws-cpp-sdk-core.a + libaws-crt-cpp.a + libaws-c-mqtt.a + libaws-c-event-stream.a + libaws-c-s3.a + libaws-c-auth.a + libaws-c-http.a + libaws-c-io.a + libs2n.a + libaws-c-compression.a + libaws-c-cal.a + libaws-c-sdkutils.a + libaws-checksums.a + libaws-c-common.a +) +target_link_libraries(s3 PUBLIC oblib_base_base_base) -target_link_libraries(restore PUBLIC oss cos_sdk oblib_base) +target_link_libraries(restore PUBLIC oss cos_sdk oblib_base s3) add_subdirectory(cos) diff --git a/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.cpp b/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.cpp index 666ef5d97..5b86ad2fe 100644 --- a/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.cpp +++ b/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.cpp @@ -49,6 +49,8 @@ const int COS_PERMISSION_DENIED = 403; const int COS_APPEND_POSITION_ERROR = 409; const int COS_SERVICE_UNAVAILABLE = 503; +const int64_t OB_STORAGE_LIST_MAX_NUM = 1000; + //datetime formate : Tue, 09 Apr 2019 06:24:00 GMT //time unit is second static int64_t strtotime(const char *date_time) @@ -58,7 +60,7 @@ static int64_t strtotime(const char *date_time) memset(&tm_time, 0, sizeof(struct tm)); if (NULL == strptime(date_time, "%a, %d %b %Y %H:%M:%S %Z", &tm_time)) { //skip set ret, for compat data formate - cos_warn_log("[COS]failed to transform time, time=%s\n", date_time); + cos_warn_log("[COS]fail to transform time, time=%s\n", date_time); } else { time = mktime(&tm_time); } @@ -84,6 +86,10 @@ static void convert_io_error(cos_status_t *cos_ret, int &ob_errcode) ob_errcode = OB_BACKUP_PWRITE_OFFSET_NOT_MATCH; break; } + case COS_SERVICE_UNAVAILABLE: { + ob_errcode = OB_IO_LIMIT; + break; + } default: { ob_errcode = OB_COS_ERROR; } @@ -141,31 +147,31 @@ int ObCosAccount::parse_from(const char *storage_info, uint32_t size) break; } else if (0 == strncmp(HOST, token, strlen(HOST))) { if (OB_SUCCESS != (ret = set_field(token + strlen(HOST), endpoint_, sizeof(endpoint_)))) { - cos_warn_log("[COS]failed to set endpoint=%s, ret=%d\n", token, ret); + cos_warn_log("[COS]fail to set endpoint=%s, ret=%d\n", token, ret); } else { bitmap |= 1; } } else if (0 == strncmp(ACCESS_ID, token, strlen(ACCESS_ID))) { if (OB_SUCCESS != (ret = set_field(token + strlen(ACCESS_ID), access_id_, sizeof(access_id_)))) { - cos_warn_log("[COS]failed to set access_id=%s, ret=%d\n", token, ret); + cos_warn_log("[COS]fail to set access_id=%s, ret=%d\n", token, ret); } else { bitmap |= (1 << 1); } } else if (0 == strncmp(ACCESS_KEY, token, strlen(ACCESS_KEY))) { if (OB_SUCCESS != (ret = set_field(token + strlen(ACCESS_KEY), access_key_, sizeof(access_key_)))) { - cos_warn_log("[COS]failed to set access_key=%s, ret=%d\n", token, ret); + cos_warn_log("[COS]fail to set access_key=%s, ret=%d\n", token, ret); } else { bitmap |= (1 << 2); } } else if (0 == strncmp(APPID, token, strlen(APPID))) { if (OB_SUCCESS != (ret = set_field(token + strlen(APPID), appid_, sizeof(appid_)))) { - cos_warn_log("[COS]failed to set appid=%s, ret=%d\n", token, ret); + cos_warn_log("[COS]fail to set appid=%s, ret=%d\n", token, ret); } else { bitmap |= (1 << 3); } } else if (0 == strncmp(DELETE_MODE, token, strlen(DELETE_MODE))) { if (OB_SUCCESS != (ret = set_field(token + strlen(DELETE_MODE), delete_mode_, sizeof(delete_mode_)))) { - cos_warn_log("[COS]failed to set delete_mode=%s, ret=%d", token, ret); + cos_warn_log("[COS]fail to set delete_mode=%s, ret=%d", token, ret); } } else { cos_warn_log("[COS]unkown token:%s\n", token); @@ -174,7 +180,7 @@ int ObCosAccount::parse_from(const char *storage_info, uint32_t size) if (OB_SUCCESS == ret && bitmap != 0x0F) { ret = OB_COS_ERROR; - cos_warn_log("[COS]failed to parse cos account storage_info=%s, bitmap=%x, ret=%d\n", storage_info, bitmap, ret); + cos_warn_log("[COS]fail to parse cos account storage_info=%s, bitmap=%x, ret=%d\n", storage_info, bitmap, ret); } } @@ -226,19 +232,37 @@ struct CosContext static void log_status(cos_status_t *s) { if (NULL != s) { - cos_warn_log("status->code: %d", s->code); + cos_warn_log("[COS]status->code: %d", s->code); if (s->error_code) { - cos_warn_log("status->error_code: %s", s->error_code); + cos_warn_log("[COS]status->error_code: %s", s->error_code); } if (s->error_msg) { - cos_warn_log("status->error_msg: %s", s->error_msg); + cos_warn_log("[COS]status->error_msg: %s", s->error_msg); } if (s->req_id) { - cos_warn_log("status->req_id: %s", s->req_id); + cos_warn_log("[COS]status->req_id: %s", s->req_id); } } } +int ObCosWrapper::CosListObjPara::set_cur_obj_meta( + char *obj_full_path, + const int64_t full_path_size, + char *object_size_str) +{ + int ret = OB_SUCCESS; + if (NULL == obj_full_path || full_path_size <= 0 || NULL == object_size_str) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]invalid object meta, obj_full_path=%s, full_path_size=%ld, object_size_str=%s, ret=%d\n", + obj_full_path, full_path_size, object_size_str, ret); + } else { + cur_obj_full_path_ = obj_full_path; + full_path_size_ = full_path_size; + cur_object_size_str_ = object_size_str; + } + return ret; +} + int ObCosWrapper::create_cos_handle( OB_COS_customMem &custom_mem, const struct ObCosAccount &account, @@ -349,10 +373,10 @@ int ObCosWrapper::put( cos_warn_log("[COS]fail to pack buf, ret=%d\n", ret); } else { cos_list_add_tail(&content->node, &buffer); - if(NULL == (cos_ret = cos_put_object_from_buffer(ctx->options, &bucket, &object, &buffer, NULL, &resp_headers)) + if (NULL == (cos_ret = cos_put_object_from_buffer(ctx->options, &bucket, &object, &buffer, NULL, &resp_headers)) || !cos_status_is_ok(cos_ret)) { convert_io_error(cos_ret, ret); - cos_warn_log("[COS]failed to put one object to cos, ret=%d\n", ret); + cos_warn_log("[COS]fail to put one object to cos, ret=%d\n", ret); log_status(cos_ret); } } @@ -410,7 +434,7 @@ int ObCosWrapper::append( if(NULL == (cos_ret = cos_append_object_from_buffer(ctx->options, &bucket, &object, offset, &buffer, NULL, &resp_headers)) || !cos_status_is_ok(cos_ret)) { convert_io_error(cos_ret, ret); - cos_warn_log("[COS]failed to append object from buffer to cos, ret=%d\n", ret); + cos_warn_log("[COS]fail to append object from buffer to cos, ret=%d\n", ret); log_status(cos_ret); } } @@ -464,7 +488,7 @@ int ObCosWrapper::head_object_meta( is_exist = false; } else { convert_io_error(cos_ret, ret); - cos_warn_log("[COS]failed to get file meta, ret=%d.\n", ret); + cos_warn_log("[COS]fail to get file meta, ret=%d.\n", ret); log_status(cos_ret); } } @@ -508,7 +532,7 @@ int ObCosWrapper::head_object_meta( meta.last_modified_ts_ = strtotime(last_modified_ptr); } else { ret = OB_COS_ERROR; - cos_warn_log("[COS]failed to get last modified from apr table, something wrong unexpected, ret=%d.\n", ret); + cos_warn_log("[COS]fail to get last modified from apr table, something wrong unexpected, ret=%d.\n", ret); } } } @@ -545,7 +569,7 @@ int ObCosWrapper::del( if (NULL == (cos_ret = cos_delete_object(ctx->options, &bucket, &object, &resp_headers)) || !cos_status_is_ok(cos_ret)) { convert_io_error(cos_ret, ret); - cos_warn_log("[COS]failed to delete object, ret=%d\n", ret); + cos_warn_log("[COS]fail to delete object, ret=%d\n", ret); log_status(cos_ret); } } @@ -595,7 +619,7 @@ int ObCosWrapper::tag( NULL, NULL, params, &resp_headers)) || !cos_status_is_ok(cos_ret)) { convert_io_error(cos_ret, ret); - cos_warn_log("[COS]failed to tag object, ret=%d\n", ret); + cos_warn_log("[COS]fail to tag object, ret=%d\n", ret); log_status(cos_ret); } } @@ -643,7 +667,7 @@ int ObCosWrapper::del_objects_in_dir( } else { cos_str_set(¶ms->prefix, dir_name.data_); cos_str_set(¶ms->marker, "/"); - params->max_ret = 1000; + params->max_ret = OB_STORAGE_LIST_MAX_NUM; cos_string_t bucket; cos_str_set(&bucket, bucket_name.data_); cos_list_object_content_t *content = NULL; @@ -657,15 +681,10 @@ int ObCosWrapper::del_objects_in_dir( cos_list_init(&deleted_object_list); do { - // List objects from cos, limit 1000. + // List objects from cos, limit OB_STORAGE_LIST_MAX_NUM. cos_ret = cos_list_object(ctx->options, &bucket, params, NULL); if (NULL == cos_ret || !cos_status_is_ok(cos_ret)) { - if (NULL != cos_ret && COS_SERVICE_UNAVAILABLE == cos_ret->code) { - // Request is limited by cos, returne "SlowDown, Reduce your request rate". - ret = OB_IO_LIMIT; - } else { - ret = OB_COS_ERROR; - } + convert_io_error(cos_ret, ret); cos_warn_log("[COS]fail to list object, ret=%d\n", ret); log_status(cos_ret); } else { @@ -690,7 +709,7 @@ int ObCosWrapper::del_objects_in_dir( } } - // Delete current batch of objects, limit 1000. + // Delete current batch of objects, limit OB_STORAGE_LIST_MAX_NUM. if (OB_SUCCESS == ret && !cos_list_empty(&to_delete_object_list)) { if (NULL == (cos_ret = cos_delete_objects(ctx->options, &bucket, &to_delete_object_list, COS_FALSE, &resp_headers, &deleted_object_list)) || !cos_status_is_ok(cos_ret)) { @@ -759,10 +778,10 @@ int ObCosWrapper::update_object_modified_ts( cos_copy_object_params_t *params = NULL; if (NULL == (headers = cos_table_make(ctx->mem_pool, 2))) { ret = OB_ALLOCATE_MEMORY_FAILED; - cos_warn_log("[COS]failed to allocate header memory, ret=%d\n", ret); + cos_warn_log("[COS]fail to allocate header memory, ret=%d\n", ret); } else if (NULL == (params = cos_create_copy_object_params(ctx->mem_pool))) { ret = OB_ALLOCATE_MEMORY_FAILED; - cos_warn_log("[COS]failed to allocate copy object param memory, ret=%d\n", ret); + cos_warn_log("[COS]fail to allocate copy object param memory, ret=%d\n", ret); } else { // In Cos, things are different. We cannot refresh the object's latest modified // time by updating ACL just like what we do for OSS. Instead, we use "PUT Object - Copy". @@ -1004,114 +1023,247 @@ int ObCosWrapper::is_object_tagging( return ret; } +struct CosListArguments +{ + CosListArguments( + CosContext *ctx, + const CosStringBuffer &bucket_name, + const CosStringBuffer &full_dir_path, + const char *next_marker, + const char *delimiter, + const int64_t max_ret) + : ctx_(ctx), bucket_name_(bucket_name), full_dir_path_(full_dir_path), + next_marker_(next_marker), delimiter_(delimiter), max_ret_(max_ret) + { + } + + int check_validity() const + { + int ret = OB_SUCCESS; + if (nullptr == ctx_) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]cos context is null, ret=%d\n", ret); + } else if (bucket_name_.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]bucket name is empty, ret=%d\n", ret); + } else if (!full_dir_path_.is_end_with_slash_and_null()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]full_dir_path format not right, dir=%s, dir_len=%ld, ret=%d\n", + full_dir_path_.data_, full_dir_path_.size_, ret); + } else if (max_ret_ <= 0 || max_ret_ > OB_STORAGE_LIST_MAX_NUM) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]max_ret invalid, max_ret=%ld, ret=%d\n", max_ret_, ret); + } + // 'next_marker' and 'delimiter' are not required parameters + // for the 'cos_list_object' interface, so they can be NULL. + return ret; + } + + CosContext *ctx_; + const CosStringBuffer &bucket_name_; + const CosStringBuffer &full_dir_path_; + const char *next_marker_; + const char *delimiter_; + const int64_t max_ret_; +}; + +static int do_list_( + const CosListArguments &cos_list_args, + cos_list_object_params_t *¶ms, + cos_table_t **resp_headers) +{ + int ret = OB_SUCCESS; + cos_string_t bucket; + cos_status_t *cos_ret = nullptr; + + if (OB_SUCCESS != (ret = cos_list_args.check_validity())) { + cos_warn_log("[COS]cos_list_args is invalid, ret=%d\n", ret); + } else { + if (nullptr == params) { + if (nullptr == (params = cos_create_list_object_params(cos_list_args.ctx_->mem_pool))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + cos_warn_log("[COS]fail to create list object params, ret=%d\n", ret); + } + } else { + // reuse params + cos_list_init(¶ms->object_list); + cos_list_init(¶ms->common_prefix_list); + } + + + if (OB_SUCCESS == ret) { + params->max_ret = cos_list_args.max_ret_; + cos_str_set(&bucket, cos_list_args.bucket_name_.data_); + cos_str_set(¶ms->prefix, cos_list_args.full_dir_path_.data_); + if (nullptr != cos_list_args.next_marker_ && strlen(cos_list_args.next_marker_) > 0) { + cos_str_set(¶ms->marker, cos_list_args.next_marker_); + } + if (nullptr != cos_list_args.delimiter_ && strlen(cos_list_args.delimiter_) > 0) { + cos_str_set(¶ms->delimiter, cos_list_args.delimiter_); + } + } + + if (OB_SUCCESS == ret) { + if (nullptr == (cos_ret = cos_list_object(cos_list_args.ctx_->options, &bucket, params, resp_headers)) + || !cos_status_is_ok(cos_ret)) { + convert_io_error(cos_ret, ret); + cos_warn_log("[COS]fail to list object, ret=%d\n", ret); + log_status(cos_ret); + } + } + } + + return ret; +} + int ObCosWrapper::list_objects( Handle *h, const CosStringBuffer &bucket_name, - const CosStringBuffer &dir_name, + const CosStringBuffer &full_dir_path, + handleObjectNameFunc handle_object_name_f, + void *arg) +{ + int ret = OB_SUCCESS; + CosContext *ctx = reinterpret_cast(h); + CosListArguments cos_list_args(ctx, bucket_name, full_dir_path, + NULL/*next_marker*/, NULL/*delimiter*/, OB_STORAGE_LIST_MAX_NUM); + + if (OB_SUCCESS != (ret = cos_list_args.check_validity())) { + cos_warn_log("[COS]cos_list_args is invalid, ret=%d\n", ret); + } else if (NULL == handle_object_name_f) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]handle_object_name_f is null, ret=%d\n", ret); + } else { + cos_list_object_params_t *params = NULL; + cos_list_object_content_t *content = NULL; + cos_table_t *resp_headers = NULL; + + CosListObjPara para; + para.arg_ = arg; + para.type_ = CosListObjPara::CosListType::COS_LIST_CB_ARG; + + do { + if (OB_SUCCESS != (ret = do_list_(cos_list_args, params, &resp_headers))) { + cos_warn_log("[COS]fail to do list, bucket=%s, path=%s, next_marker=%s, ret=%d\n", + bucket_name.data_, full_dir_path.data_, cos_list_args.next_marker_, ret); + } else { + char *request_id = (char*)apr_table_get(resp_headers, "x-cos-request-id"); + // Traverse the returned objects + cos_list_for_each_entry(cos_list_object_content_t, content, ¶ms->object_list, node) { + // check if the prefix of returned object key match the full_dir_path + size_t full_dir_path_len = strlen(full_dir_path.data_); + if (false == full_dir_path.is_prefix_of(content->key.data, content->key.len)) { + ret = OB_COS_ERROR; + cos_warn_log("[COS]returned object prefix not match, dir=%s, object=%s, requestid=%s, ret=%d\n", + full_dir_path.data_, content->key.data, request_id, ret); + } else if (content->key.len == full_dir_path_len) { + // skip + cos_info_log("[COS]exist object path length is same with dir path length, dir=%s, object=%s, requestid=%s, ret=%d\n", + full_dir_path.data_, content->key.data, request_id, ret); + } else if (OB_SUCCESS != (ret = para.set_cur_obj_meta(content->key.data, + content->key.len, + content->size.data))) { + cos_warn_log("[COS]fail to set cur obj meta, ret=%d, obj_full_path=%s, full_path_size=%ld, obj_size_str=%s, requestid=%s\n", + ret, content->key.data, content->key.len, content->size.data, request_id); + } else if (OB_SUCCESS != (ret = handle_object_name_f(para))) { + cos_warn_log("[COS]handle object name failed, ret=%d, object=%s, requestid=%s\n", + ret, content->key.data, request_id); + } + + if (OB_SUCCESS != ret) { + break; + } + } // end cos_list_for_each_entry + + if (OB_SUCCESS == ret && COS_TRUE == params->truncated) { + if (NULL == (cos_list_args.next_marker_ = apr_psprintf(ctx->mem_pool, "%.*s", + params->next_marker.len, + params->next_marker.data))) { + ret = OB_COS_ERROR; + cos_warn_log("[COS]get next marker is null, ret=%d\n", ret); + } + } + } + } while (COS_TRUE == params->truncated && OB_SUCCESS == ret); + } + + return ret; +} + +int ObCosWrapper::list_part_objects( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &full_dir_path, const CosStringBuffer &next_marker, handleObjectNameFunc handle_object_name_f, void *arg) { int ret = OB_SUCCESS; - - const char *seperator = "/"; - // dir_name must be end with '/\0' - const int32_t min_dir_name_str_len = 2; - CosContext *ctx = reinterpret_cast(h); + CosListArguments cos_list_args(ctx, bucket_name, full_dir_path, + next_marker.data_, NULL/*delimiter*/, OB_STORAGE_LIST_MAX_NUM); - if (NULL == h) { - ret = OB_INVALID_ARGUMENT; - cos_warn_log("[COS]cos handle is null, ret=%d\n", ret); - } else if (bucket_name.empty()) { - ret = OB_INVALID_ARGUMENT; - cos_warn_log("[COS]bucket name is null, ret=%d\n", ret); - } else if (NULL == dir_name.data_ || min_dir_name_str_len >= dir_name.size_) { - ret = OB_INVALID_ARGUMENT; - cos_warn_log("[COS]dir_name size too short, size=%d, ret=%d\n", dir_name.size_, ret); - } else if (dir_name.data_[dir_name.size_ - 2] != *seperator) { - ret = OB_INVALID_ARGUMENT; - cos_warn_log("[COS]dir_name format not right, dir=%s, ret=%d\n", dir_name.data_, ret); - } else if (next_marker.empty()) { - // next_marker end with '\0', its size must be > 0 - ret = OB_INVALID_ARGUMENT; - cos_warn_log("[COS]next_marker is null, ret=%d\n", ret); + if (OB_SUCCESS != (ret = cos_list_args.check_validity())) { + cos_warn_log("[COS]cos_list_args is invalid, ret=%d\n", ret); } else if (NULL == handle_object_name_f) { ret = OB_INVALID_ARGUMENT; cos_warn_log("[COS]handle_object_name_f is null, ret=%d\n", ret); } else { - cos_status_t *cos_ret = NULL; cos_list_object_params_t *params = NULL; + cos_list_object_content_t *content = NULL; + cos_table_t *resp_headers = NULL; - if (NULL == (params = cos_create_list_object_params(ctx->mem_pool))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - cos_warn_log("[COS]fail to create list object params, ret=%d\n", ret); + CosListObjPara para; + para.arg_ = arg; + para.type_ = CosListObjPara::CosListType::COS_PART_LIST_CTX; + + if (OB_SUCCESS != (ret = do_list_(cos_list_args, params, &resp_headers))) { + cos_warn_log("[COS]fail to do list, bucket=%s, path=%s, next_marker=%s, ret=%d\n", + bucket_name.data_, full_dir_path.data_, cos_list_args.next_marker_, ret); } else { - cos_str_set(¶ms->prefix, dir_name.data_); - cos_str_set(¶ms->marker, next_marker.data_); - params->max_ret = 1000; - cos_string_t bucket; - cos_str_set(&bucket, bucket_name.data_); - cos_list_object_content_t *content = NULL; - - CosListObjPara para; - para.arg_ = arg; - para.type_ = CosListObjPara::CosListType::COS_LIST_CB_ARG; - - do { - cos_table_t *resp_headers = NULL; - // List objects from cos - if (NULL == (cos_ret = cos_list_object(ctx->options, &bucket, params, &resp_headers)) - || !cos_status_is_ok(cos_ret)) { - if (NULL != cos_ret && COS_SERVICE_UNAVAILABLE == cos_ret->code) { - // Request is limited by cos, returne "SlowDown, Reduce your request rate". - ret = OB_IO_LIMIT; - } else { - convert_io_error(cos_ret, ret); - } - cos_warn_log("[COS]fail to list object, ret=%d\n", ret); - log_status(cos_ret); - } else { - char *request_id = (char*)apr_table_get(resp_headers, "x-cos-request-id"); - // Traverse the returned objects - cos_list_for_each_entry(cos_list_object_content_t, content, ¶ms->object_list, node) { - // check if the prefix of returned object key match the dir_name - size_t dir_name_str_len = strlen(dir_name.data_); - if (0 != memcmp(content->key.data, dir_name.data_, dir_name_str_len)) { - ret = OB_COS_ERROR; - cos_warn_log("[COS]returned object prefix not match, dir=%s, object=%s, requestid=%s, ret=%d\n", dir_name.data_, content->key.data, request_id, ret); - } else if (NULL == content->size.data || 0 == content->size.len) { - ret = OB_COS_ERROR; - cos_warn_log("[COS]returned object size is empty, dir=%s, object=%s, requestid=%s, ret=%d\n", dir_name.data_, content->key.data, request_id, ret); - } else { - // Callback to handle the object name, it is a absolute path. - para.cur_full_path_slice_name_ = content->key.data; - para.full_path_size_ = content->key.len; - para.cur_object_size_ = cos_atoi64(content->size.data); - ret = handle_object_name_f(para); - if (OB_SUCCESS != ret){ - cos_warn_log("[COS]handle object name failed, ret=%d, object=%s, requestid=%s\n", ret, content->key.data, request_id); - } - } - - if (OB_SUCCESS != ret || !para.next_flag_) { - break; - } - } - - char *next_marker_str = NULL; - if (OB_SUCCESS == ret && para.next_flag_ && COS_TRUE == params->truncated) { - if (NULL == (next_marker_str = apr_psprintf(ctx->mem_pool, "%.*s", params->next_marker.len, params->next_marker.data))) { - ret = OB_COS_ERROR; - cos_warn_log("[COS]get next marker is null, ret=%d\n", ret); - } else { - cos_str_set(¶ms->marker, next_marker_str); - cos_list_init(¶ms->object_list); - cos_list_init(¶ms->common_prefix_list); - } - } + char *request_id = (char*)apr_table_get(resp_headers, "x-cos-request-id"); + // Traverse the returned objects + cos_list_for_each_entry(cos_list_object_content_t, content, ¶ms->object_list, node) { + // check if the prefix of returned object key match the full_dir_path + size_t full_dir_path_len = strlen(full_dir_path.data_); + if (false == full_dir_path.is_prefix_of(content->key.data, content->key.len)) { + ret = OB_COS_ERROR; + cos_warn_log("[COS]returned object prefix not match, dir=%s, object=%s, requestid=%s, ret=%d\n", + full_dir_path.data_, content->key.data, request_id, ret); + } else if (content->key.len == full_dir_path_len) { + // skip + cos_info_log("[COS]exist object path length is same with dir path length, dir=%s, object=%s, requestid=%s, ret=%d\n", + full_dir_path.data_, content->key.data, request_id, ret); + } else if (OB_SUCCESS != (ret = para.set_cur_obj_meta(content->key.data, + content->key.len, + content->size.data))) { + cos_warn_log("[COS]fail to set cur obj meta, ret=%d, obj_full_path=%s, full_path_size=%ld, obj_size_str=%s, requestid=%s\n", + ret, content->key.data, content->key.len, content->size.data, request_id); + } else if (OB_SUCCESS != (ret = handle_object_name_f(para))) { + cos_warn_log("[COS]handle object name failed, ret=%d, object=%s, requestid=%s\n", + ret, content->key.data, request_id); } - } while (COS_TRUE == params->truncated && OB_SUCCESS == ret && para.next_flag_); + + if (OB_SUCCESS != ret) { + break; + } + } // end cos_list_for_each_entry + + if (OB_SUCCESS == ret) { + para.finish_part_list_ = true; + para.next_flag_ = (COS_TRUE == params->truncated); + if (para.next_flag_) { + para.next_token_ = params->next_marker.data; + para.next_token_size_ = params->next_marker.len; + } + + // Here, we invoke handle_object_name_f, just for flaging 'has_next' and saving 'next_token' + // It won't handle the last object twice. + ret = handle_object_name_f(para); + if (OB_SUCCESS != ret){ + cos_warn_log("[COS]handle object name failed, ret=%d\n", ret); + } + } } } @@ -1128,106 +1280,79 @@ int ObCosWrapper::list_directories( void *arg) { int ret = OB_SUCCESS; - - const char *seperator = "/"; - // dir_name must be end with '/\0' - const int32_t min_dir_name_str_len = 2; - + const char seperator = '/'; CosContext *ctx = reinterpret_cast(h); + CosListArguments cos_list_args(ctx, bucket_name, dir_name, + next_marker.data_, delimiter.data_, OB_STORAGE_LIST_MAX_NUM); - if (NULL == h) { + if (OB_SUCCESS != (ret = cos_list_args.check_validity())) { + cos_warn_log("[COS]cos_list_args is invalid, ret=%d\n", ret); + } else if (delimiter.empty() || delimiter.data_[0] != seperator) { ret = OB_INVALID_ARGUMENT; - cos_warn_log("[COS]cos handle is null, ret=%d\n", ret); - } else if (bucket_name.empty()) { - ret = OB_INVALID_ARGUMENT; - cos_warn_log("[COS]bucket name is null, ret=%d\n", ret); - } else if (NULL == dir_name.data_ || min_dir_name_str_len >= dir_name.size_) { - ret = OB_INVALID_ARGUMENT; - cos_warn_log("[COS]dir_name size too short, size=%d, ret=%d\n", dir_name.size_, ret); - } else if (dir_name.data_[dir_name.size_ - 2] != *seperator) { - ret = OB_INVALID_ARGUMENT; - cos_warn_log("[COS]dir_name format not right, dir=%s, ret=%d\n", dir_name.data_, ret); - } else if (next_marker.empty()) { - // next_marker end with '\0', its size must be > 0 - ret = OB_INVALID_ARGUMENT; - cos_warn_log("[COS]next_marker is null, ret=%d\n", ret); + cos_warn_log("[COS]delimiter is invalid, delimiter=%s, ret=%d\n", delimiter.data_); } else if (NULL == handle_directory_name_f) { ret = OB_INVALID_ARGUMENT; cos_warn_log("[COS]handle_directory_name_f is null, ret=%d\n", ret); } else { - cos_status_t *cos_ret = NULL; cos_list_object_params_t *params = NULL; + cos_list_object_common_prefix_t *common_prefix = NULL; + cos_table_t *resp_headers = NULL; - if (NULL == (params = cos_create_list_object_params(ctx->mem_pool))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - cos_warn_log("[COS]fail to create list object params, ret=%d\n", ret); - } else { - cos_str_set(¶ms->prefix, dir_name.data_); - cos_str_set(¶ms->marker, next_marker.data_); - cos_str_set(¶ms->delimiter, delimiter.data_); - params->max_ret = 1000; - cos_string_t bucket; - cos_str_set(&bucket, bucket_name.data_); - cos_list_object_common_prefix_t *common_prefix = NULL; - - // A mark which indicates whether to process the next object. - do { - cos_table_t *resp_headers = NULL; - // List objects from cos - if (NULL == (cos_ret = cos_list_object(ctx->options, &bucket, params, &resp_headers)) - || !cos_status_is_ok(cos_ret)) { - if (NULL != cos_ret && COS_SERVICE_UNAVAILABLE == cos_ret->code) { - // Request is limited by cos, returne "SlowDown, Reduce your request rate". - ret = OB_IO_LIMIT; - } else { + do { + if (OB_SUCCESS != (ret = do_list_(cos_list_args, params, &resp_headers))) { + cos_warn_log("[COS]fail to do list, bucket=%s, path=%s, next_marker=%s, ret=%d\n", + bucket_name.data_, dir_name.data_, cos_list_args.next_marker_, ret); + } else { + char *request_id = (char*)apr_table_get(resp_headers, "x-cos-request-id"); + // Traverse the returned objects + cos_list_for_each_entry(cos_list_object_common_prefix_t, common_prefix, ¶ms->common_prefix_list, node) { + // For example, + // dir1 + // --file1 + // --dir11 + // --file11 + // if we list directories in 'dir1', then full_dir_path == 'dir1/' + // and listed_dir_full_path == 'dir1/dir11/', which represents the full directory path of 'dir11' + const char *listed_dir_full_path = common_prefix->prefix.data; + const int64_t listed_dir_full_path_len = common_prefix->prefix.len; + // check if the prefix of returned object key match the dir_name + const size_t dir_name_str_len = strlen(dir_name.data_); + if (false == dir_name.is_prefix_of(listed_dir_full_path, listed_dir_full_path_len)) { ret = OB_COS_ERROR; - } - cos_warn_log("[COS]fail to list object, ret=%d\n", ret); - log_status(cos_ret); - } else { - char *request_id = (char*)apr_table_get(resp_headers, "x-cos-request-id"); - // Traverse the returned objects - cos_list_for_each_entry(cos_list_object_common_prefix_t, common_prefix, ¶ms->common_prefix_list, node) { - // check if the prefix of returned object key match the dir_name - const size_t dir_name_str_len = strlen(dir_name.data_); - const size_t prefix_str_len = strlen(common_prefix->prefix.data); - if (prefix_str_len < dir_name_str_len) { - ret = OB_COS_ERROR; - cos_warn_log("[COS]prefix str len should not smaller than dir name str len. prefix str len : %lu, " - "dir name str len : %lu, ret = %d \n", prefix_str_len, dir_name_str_len, ret); - } else if (0 != memcmp(common_prefix->prefix.data, dir_name.data_, dir_name_str_len)) { - ret = OB_COS_ERROR; - cos_warn_log("[COS]returned object prefix not match, dir=%s, object=%s, requestid=%s, ret=%d\n", dir_name.data_, common_prefix->prefix.data, request_id, ret); - } else { - // Callback to handle the object name, it is a absolute path. - const int64_t object_size = common_prefix->prefix.len - dir_name_str_len; //include '/' - CosListObjPara::CosListType type = CosListObjPara::CosListType::COS_LIST_CB_ARG; - ret = handle_directory_name_f(arg, type, common_prefix->prefix.data + dir_name_str_len, object_size); - if (OB_SUCCESS != ret) { - // Something wrong happened when handle object name - cos_warn_log("[COS]handle object name failed, ret=%d, object=%s, requestid=%s\n", ret, common_prefix->prefix.data, request_id); - } - } - - if (OB_SUCCESS != ret) { - break; + cos_warn_log("[COS]returned object prefix not match, dir=%s, object=%s, requestid=%s, ret=%d, obj_path_len=%d, dir_name_len=%d\n", + dir_name.data_, listed_dir_full_path, request_id, ret, listed_dir_full_path_len, dir_name.size_); + } else if (seperator != listed_dir_full_path[listed_dir_full_path_len - 1]) { + ret = OB_COS_ERROR; + cos_warn_log("[COS]the data has no directory, dir=%s, object=%s, requestid=%s, ret=%d obj_len=%d\n", + dir_name.data_, listed_dir_full_path, request_id, ret, listed_dir_full_path_len); + } else { + // Callback to handle the object name, it is a absolute path. + // remove trailing '/' + const int64_t listed_dir_name_len = listed_dir_full_path_len - 1 - dir_name_str_len; + CosListObjPara::CosListType type = CosListObjPara::CosListType::COS_LIST_CB_ARG; + if (OB_SUCCESS != (ret = handle_directory_name_f(arg, type, + listed_dir_full_path + dir_name_str_len, + listed_dir_name_len))) { + // Something wrong happened when handle object name + cos_warn_log("[COS]handle object name failed, ret=%d, object=%s, requestid=%s\n", ret, listed_dir_full_path, request_id); } } - char *next_marker_str = NULL; - if (OB_SUCCESS == ret && COS_TRUE == params->truncated) { - if (NULL == (next_marker_str = apr_psprintf(ctx->mem_pool, "%.*s", params->next_marker.len, params->next_marker.data))) { + if (OB_SUCCESS != ret) { + break; + } + } // end cos_list_for_each_entry + + if (OB_SUCCESS == ret && COS_TRUE == params->truncated) { + if (NULL == (cos_list_args.next_marker_ = apr_psprintf(ctx->mem_pool, "%.*s", + params->next_marker.len, + params->next_marker.data))) { ret = OB_COS_ERROR; cos_warn_log("[COS]get next marker is null, ret=%d\n", ret); - } else { - cos_str_set(¶ms->marker, next_marker_str); - cos_list_init(¶ms->object_list); - cos_list_init(¶ms->common_prefix_list); - } } } - } while (COS_TRUE == params->truncated && OB_SUCCESS == ret); - } + } + } while (COS_TRUE == params->truncated && OB_SUCCESS == ret); } return ret; @@ -1241,10 +1366,36 @@ int ObCosWrapper::is_empty_directory( { int ret = OB_SUCCESS; is_empty_dir = false; - const char *seperator = "/"; - // dir_name must be end with '/\0' - const int32_t min_dir_name_str_len = 2; + CosContext *ctx = reinterpret_cast(h); + // it just decides if it is not empty + const int64_t max_ret = 1; + CosListArguments cos_list_args(ctx, bucket_name, dir_name, + NULL/*next_marker*/, NULL/*delimiter*/, max_ret); + if (OB_SUCCESS != (ret = cos_list_args.check_validity())) { + cos_warn_log("[COS]cos_list_args is invalid, ret=%d\n", ret); + } else { + cos_list_object_params_t *params = NULL; + cos_table_t *resp_headers = NULL; + + if (OB_SUCCESS != (ret = do_list_(cos_list_args, params, &resp_headers))) { + cos_warn_log("[COS]fail to do list, bucket=%s, path=%s, next_marker=%s, ret=%d\n", + bucket_name.data_, dir_name.data_, cos_list_args.next_marker_, ret); + } else { + is_empty_dir = static_cast(cos_list_empty(¶ms->object_list)); + } + } + + return ret; +} + +int ObCosWrapper::init_multipart_upload( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name, + char *&upload_id_str) +{ + int ret = OB_SUCCESS; CosContext *ctx = reinterpret_cast(h); if (NULL == h) { @@ -1253,43 +1404,315 @@ int ObCosWrapper::is_empty_directory( } else if (bucket_name.empty()) { ret = OB_INVALID_ARGUMENT; cos_warn_log("[COS]bucket name is null, ret=%d\n", ret); - } else if (NULL == dir_name.data_ || min_dir_name_str_len >= dir_name.size_) { + } else if (object_name.empty()) { ret = OB_INVALID_ARGUMENT; - cos_warn_log("[COS]dir_name size too short, size=%d, ret=%d\n", dir_name.size_, ret); - } else if (dir_name.data_[dir_name.size_ - 2] != *seperator) { - ret = OB_INVALID_ARGUMENT; - cos_warn_log("[COS]dir_name format not right, dir=%s, ret=%d\n", dir_name.data_, ret); + cos_warn_log("[COS]object name is null, ret=%d\n", ret); } else { - cos_status_t *cos_ret = NULL; - cos_list_object_params_t *params = NULL; + cos_string_t bucket; + cos_string_t object; + cos_str_set(&bucket, bucket_name.data_); + cos_str_set(&object, object_name.data_); + cos_string_t upload_id; - if (NULL == (params = cos_create_list_object_params(ctx->mem_pool))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - cos_warn_log("[COS]fail to create list object params, ret=%d\n", ret); + cos_status_t *cos_ret = NULL; + cos_table_t *resp_headers = NULL; + + if (NULL == (cos_ret = cos_init_multipart_upload(ctx->options, &bucket, &object, &upload_id, NULL, &resp_headers)) || + !cos_status_is_ok(cos_ret) || upload_id.len < 1) { + convert_io_error(cos_ret, ret); + cos_warn_log("[COS]fail to init multipart upload, ret=%d, upload_id_length=%d\n", ret, upload_id.len); + log_status(cos_ret); } else { - cos_str_set(¶ms->prefix, dir_name.data_); - // it just decides if it is not empty - params->max_ret = 1; - cos_string_t bucket; - cos_str_set(&bucket, bucket_name.data_); - cos_table_t *resp_headers = NULL; - // List objects from cos - if (NULL == (cos_ret = cos_list_object(ctx->options, &bucket, params, &resp_headers)) - || !cos_status_is_ok(cos_ret)) { - if (NULL != cos_ret && COS_SERVICE_UNAVAILABLE == cos_ret->code) { - // Request is limited by cos, returne "SlowDown, Reduce your request rate". - ret = OB_IO_LIMIT; - } else { - ret = OB_COS_ERROR; - } - cos_warn_log("[COS]fail to list object, ret=%d\n", dir_name.data_, ret); - log_status(cos_ret); + const int64_t upload_id_len = upload_id.len + 1; + upload_id_str = static_cast(ctx->custom_mem.customAlloc(ctx->custom_mem.opaque, upload_id_len)); + if (NULL == upload_id_str) { + ret = OB_ALLOCATE_MEMORY_FAILED; + cos_warn_log("[COS]fail to alloc upload_id mem, ret=%d, len=%d\n", ret, upload_id_len); } else { - is_empty_dir = static_cast(cos_list_empty(¶ms->object_list)); + memcpy(upload_id_str, upload_id.data, upload_id.len); + upload_id_str[upload_id.len] = '\0'; } } } + return ret; +} +int ObCosWrapper::upload_part_from_buffer( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name, + const CosStringBuffer &upload_id_str, + const int part_num, + const char *buf, + const int64_t buf_size) +{ + int ret = OB_SUCCESS; + CosContext *ctx = reinterpret_cast(h); + + if (NULL == h) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]cos handle is null, ret=%d\n", ret); + } else if (bucket_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]bucket name is null, ret=%d\n", ret); + } else if (object_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]object name is null, ret=%d\n", ret); + } else if (upload_id_str.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]upload_id is null, ret=%d\n", ret); + } else if (NULL == buf || buf_size < 1 || part_num < 1 || part_num > 10000) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]invalid buf/buf_size/part_num, ret=%d, buf_size=%d, part_num=%d\n", + ret, buf_size, part_num); + } else { + cos_string_t bucket; + cos_string_t object; + cos_string_t upload_id; + cos_str_set(&bucket, bucket_name.data_); + cos_str_set(&object, object_name.data_); + cos_str_set(&upload_id, upload_id_str.data_); + cos_status_t *cos_ret = NULL; + cos_table_t *resp_headers = NULL; + cos_list_t buffer; + cos_buf_t *content = NULL; + cos_list_init(&buffer); + + if (NULL == (content = cos_buf_pack(ctx->mem_pool, buf, static_cast(buf_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + cos_warn_log("[COS]fail to pack buf, ret=%d, buf_size=%d\n", ret, buf_size); + } else { + cos_list_add_tail(&content->node, &buffer); + if (NULL == (cos_ret = cos_upload_part_from_buffer(ctx->options, &bucket, &object, &upload_id, part_num, &buffer, &resp_headers)) + || !cos_status_is_ok(cos_ret)) { + convert_io_error(cos_ret, ret); + cos_warn_log("[COS]fail to upload part to cos, ret=%d, part_num=%d\n", ret, part_num); + log_status(cos_ret); + } + } + } + return ret; +} + +int ObCosWrapper::complete_multipart_upload( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name, + const CosStringBuffer &upload_id_str) +{ + int ret = OB_SUCCESS; + CosContext *ctx = reinterpret_cast(h); + + if (NULL == h) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]cos handle is null, ret=%d\n", ret); + } else if (bucket_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]bucket name is null, ret=%d\n", ret); + } else if (object_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]object name is null, ret=%d\n", ret); + } else if (upload_id_str.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]upload_id is null, ret=%d\n", ret); + } else { + cos_string_t bucket; + cos_string_t object; + cos_string_t upload_id; + cos_str_set(&bucket, bucket_name.data_); + cos_str_set(&object, object_name.data_); + cos_str_set(&upload_id, upload_id_str.data_); + cos_status_t *cos_ret = NULL; + + cos_table_t *resp_headers = NULL; + cos_list_t complete_part_list; + cos_list_init(&complete_part_list); + cos_list_upload_part_params_t *params = cos_create_list_upload_part_params(ctx->mem_pool); + + if (NULL != params) { + // get all upload-completed part + params->max_ret = OB_STORAGE_LIST_MAX_NUM; + cos_list_part_content_t *part_content = NULL; + cos_complete_part_content_t *complete_part = NULL; + do { + if (NULL == (cos_ret = cos_list_upload_part(ctx->options, &bucket, &object, &upload_id, params, &resp_headers)) + || !cos_status_is_ok(cos_ret)) { + convert_io_error(cos_ret, ret); + cos_warn_log("[COS]fail to list upload part, ret=%d\n", ret); + log_status(cos_ret); + } else { + cos_list_for_each_entry(cos_list_part_content_t, part_content, ¶ms->part_list, node) { + complete_part = cos_create_complete_part_content(ctx->mem_pool); + if (NULL == complete_part) { + ret = OB_ALLOCATE_MEMORY_FAILED; + cos_warn_log("[COS]fail to create complete part content, ret=%d\n", ret); + } else { + cos_str_set(&complete_part->part_number, part_content->part_number.data); + cos_str_set(&complete_part->etag, part_content->etag.data); + cos_list_add_tail(&complete_part->node, &complete_part_list); + } + } + + if (OB_SUCCESS == ret && COS_TRUE == params->truncated) { + const char *next_part_number_marker = NULL; + if (NULL == (next_part_number_marker = + apr_psprintf(ctx->mem_pool, "%.*s", + params->next_part_number_marker.len, + params->next_part_number_marker.data))) { + ret = OB_COS_ERROR; + cos_warn_log("[COS]next part number marker is NULL, ret=%d\n", ret); + } else { + cos_str_set(¶ms->part_number_marker, next_part_number_marker); + cos_list_init(¶ms->part_list); + } + } + } + } while (OB_SUCCESS == ret && COS_TRUE == params->truncated); + + if (OB_SUCCESS == ret) { + if (NULL == (cos_ret = cos_complete_multipart_upload(ctx->options, &bucket, &object, &upload_id, &complete_part_list, NULL, &resp_headers)) + || !cos_status_is_ok(cos_ret)) { + convert_io_error(cos_ret, ret); + cos_warn_log("[COS]fail to complete multipart upload, ret=%d\n", ret); + log_status(cos_ret); + } + } + + } else { + ret = OB_ALLOCATE_MEMORY_FAILED; + cos_warn_log("[COS]fail to create list upload part params, ret=%d\n", ret); + } + } + return ret; +} + +int ObCosWrapper::abort_multipart_upload( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name, + const CosStringBuffer &upload_id_str) +{ + int ret = OB_SUCCESS; + CosContext *ctx = reinterpret_cast(h); + + if (NULL == h) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]cos handle is null, ret=%d\n", ret); + } else if (bucket_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]bucket name is null, ret=%d\n", ret); + } else if (object_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]object name is null, ret=%d\n", ret); + } else if (upload_id_str.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]upload_id is null, ret=%d\n", ret); + } else { + cos_string_t bucket; + cos_string_t object; + cos_string_t upload_id; + cos_str_set(&bucket, bucket_name.data_); + cos_str_set(&object, object_name.data_); + cos_str_set(&upload_id, upload_id_str.data_); + cos_status_t *cos_ret = NULL; + + cos_table_t *resp_headers = NULL; + if (NULL == (cos_ret = cos_abort_multipart_upload(ctx->options, &bucket, &object, &upload_id, &resp_headers)) + || !cos_status_is_ok(cos_ret)) { + convert_io_error(cos_ret, ret); + cos_warn_log("[COS]fail to abort multipart upload, ret=%d, bucket=%s, object=%s, upload_id=%s\n", + ret, bucket_name.data_, object_name.data_, upload_id_str.data_); + log_status(cos_ret); + } + } + return ret; +} + +int ObCosWrapper::del_unmerged_parts( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name) +{ + int ret = OB_SUCCESS; + CosContext *ctx = reinterpret_cast(h); + cos_list_multipart_upload_params_t *params = NULL; + const char *next_key_marker = ""; + const char *next_upload_id_marker = ""; + + if (NULL == h) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]cos handle is null, ret=%d\n", ret); + } else if (bucket_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]bucket name is null, ret=%d\n", ret); + } else if (object_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]object name is null, ret=%d\n", ret); + } else if (NULL == (params = cos_create_list_multipart_upload_params(ctx->mem_pool))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + cos_warn_log("[COS]fail to create multipart upload params, ret=%d, bucket=%s\n", + ret, bucket_name.data_); + } else { + cos_str_set(¶ms->prefix, object_name.data_); + cos_str_set(¶ms->next_key_marker, next_key_marker); + cos_str_set(¶ms->next_upload_id_marker, next_upload_id_marker); + params->max_ret = OB_STORAGE_LIST_MAX_NUM; + cos_string_t bucket; + cos_string_t object; + cos_str_set(&bucket, bucket_name.data_); + cos_list_multipart_upload_content_t *content = NULL; + cos_table_t *resp_headers = NULL; + cos_status_t *cos_ret = NULL; + + do { + // There is a bug in the 'cos_list_multipart_upload' + // when parsing the response data to retrieve the upload id of multipart uploads + // in the 'cos_xml.c:cos_list_multipart_uploads_content_parse' function, + // the identifier being parsed should be 'UploadId' instead of 'UploadID'. + // This issue results in the inability to correctly parse the upload id for multipart upload events. + // This bug will be fixed in the next version of the cos SDK, + // therefore, the del_unmerged_parts feature is currently not supported. + if (NULL == (cos_ret = cos_list_multipart_upload(ctx->options, &bucket, params, &resp_headers)) + || !cos_status_is_ok(cos_ret)) { + convert_io_error(cos_ret, ret); + cos_warn_log("[COS]fail to list multipart uploads, ret=%d\n", ret); + log_status(cos_ret); + } else { + cos_list_for_each_entry(cos_list_multipart_upload_content_t, content, ¶ms->upload_list, node) { + if (NULL == (cos_ret = cos_abort_multipart_upload(ctx->options, &bucket, &(content->key), + &(content->upload_id), &resp_headers)) + || !cos_status_is_ok(cos_ret)) { + convert_io_error(cos_ret, ret); + cos_warn_log("[COS]fail to abort multipart upload, ret=%d, bucket=%s, object=%s, upload_id=%s\n", + ret, bucket_name.data_, content->key.data, content->upload_id.data); + log_status(cos_ret); + break; + } else { + cos_info_log("[COS]succeed to abort multipart upload, bucket=%s, object=%s, upload_id=%s\n", + bucket_name.data_, content->key.data, content->upload_id.data); + } + } + } + + if (OB_SUCCESS == ret && COS_TRUE == params->truncated) { + if (NULL == (next_key_marker = apr_psprintf(ctx->mem_pool, "%.*s", + params->next_key_marker.len, + params->next_key_marker.data))) { + ret = OB_COS_ERROR; + cos_warn_log("[COS]get next key marker is null, ret=%d\n", ret); + } else if (NULL == (next_upload_id_marker = apr_psprintf(ctx->mem_pool, "%.*s", + params->next_upload_id_marker.len, + params->next_upload_id_marker.data))) { + ret = OB_COS_ERROR; + cos_warn_log("[COS]get next upload id marker is null, ret=%d\n", ret); + } else { + cos_str_set(¶ms->key_marker, next_key_marker); + cos_str_set(¶ms->upload_id_marker, next_upload_id_marker); + cos_list_init(¶ms->upload_list); + } + } + } while (COS_TRUE == params->truncated && OB_SUCCESS == ret); + } return ret; } diff --git a/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.h b/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.h index 2fac5d4c6..b035bbe17 100644 --- a/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.h +++ b/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.h @@ -132,6 +132,30 @@ struct OB_PUBLIC_API CosStringBuffer } return match; } + + int32_t get_data_size() const + { + return (!empty() && '\0' == data_[size_ - 1]) ? size_ - 1 : size_; + } + + bool is_prefix_of(const char *str, const int32_t str_len) const + { + bool match = false; + const int32_t data_size = get_data_size(); + if (str == data_) { + match = (data_size <= str_len) ? true : false; + } else if (data_size > str_len) { + match = false; + } else if (0 == memcmp(data_, str, data_size)) { + match = true; + } + return match; + } + + bool is_end_with_slash_and_null() const + { + return (NULL != data_ && size_ >= 2 && data_[size_ - 1] == '\0' && data_[size_ - 2] == '/'); + } }; @@ -263,29 +287,40 @@ public: const CosStringBuffer &object_name, bool &is_tagging); - struct CosListObjPara { + struct CosListObjPara + { enum class CosListType { COS_LIST_INVALID, - COS_LIST_CB_ARG + COS_LIST_CB_ARG, + COS_PART_LIST_CTX }; CosListObjPara() - : arg_(NULL), cur_full_path_slice_name_(NULL), - full_path_size_(0), cur_object_size_(0), - next_flag_(false), type_(CosListType::COS_LIST_INVALID) + : arg_(NULL), cur_obj_full_path_(NULL), + full_path_size_(0), cur_object_size_str_(NULL), + next_flag_(false), type_(CosListType::COS_LIST_INVALID), + next_token_(NULL), next_token_size_(0), finish_part_list_(false) { last_container_name_.d_name[0] = '\0'; last_container_name_.d_type = DT_REG; } - void* arg_; - char* cur_full_path_slice_name_; + int set_cur_obj_meta( + char *obj_full_path, + const int64_t full_path_size, + char *object_size_str); + + void *arg_; + char *cur_obj_full_path_; struct dirent last_container_name_; int64_t full_path_size_; - int64_t cur_object_size_; + char *cur_object_size_str_; bool next_flag_; CosListType type_; + char *next_token_; + int64_t next_token_size_; + bool finish_part_list_; }; typedef int (*handleObjectNameFunc)(CosListObjPara&); @@ -294,6 +329,14 @@ public: // in the inner sub directories. // dir_name must be end with "/\0". static int list_objects( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &dir_name, + handleObjectNameFunc handle_object_name_f, + void *arg); + + // Only list up-to 1000 objects + static int list_part_objects( Handle *h, const CosStringBuffer &bucket_name, const CosStringBuffer &dir_name, @@ -315,8 +358,39 @@ public: const CosStringBuffer &bucket_name, const CosStringBuffer &dir_name, bool &is_empty_dir); -}; + static int init_multipart_upload( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name, + char *&upload_id_str); + + static int upload_part_from_buffer( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name, + const CosStringBuffer &upload_id_str, + const int part_num, /*the sequence number of this part, [1, 10000]*/ + const char *buf, + const int64_t buf_size); + + static int complete_multipart_upload( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name, + const CosStringBuffer &upload_id_str); + + static int abort_multipart_upload( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name, + const CosStringBuffer &upload_id_str); + + static int del_unmerged_parts( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name); +}; #undef OB_PUBLIC_API } diff --git a/deps/oblib/src/lib/restore/cos/ob_cos_wrapper_handle.cpp b/deps/oblib/src/lib/restore/cos/ob_cos_wrapper_handle.cpp index 0ca3d684e..1f27bb829 100644 --- a/deps/oblib/src/lib/restore/cos/ob_cos_wrapper_handle.cpp +++ b/deps/oblib/src/lib/restore/cos/ob_cos_wrapper_handle.cpp @@ -231,5 +231,15 @@ int ObCosWrapperHandle::set_delete_mode(const char *parameter) return ret; } +void* ObCosWrapperHandle::alloc_mem(size_t size) +{ + return allocator_.alloc(size); +} + +void ObCosWrapperHandle::free_mem(void *addr) +{ + allocator_.free(addr); +} + } // common } // oceanbase diff --git a/deps/oblib/src/lib/restore/cos/ob_cos_wrapper_handle.h b/deps/oblib/src/lib/restore/cos/ob_cos_wrapper_handle.h index 3a5671f31..caaee5a92 100644 --- a/deps/oblib/src/lib/restore/cos/ob_cos_wrapper_handle.h +++ b/deps/oblib/src/lib/restore/cos/ob_cos_wrapper_handle.h @@ -61,6 +61,9 @@ public: int set_delete_mode(const char *parameter); int64_t get_delete_mode() const { return delete_mode_; } + void *alloc_mem(size_t size); + void free_mem(void *addr); + private: bool is_inited_; qcloud_cos::ObCosWrapper::Handle *handle_; diff --git a/deps/oblib/src/lib/restore/ob_i_storage.cpp b/deps/oblib/src/lib/restore/ob_i_storage.cpp new file mode 100644 index 000000000..84a337192 --- /dev/null +++ b/deps/oblib/src/lib/restore/ob_i_storage.cpp @@ -0,0 +1,600 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "ob_i_storage.h" +#include "lib/container/ob_se_array_iterator.h" + +namespace oceanbase +{ +namespace common +{ + +static const char SLASH = '/'; +bool is_end_with_slash(const char *str) +{ + bool bret = false; + int64_t str_len = -1; + if (OB_NOT_NULL(str) && (str_len = strlen(str)) > 0) { + bret = (SLASH == str[str_len - 1]); + } + return bret; +} + +int c_str_to_int(const char *str, int64_t &num) +{ + int ret = OB_SUCCESS; + errno = 0; + char *end_str = NULL; + if (OB_ISNULL(str) || OB_UNLIKELY(0 == strlen(str))) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "c_str_to_int str should not be null/empty", KP(str)); + } else { + num = strtoll(str, &end_str, 10); + if (errno != 0 || (NULL != end_str && *end_str != '\0')) { + ret = OB_INVALID_DATA; + OB_LOG(WARN, "strtoll convert string to int value fail", K(str), K(num), + "error", strerror(errno), K(end_str)); + } + } + return ret; +} + +int handle_listed_object(ObBaseDirEntryOperator &op, + const char *obj_name, const int64_t obj_name_len, const int64_t obj_size) +{ + int ret = OB_SUCCESS; + dirent entry; + entry.d_type = DT_REG; + if (OB_ISNULL(obj_name) + || OB_UNLIKELY(sizeof(entry.d_name) <= obj_name_len || obj_name_len <= 0)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid arguments", + K(ret), K(obj_name), K(obj_name_len), K(sizeof(entry.d_name))); + } else { + if (op.need_get_file_size()) { + if (OB_UNLIKELY(obj_size < 0)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid object size", K(obj_size)); + } else { + op.set_size(obj_size); + } + } + + if (OB_SUCC(ret)) { + MEMCPY(entry.d_name, obj_name, obj_name_len); + entry.d_name[obj_name_len] = '\0'; + if (OB_FAIL(op.func(&entry))) { + OB_LOG(WARN, "fail to exe application callback for listed object", + K(ret), K(obj_name), K(obj_name_len), K(obj_size)); + } + } + } + return ret; +} + +int handle_listed_directory(ObBaseDirEntryOperator &op, + const char *dir_name, const int64_t dir_name_len) +{ + int ret = OB_SUCCESS; + dirent entry; + entry.d_type = DT_DIR; + if (OB_ISNULL(dir_name) + || OB_UNLIKELY(sizeof(entry.d_name) <= dir_name_len || dir_name_len <= 0)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid arguments", + K(ret), K(dir_name), K(dir_name_len), K(sizeof(entry.d_name))); + } else { + MEMCPY(entry.d_name, dir_name, dir_name_len); + entry.d_name[dir_name_len] = '\0'; + if (OB_FAIL(op.func(&entry))) { + OB_LOG(WARN, "fail to exe application callback for listed directory", + K(ret), K(dir_name), K(dir_name_len)); + } + } + return ret; +} + +static int get_storage_prefix_from_path(const common::ObString &uri, const char *&prefix) +{ + int ret = OB_SUCCESS; + if (uri.prefix_match(OB_OSS_PREFIX)) { + prefix = OB_OSS_PREFIX; + } else if (uri.prefix_match(OB_COS_PREFIX)) { + prefix = OB_COS_PREFIX; + } else if (uri.prefix_match(OB_S3_PREFIX)) { + prefix = OB_S3_PREFIX; + } else if (uri.prefix_match(OB_FILE_PREFIX)) { + prefix = OB_FILE_PREFIX; + } else { + ret = OB_INVALID_BACKUP_DEST; + STORAGE_LOG(ERROR, "invalid backup uri", K(ret), K(uri)); + } + return ret; +} + +int build_bucket_and_object_name(ObIAllocator &allocator, + const ObString &uri, ObString &bucket, ObString &object) +{ + int ret = OB_SUCCESS; + ObString::obstr_size_t bucket_start = 0; + ObString::obstr_size_t bucket_end = 0; + ObString::obstr_size_t object_start = 0; + char *bucket_name_buff = NULL; + char *object_name_buff = NULL; + + const char *prefix = "UNKNOWN"; + if (OB_FAIL(get_storage_prefix_from_path(uri, prefix))) { + OB_LOG(WARN, "fail to get storage type", K(ret), K(uri)); + } else { + bucket_start = static_cast(strlen(prefix)); + if (OB_UNLIKELY(bucket_start >= uri.length())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "bucket name is empty", K(uri), K(ret), K(bucket_start)); + } + for (int64_t i = bucket_start; OB_SUCC(ret) && i < uri.length() - 1; i++) { + if ('/' == *(uri.ptr() + i) && '/' == *(uri.ptr() + i + 1)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "uri has two // ", K(uri), K(ret), K(i)); + break; + } + } + } + + if (OB_FAIL(ret)) { + } else if (prefix == OB_FILE_PREFIX) { + // for nfs, bucket is empty + if (OB_FAIL(ob_write_string(allocator, uri.ptr() + bucket_start, object, true/*c_style*/))) { + OB_LOG(WARN, "fail to deep copy object", K(uri), K(bucket_start), K(ret)); + } + } else { + for (bucket_end = bucket_start; OB_SUCC(ret) && bucket_end < uri.length(); ++bucket_end) { + if ('/' == *(uri.ptr() + bucket_end)) { + ObString::obstr_size_t bucket_length = bucket_end - bucket_start; + //must end with '\0' + if (OB_UNLIKELY(bucket_length <= 0)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "bucket name is empty", K(ret), K(uri), K(bucket_start), K(bucket_length)); + } else if (OB_UNLIKELY(bucket_end + 1 >= uri.length())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "object name is empty", K(uri), K(ret), K(bucket_end)); + } else if (OB_ISNULL(bucket_name_buff = + static_cast(allocator.alloc(bucket_length + 1)))) { + OB_LOG(WARN, "failed to alloc bucket name buff", K(ret), K(uri), K(bucket_length)); + } else if (OB_FAIL(databuff_printf(bucket_name_buff, bucket_length + 1, + "%.*s", bucket_length, uri.ptr() + bucket_start))) { + OB_LOG(WARN, "fail to deep copy bucket", K(uri), K(bucket_start), K(bucket_length), K(ret)); + } else { + bucket.assign_ptr(bucket_name_buff, bucket_length + 1);// must include '\0' + } + break; + } + } + + // parse the object name + if (OB_SUCC(ret)) { + object_start = bucket_end + 1; + ObString::obstr_size_t object_length = uri.length() - object_start; + //must end with '\0' + if (OB_UNLIKELY(object_length <= 0)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "bucket name is empty", K(ret), K(uri), K(object_start), K(object_length)); + } else if (OB_FAIL(ob_write_string(allocator, uri.ptr() + object_start, object, true/*c_style*/))) { + OB_LOG(WARN, "fail to deep copy object", K(uri), K(object_start), K(object_length), K(ret)); + } + } + } + + if (OB_SUCC(ret)) { + OB_LOG(DEBUG, "get bucket object name", K(uri), K(bucket), K(object)); + } + return ret; +} + +int construct_fragment_full_name(const ObString &logical_appendable_object_name, + const char *fragment_name, char *name_buf, const int64_t name_buf_len) +{ + int ret = OB_SUCCESS; + int64_t pos = 0; + const char *suffix = NULL; + if (OB_ISNULL(fragment_name) || OB_ISNULL(name_buf) || OB_UNLIKELY(strlen(fragment_name) <= 0) + || OB_UNLIKELY(logical_appendable_object_name.empty() || name_buf_len <= 0)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid arguments", K(ret), + K(logical_appendable_object_name), KP(fragment_name), KP(name_buf), K(name_buf_len)); + } else if (OB_FAIL(databuff_printf(name_buf, name_buf_len, pos, "%s/%s%s", + logical_appendable_object_name.ptr(), + OB_S3_APPENDABLE_FRAGMENT_PREFIX, fragment_name))) { + OB_LOG(WARN, "failed to construct formatted mock append object fragment name", + K(ret), K(logical_appendable_object_name), K(fragment_name)); + } else { + suffix = logical_appendable_object_name.reverse_find('.'); + if (OB_NOT_NULL(suffix)) { + if (OB_UNLIKELY(strlen(suffix) <= 1 || strlen(suffix) >= MAX_APPENDABLE_FRAGMENT_SUFFIX_LENGTH)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "object name has invalid suffix", + K(ret), K(logical_appendable_object_name), K(suffix)); + } else if (OB_FAIL(databuff_printf(name_buf, name_buf_len, pos, "%s", suffix))) { + OB_LOG(WARN, "failed to set formatted mock append object fragment suffix", + K(ret), K(logical_appendable_object_name), K(fragment_name), K(suffix)); + } + } + } + return ret; +} + +int construct_fragment_full_name(const ObString &logical_appendable_object_name, + const int64_t start, const int64_t end, char *name_buf, const int64_t name_buf_len) +{ + int ret = OB_SUCCESS; + char fragment_name[MAX_APPENDABLE_FRAGMENT_LENGTH] = { 0 }; + if (OB_UNLIKELY(start < 0 || end <= start)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid arguments", K(ret), K(start), K(end)); + } else if (OB_FAIL(databuff_printf(fragment_name, sizeof(fragment_name), "%ld-%ld", start, end))) { + OB_LOG(WARN, "failed to construct mock append object fragment name", K(ret), K(start), K(end)); + } else if (OB_FAIL(construct_fragment_full_name(logical_appendable_object_name, + fragment_name, name_buf, name_buf_len))) { + OB_LOG(WARN, "failed to construct mock append object fragment name", + K(ret), K(start), K(end), K(fragment_name), K(logical_appendable_object_name)); + } + return ret; +} + +/*--------------------------------ObAppendableFragmentMeta--------------------------------*/ +OB_SERIALIZE_MEMBER(ObAppendableFragmentMeta, start_, end_); + +int ObAppendableFragmentMeta::assign(const ObAppendableFragmentMeta &other) +{ + int ret = OB_SUCCESS; + start_ = other.start_; + end_ = other.end_; + type_ = other.type_; + MEMCPY(suffix_, other.suffix_, sizeof(suffix_)); + return ret; +} + +int ObAppendableFragmentMeta::parse_from(ObString &fragment_name) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!fragment_name.prefix_match(OB_S3_APPENDABLE_FRAGMENT_PREFIX))) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid fragment prefix", K(ret), K(fragment_name)); + } else { + fragment_name += strlen(OB_S3_APPENDABLE_FRAGMENT_PREFIX); + const char *fragment_suffix = fragment_name.reverse_find('.'); + fragment_name.clip(fragment_suffix); + if (OB_NOT_NULL(fragment_suffix)) { + if (strlen(fragment_suffix) <= 1 || strlen(fragment_suffix) >= MAX_APPENDABLE_FRAGMENT_SUFFIX_LENGTH) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid fragment suffix", K(ret), K(fragment_suffix)); + } else { + STRCPY(suffix_, fragment_suffix); + } + } + } + + if (OB_FAIL(ret)) { + } else if (0 == fragment_name.compare(OB_S3_APPENDABLE_FORMAT_META)) { + type_ = ObAppendableFragmentType::APPENDABLE_FRAGMENT_FORMAT_META; + } else if (0 == fragment_name.compare(OB_S3_APPENDABLE_SEAL_META)) { + type_ = ObAppendableFragmentType::APPENDABLE_FRAGMENT_SEAL_META; + } else { + ObArenaAllocator allocator(ObModIds::BACKUP); + ObString start_part = fragment_name.split_on('-'); + ObString start_string; + ObString end_string; + if (OB_UNLIKELY(!start_part.is_numeric() || !fragment_name.is_numeric())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "unexpected fragment name", K(start_part), K(fragment_name)); + } else if (OB_FAIL(ob_write_string(allocator, start_part, start_string, true))) { + OB_LOG(WARN, "fail to deep copy start part of fragment name", + K(ret), K(start_string), K(fragment_name), K_(suffix)); + } else if (OB_FAIL(ob_write_string(allocator, fragment_name, end_string, true))) { + OB_LOG(WARN, "fail to deep copy end part of fragment name", + K(ret), K(start_string), K(fragment_name), K_(suffix)); + } else if (OB_FAIL(c_str_to_int(start_string.ptr(), start_))) { + OB_LOG(WARN, "fail to parse 'start'", K(ret), K(start_string), K(fragment_name)); + } else if (OB_FAIL(c_str_to_int(end_string.ptr(), end_))) { + OB_LOG(WARN, "fail to parse 'end'", K(ret), K(end_string), K(fragment_name)); + } else { + type_ = ObAppendableFragmentType::APPENDABLE_FRAGMENT_DATA; + if (OB_UNLIKELY(!is_valid())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid fragment name", K(ret), K_(type), K(start_string), K(end_string), + K(fragment_name), K_(start), K_(end), K_(suffix)); + } + } + } + return ret; +} + +int64_t ObAppendableFragmentMeta::to_string(char *buf, const int64_t len) const +{ + int64_t pos = 0; + if (OB_NOT_NULL(buf) && OB_LIKELY(len > 0)) { + if (type_ == ObAppendableFragmentType::APPENDABLE_FRAGMENT_DATA) { + pos = snprintf(buf, len, "%ld-%ld%s", start_, end_, suffix_); + } else { + const char *meta_name = (type_ == ObAppendableFragmentType::APPENDABLE_FRAGMENT_FORMAT_META) ? + OB_S3_APPENDABLE_FORMAT_META : + OB_S3_APPENDABLE_SEAL_META; + pos = snprintf(buf, len, "%s%s%s", OB_S3_APPENDABLE_FRAGMENT_PREFIX, meta_name, suffix_); + } + + if (pos < 0) { + pos = 0; + } else if (pos >= len) { + pos = len - 1; + } + } + return pos; +} + +/*--------------------------------ObStorageObjectMeta--------------------------------*/ +OB_SERIALIZE_MEMBER(ObStorageObjectMetaBase, length_); +OB_SERIALIZE_MEMBER((ObStorageObjectMeta, ObStorageObjectMetaBase), type_, fragment_metas_); + +void ObStorageObjectMeta::reset() +{ + // reset do not change obj type + ObStorageObjectMetaBase::reset(); + fragment_metas_.reset(); +} + +bool ObStorageObjectMeta::is_valid() const +{ + bool is_valid_flag = (length_ >= 0); + if (is_simulate_append_type()) { + for (int64_t i = 0; is_valid_flag && i < fragment_metas_.count(); i++) { + is_valid_flag = fragment_metas_[i].is_valid(); + } + if (is_valid_flag && fragment_metas_.count() > 1) { + for (int64_t i = 1; is_valid_flag && i < fragment_metas_.count(); i++) { + is_valid_flag = (fragment_metas_[i - 1].start_ < fragment_metas_[i].start_ + && fragment_metas_[i - 1].end_ < fragment_metas_[i].end_); + } + } + } else { + // for normal objs, fragment_metas_ must be empty; + is_valid_flag &= fragment_metas_.empty(); + } + + return is_valid_flag; +} + +bool ObStorageObjectMeta::fragment_meta_cmp_func( + const ObAppendableFragmentMeta &left, + const ObAppendableFragmentMeta &right) +{ + // for fragments with the same start offset, prioritize placing the largest fragment at the beginning, + // to facilitate subsequent cleaning of overlapping fragments + return left.start_ < right.start_ || (left.start_ == right.start_ && left.end_ > right.end_); +} + +int ObStorageObjectMeta::get_needed_fragments( + const int64_t start, + const int64_t end, + ObArray &fragments) +{ + int ret = OB_SUCCESS; + fragments.reset(); + if (OB_UNLIKELY(start < 0 || end <= start)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid arguments", K(start), K(end)); + } else if (OB_UNLIKELY(!is_simulate_append_type() || !is_valid())) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "invalid storage object meta", K(ret), K_(type), K_(fragment_metas)); + } else if (OB_UNLIKELY(fragment_metas_.empty())) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "empty storage appendable object", K(ret)); + } else if (fragment_metas_[fragment_metas_.count() - 1].end_ <= start) { + // the data to be read does not exist + } else { + int64_t cur_fragment_idx = -1; + ObAppendableFragmentMeta start_meta; + start_meta.start_ = start; + ObSEArray::iterator it = + std::upper_bound(fragment_metas_.begin(), fragment_metas_.end(), start_meta, fragment_meta_cmp_func); + if (it == fragment_metas_.begin()) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "the object data may contain holes, can't read correct data", K(ret), K(start), K(end), + K(fragment_metas_[0].start_)); + } else if (FALSE_IT(cur_fragment_idx = it - fragment_metas_.begin() - 1)) { + } else { + int64_t last_fragment_end = fragment_metas_[cur_fragment_idx].start_; + while (OB_SUCC(ret) && cur_fragment_idx < fragment_metas_.count() + && fragment_metas_[cur_fragment_idx].start_ < end + && last_fragment_end < end) { + if (fragment_metas_[cur_fragment_idx].start_ > last_fragment_end) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "the object data may contain holes, can't read correct data", K(ret), K(start), K(end), + K(fragment_metas_[cur_fragment_idx]), K(last_fragment_end)); + } else if (OB_FAIL(fragments.push_back(fragment_metas_[cur_fragment_idx]))) { + OB_LOG(WARN, "fail to push back fragement", K(ret), K(fragments)); + } else { + last_fragment_end = fragment_metas_[cur_fragment_idx].end_; + cur_fragment_idx++; + } + } + } + } + return ret; +} + +/*--------------------------------ObStorageListCtxBase--------------------------------*/ +int ObStorageListCtxBase::init( + ObArenaAllocator &allocator, + const int64_t max_list_num, + const bool need_size) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(max_list_num < 1)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid argument", K(ret), K(max_list_num)); + } else { + max_list_num_ = max_list_num; + max_name_len_ = OB_MAX_URI_LENGTH; + need_size_ = need_size; + if (OB_ISNULL(name_arr_ = static_cast(allocator.alloc(sizeof(void *) * max_list_num_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "fail to alloc name_arr buff", K(ret), K(*this)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && (i < max_list_num_); ++i) { + if (OB_ISNULL(name_arr_[i] = static_cast(allocator.alloc(max_name_len_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "fail to alloc name buff", K(ret), K(i), K(*this)); + } else { + name_arr_[i][0] = '\0'; + } + } + } + + if (OB_SUCC(ret) && need_size) { + if (OB_ISNULL(size_arr_ = static_cast(allocator.alloc(sizeof(int64_t) * max_list_num_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "fail to alloc size_arr buff", K(ret), K(*this)); + } + } + } + return ret; +} + +void ObStorageListCtxBase::reset() +{ + max_list_num_ = 0; + name_arr_ = NULL; + max_name_len_ = 0; + rsp_num_ = 0; + has_next_ = false; + need_size_ = false; + size_arr_ = NULL; +} + +bool ObStorageListCtxBase::is_valid() const +{ + bool bret = (max_list_num_ > 0) && (name_arr_ != NULL) && (max_name_len_ > 0); + if (need_size_) { + bret &= (size_arr_ != NULL); + } + return bret; +} + +/*--------------------------------ObStorageListObjectsCtx--------------------------------*/ +void ObStorageListObjectsCtx::reset() +{ + next_token_ = NULL; + next_token_buf_len_ = 0; + cur_appendable_full_obj_path_ = NULL; + ObStorageListCtxBase::reset(); +} + +int ObStorageListObjectsCtx::init( + ObArenaAllocator &allocator, + const int64_t max_list_num, + const bool need_size) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ObStorageListCtxBase::init(allocator, max_list_num, need_size))) { + OB_LOG(WARN, "fail to init storage_list_ctx_base", K(ret), K(max_list_num), K(need_size)); + } else { + next_token_buf_len_ = OB_MAX_URI_LENGTH; + if (OB_ISNULL(next_token_ = static_cast(allocator.alloc(next_token_buf_len_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "fail to alloc next_token buff", K(ret), K(*this)); + } else { + next_token_[0] = '\0'; + + if (OB_ISNULL(cur_appendable_full_obj_path_ = static_cast(allocator.alloc(OB_MAX_URI_LENGTH)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "fail to alloc cur appendable full obj path buff", K(ret), K(*this)); + } else { + cur_appendable_full_obj_path_[0] = '\0'; + } + } + } + return ret; +} + +int ObStorageListObjectsCtx::set_next_token( + const bool has_next, + const char *next_token, + const int64_t next_token_len) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_valid())) { + ret = OB_NOT_INIT; + OB_LOG(WARN, "ObStorageListObjectsCtx not init", K(ret)); + } else { + has_next_ = has_next; + if (has_next) { + if (OB_ISNULL(next_token) || OB_UNLIKELY(next_token_len <= 0)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid arguments", K(ret), K(has_next), K(next_token), K(next_token_len)); + } else if (OB_UNLIKELY(next_token_len >= next_token_buf_len_)) { + ret = OB_SIZE_OVERFLOW; + OB_LOG(WARN, "fail to set next token, size overflow", + K(ret), K(has_next), K(next_token_len)); + } else { + MEMCPY(next_token_, next_token, next_token_len); + next_token_[next_token_len] = '\0'; + } + } else { + next_token_[0] = '\0'; + } + } + return ret; +} + +int ObStorageListObjectsCtx::handle_object( + const char *obj_path, + const int obj_path_len, + const int64_t obj_size) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_valid())) { + ret = OB_NOT_INIT; + OB_LOG(WARN, "ObStorageListObjectsCtx not init", K(ret)); + } else if (OB_UNLIKELY(obj_size < 0 || obj_path_len >= max_name_len_ + || obj_path_len <= 0 || rsp_num_ >= max_list_num_) + || OB_ISNULL(obj_path) || OB_ISNULL(name_arr_[rsp_num_])) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid arguments", K(ret), K(obj_path), K(obj_path_len), K(obj_size), K(*this)); + } else { + if (need_size_) { + size_arr_[rsp_num_] = obj_size; + } + MEMCPY(name_arr_[rsp_num_], obj_path, obj_path_len); + name_arr_[rsp_num_][obj_path_len] = '\0'; + ++rsp_num_; + } + return ret; +} + +/*--------------------------------ObStorageListFilesCtx--------------------------------*/ +bool ObStorageListFilesCtx::is_valid() const +{ + bool bret = ObStorageListCtxBase::is_valid(); + if (already_open_dir_) { + bret &= (open_dir_ != NULL); + } + return bret; +} + +void ObStorageListFilesCtx::reset() +{ + open_dir_ = NULL; + already_open_dir_ = false; + ObStorageListCtxBase::reset(); +} + +}//common +}//oceanbase \ No newline at end of file diff --git a/deps/oblib/src/lib/restore/ob_i_storage.h b/deps/oblib/src/lib/restore/ob_i_storage.h index b6f493370..256a9b491 100644 --- a/deps/oblib/src/lib/restore/ob_i_storage.h +++ b/deps/oblib/src/lib/restore/ob_i_storage.h @@ -16,6 +16,7 @@ #include "lib/ob_define.h" #include "lib/string/ob_string.h" #include "lib/container/ob_array.h" +#include "lib/container/ob_se_array.h" #include "common/storage/ob_device_common.h" #include "ob_storage_info.h" @@ -23,6 +24,11 @@ namespace oceanbase { namespace common { + +static constexpr int64_t MAX_APPENDABLE_FRAGMENT_SUFFIX_LENGTH = 64; +static constexpr int64_t MAX_APPENDABLE_FRAGMENT_LENGTH = 128; +static constexpr char APPENDABLE_OBJECT_ALLOCATOR[] = "AppendableAlloc"; + enum StorageOpenMode { CREATE_OPEN_LOCK = 0, // default, create and open @@ -31,6 +37,201 @@ enum StorageOpenMode CREATE_OPEN_NOLOCK = 3, // create and open nolock }; +enum ObStorageCRCAlgorithm +{ + OB_INVALID_CRC_ALGO = 0, + OB_CRC32_ALGO = 1, +}; + +enum ObStorageObjectType +{ + OB_OBJ_INVALID = 0, + OB_OBJ_NORMAL = 1, + OB_OBJ_SIMULATE_APPEND = 2, + OB_FS_DIR = 3, + OB_FS_FILE = 4, +}; + +// check the str is end with '/' or not +bool is_end_with_slash(const char *str); +int c_str_to_int(const char *str, int64_t &num); +int handle_listed_object(ObBaseDirEntryOperator &op, + const char *obj_name, const int64_t obj_name_len, const int64_t obj_size); +int handle_listed_directory(ObBaseDirEntryOperator &op, + const char *dir_name, const int64_t dir_name_len); +int build_bucket_and_object_name(ObIAllocator &allocator, + const ObString &uri, ObString &bucket, ObString &object); +int construct_fragment_full_name(const ObString &logical_appendable_object_name, + const char *fragment_name, char *name_buf, const int64_t name_buf_len); +int construct_fragment_full_name(const ObString &logical_appendable_object_name, + const int64_t start, const int64_t end, char *name_buf, const int64_t name_buf_len); + +struct ObStorageObjectMetaBase +{ + OB_UNIS_VERSION_V(1); +public: + ObStorageObjectMetaBase() : type_(ObStorageObjectType::OB_OBJ_INVALID) { reset(); } + ~ObStorageObjectMetaBase() { reset(); } + + void reset() { is_exist_ = false; length_ = -1; } + + TO_STRING_KV(K_(is_exist), K_(length)); + + bool is_exist_; + int64_t length_; + ObStorageObjectType type_; +}; + +// Each fragment meta corresponds to a normal object in a 'dir'. +// The 'dir' name is the S3 appendable object name. +// Fragment name format: /xxx/xxx/appendable_obj_name/prefix-start-end[-suffix] +// 'prefix' is a special string which represents this object is a S3 appendable object fragment. +// 'start-end' means the data range covered by this file. [start, end), include start、not include end. +// 'suffix' may exist, mainly used by deleting file situation. +struct ObAppendableFragmentMeta +{ + OB_UNIS_VERSION_V(1); +public: + enum ObAppendableFragmentType + { + APPENDABLE_FRAGMENT_DATA = 0, + APPENDABLE_FRAGMENT_FORMAT_META = 1, + APPENDABLE_FRAGMENT_SEAL_META = 2, + }; + + ObAppendableFragmentMeta() + : start_(-1), end_(-1), type_(ObAppendableFragmentType::APPENDABLE_FRAGMENT_DATA) { + suffix_[0] = '\0'; + } + virtual ~ObAppendableFragmentMeta() {} + + bool is_format_meta() const { return type_ == ObAppendableFragmentType::APPENDABLE_FRAGMENT_FORMAT_META; } + bool is_seal_meta() const { return type_ == ObAppendableFragmentType::APPENDABLE_FRAGMENT_SEAL_META; } + bool is_data() const { return type_ == ObAppendableFragmentType::APPENDABLE_FRAGMENT_DATA; } + bool is_valid() const + { + return (is_format_meta()) || (is_seal_meta()) || (is_data() && start_ >= 0 && end_ > start_); + } + + int assign(const ObAppendableFragmentMeta &other); + int parse_from(ObString &fragment_name); + int64_t to_string(char *buf, const int64_t len) const; + + int64_t get_length() const { return end_ - start_; } + + int64_t start_; + int64_t end_; + char suffix_[MAX_APPENDABLE_FRAGMENT_SUFFIX_LENGTH]; + ObAppendableFragmentType type_; +}; + +struct ObStorageObjectMeta : public ObStorageObjectMetaBase +{ + OB_UNIS_VERSION_V(1); +public: + ObStorageObjectMeta() + : ObStorageObjectMetaBase(), + fragment_metas_() + {} + + ~ObStorageObjectMeta() { reset(); } + void reset(); + + bool is_valid() const; + // Based on the range[start, end), to choose the needed files and save these meta into @fragments. + int get_needed_fragments(const int64_t start, const int64_t end, + ObArray &fragments); + + bool is_object_file_type() const + { + return (type_ == ObStorageObjectType::OB_OBJ_NORMAL) || + (type_ == ObStorageObjectType::OB_FS_FILE); + } + bool is_simulate_append_type() const { return type_ == ObStorageObjectType::OB_OBJ_SIMULATE_APPEND; } + + static bool fragment_meta_cmp_func(const ObAppendableFragmentMeta &left, const ObAppendableFragmentMeta &right); + + TO_STRING_KV(K_(is_exist), K_(length), K_(type), K_(fragment_metas)); + + ObSEArray fragment_metas_; +}; + +struct ObStorageListCtxBase +{ +public: + int64_t max_list_num_; // each round list, can only get up-to @max_list_num_ items. + char **name_arr_; // for object storage, save full path; for file system, save file name. + int64_t max_name_len_; // no matter full path, or just object/file name, can not be longer than this value. + int64_t rsp_num_; // real listed-item number which is obtained from the listed result + bool has_next_; // list result can only return up-to 1000 objects once, thus may need to multi operation. + bool need_size_; // If true, that means when we list items, we also need to get each item's size + int64_t *size_arr_; // save all the length of each object/file (the order is the same with name_arr) + + ObStorageListCtxBase() + : max_list_num_(0), name_arr_(NULL), max_name_len_(0), rsp_num_(0), + has_next_(false), need_size_(false), size_arr_(NULL) + {} + + virtual ~ObStorageListCtxBase() { reset(); } + + int init(ObArenaAllocator &allocator, const int64_t max_list_num, const bool need_size); + + void reset(); + + bool is_valid() const; + + TO_STRING_KV(K_(max_list_num), K_(max_name_len), K_(rsp_num), K_(has_next), K_(need_size), + KP_(name_arr), KP_(size_arr)); +}; + +// Used for object storage +struct ObStorageListObjectsCtx : public ObStorageListCtxBase +{ +public: + char *next_token_; // save marker/continuation_token + int64_t next_token_buf_len_; // length of marker/continuation_token should not be longer than this value + char *cur_appendable_full_obj_path_; + + ObStorageListObjectsCtx() + : next_token_(NULL), next_token_buf_len_(0), cur_appendable_full_obj_path_(NULL) + {} + + virtual ~ObStorageListObjectsCtx() { reset(); } + + void reset(); + + int init(ObArenaAllocator &allocator, const int64_t max_list_num, const bool need_size); + + bool is_valid() const { return ObStorageListCtxBase::is_valid() && (next_token_ != NULL) + && (next_token_buf_len_ > 0); } + int set_next_token(const bool has_next, const char *next_token, const int64_t next_token_len); + int handle_object(const char *obj_path, const int obj_path_len, const int64_t obj_size); + + INHERIT_TO_STRING_KV("ObStorageListCtxBase", ObStorageListCtxBase, + K_(next_token), K_(next_token_buf_len), K_(cur_appendable_full_obj_path)); +}; + +// Used for file system +struct ObStorageListFilesCtx : public ObStorageListCtxBase +{ +public: + DIR *open_dir_; + struct dirent next_entry_; // If has_next=true, it will get the next entry based on this value. + bool already_open_dir_; // only during the first round, need to open dir + + ObStorageListFilesCtx() + : open_dir_(NULL), next_entry_(), already_open_dir_(false) + {} + + virtual ~ObStorageListFilesCtx() { reset(); } + + void reset(); + + bool is_valid() const; + + INHERIT_TO_STRING_KV("ObStorageListCtxBase", ObStorageListCtxBase, K_(already_open_dir)); +}; + class ObIStorageUtil { public: @@ -44,20 +245,27 @@ public: virtual void close() = 0; virtual int is_exist(const common::ObString &uri, bool &exist) = 0; virtual int get_file_length(const common::ObString &uri, int64_t &file_length) = 0; + virtual int head_object_meta(const common::ObString &uri, ObStorageObjectMetaBase &obj_meta) = 0; virtual int del_file(const common::ObString &uri) = 0; virtual int write_single_file(const common::ObString &uri, const char *buf, const int64_t size) = 0; virtual int mkdir(const common::ObString &uri) = 0; + // list all objects which are 'prefix-matched' virtual int list_files(const common::ObString &dir_path, common::ObBaseDirEntryOperator &op) = 0; + // If we want to get the listed result and handle it in other logic, we can use this interface. + // @list_ctx will save the listed result(cuz obj_storage can only return up-to 1000 items at a time). + virtual int list_files(const common::ObString &dir_path, ObStorageListCtxBase &list_ctx) = 0; virtual int del_dir(const common::ObString &uri) = 0; virtual int list_directories(const common::ObString &dir_path, common::ObBaseDirEntryOperator &op) = 0; virtual int is_tagging(const common::ObString &uri, bool &is_tagging) = 0; + virtual int del_unmerged_parts(const common::ObString &uri) = 0; }; class ObIStorageReader { public: - virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info) = 0; - virtual int pread(char *buf,const int64_t buf_size, int64_t offset, int64_t &read_size) = 0; + virtual int open(const common::ObString &uri, + common::ObObjectStorageInfo *storage_info, const bool head_meta = true) = 0; + virtual int pread(char *buf,const int64_t buf_size, const int64_t offset, int64_t &read_size) = 0; virtual int close() = 0; virtual int64_t get_length() const = 0; virtual bool is_opened() const = 0; diff --git a/deps/oblib/src/lib/restore/ob_object_device.cpp b/deps/oblib/src/lib/restore/ob_object_device.cpp index 90befd7ba..9ec210a0a 100644 --- a/deps/oblib/src/lib/restore/ob_object_device.cpp +++ b/deps/oblib/src/lib/restore/ob_object_device.cpp @@ -17,15 +17,18 @@ namespace oceanbase namespace common { -const char *OB_STORAGE_ACCESS_TYPES_STR[] = {"reader", "overwriter", "appender", "random_write"}; +const char *OB_STORAGE_ACCESS_TYPES_STR[] = {"reader", "adaptive_reader", "overwriter", + "appender", "random_write", "multipart_writer"}; ObObjectDevice::ObObjectDevice() : storage_info_(), is_started_(false), lock_(common::ObLatchIds::OBJECT_DEVICE_LOCK) { ObMemAttr attr = SET_USE_500("ObjectDevice"); reader_ctx_pool_.set_attr(attr); + adaptive_reader_ctx_pool_.set_attr(attr); appender_ctx_pool_.set_attr(attr); overwriter_ctx_pool_.set_attr(attr); + multipart_writer_ctx_pool_.set_attr(attr); } int ObObjectDevice::init(const ObIODOpts &opts) @@ -41,7 +44,9 @@ void ObObjectDevice::destroy() if (is_started_) { appender_ctx_pool_.reset(); reader_ctx_pool_.reset(); + adaptive_reader_ctx_pool_.reset(); overwriter_ctx_pool_.reset(); + multipart_writer_ctx_pool_.reset(); //close the util util_.close(); //baseinfo will be free with allocator @@ -131,12 +136,16 @@ int ObObjectDevice::get_access_type(ObIODOpts *opts, ObStorageAccessType& access OB_LOG(WARN, "can not find access type!"); } else if (0 == STRCMP(access_type , OB_STORAGE_ACCESS_TYPES_STR[OB_STORAGE_ACCESS_READER])) { access_type_flag = OB_STORAGE_ACCESS_READER; + } else if (0 == STRCMP(access_type , OB_STORAGE_ACCESS_TYPES_STR[OB_STORAGE_ACCESS_ADAPTIVE_READER])) { + access_type_flag = OB_STORAGE_ACCESS_ADAPTIVE_READER; } else if (0 == STRCMP(access_type , OB_STORAGE_ACCESS_TYPES_STR[OB_STORAGE_ACCESS_OVERWRITER])) { access_type_flag = OB_STORAGE_ACCESS_OVERWRITER; } else if (0 == STRCMP(access_type , OB_STORAGE_ACCESS_TYPES_STR[OB_STORAGE_ACCESS_APPENDER])) { access_type_flag = OB_STORAGE_ACCESS_APPENDER; } else if (0 == STRCMP(access_type , OB_STORAGE_ACCESS_TYPES_STR[OB_STORAGE_ACCESS_RANDOMWRITER])) { access_type_flag = OB_STORAGE_ACCESS_RANDOMWRITER; + } else if (0 == STRCMP(access_type , OB_STORAGE_ACCESS_TYPES_STR[OB_STORAGE_ACCESS_MULTIPART_WRITER])) { + access_type_flag = OB_STORAGE_ACCESS_MULTIPART_WRITER; } else { ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "invaild access type!", KCSTRING(access_type)); @@ -165,6 +174,27 @@ int ObObjectDevice::open_for_reader(const char *pathname, void*& ctx) return ret; } +int ObObjectDevice::open_for_adaptive_reader_(const char *pathname, void *&ctx) +{ + int ret = OB_SUCCESS; + ObStorageAdaptiveReader *adaptive_reader = adaptive_reader_ctx_pool_.alloc(); + if (OB_ISNULL(adaptive_reader)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "fail to alloc mem for object device adaptive_reader! ", K(ret), K(pathname)); + } else { + if (OB_FAIL(adaptive_reader->open(pathname, &storage_info_))) { + OB_LOG(WARN, "fail to open for read!", K(ret), K(pathname), K_(storage_info)); + } else { + ctx = (void*)adaptive_reader; + } + } + if (OB_FAIL(ret) && OB_NOT_NULL(adaptive_reader)) { + adaptive_reader_ctx_pool_.free(adaptive_reader); + adaptive_reader = nullptr; + } + return ret; +} + /*ObStorageOssMultiPartWriter is not used int the current version, if we use, later, the open func of overwriter maybe need to add para(just like the open func of appender)*/ int ObObjectDevice::open_for_overwriter(const char *pathname, void*& ctx) @@ -232,6 +262,27 @@ int ObObjectDevice::open_for_appender(const char *pathname, ObIODOpts *opts, voi return ret; } +int ObObjectDevice::open_for_multipart_writer_(const char *pathname, void *&ctx) +{ + int ret = OB_SUCCESS; + ObStorageMultiPartWriter *multipart_writer = multipart_writer_ctx_pool_.alloc(); + if (OB_ISNULL(multipart_writer)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "fail to alloc multipart_writer!", K(ret), K(pathname)); + } else { + if (OB_FAIL(multipart_writer->open(pathname, &storage_info_))) { + OB_LOG(WARN, "fail to open for multipart_writer!", K(ret), K(pathname), K_(storage_info)); + } else { + ctx = (void*)multipart_writer; + } + } + if (OB_FAIL(ret) && OB_NOT_NULL(multipart_writer)) { + multipart_writer_ctx_pool_.free(multipart_writer); + multipart_writer = nullptr; + } + return ret; +} + int ObObjectDevice::release_res(void* ctx, const ObIOFd &fd, ObStorageAccessType access_type) { int ret = OB_SUCCESS; @@ -254,12 +305,24 @@ int ObObjectDevice::release_res(void* ctx, const ObIOFd &fd, ObStorageAccessType OB_LOG(WARN, "fail to close the reader!", K(ret), K(access_type)); } reader_ctx_pool_.free(reader); + } else if (OB_STORAGE_ACCESS_ADAPTIVE_READER == access_type) { + ObStorageAdaptiveReader *adaptive_reader = static_cast(ctx); + if (OB_FAIL(adaptive_reader->close())) { + OB_LOG(WARN, "fail to close the adaptive reader!", K(ret), K(access_type)); + } + adaptive_reader_ctx_pool_.free(adaptive_reader); } else if (OB_STORAGE_ACCESS_OVERWRITER == access_type) { ObStorageWriter *overwriter = static_cast(ctx); if (OB_FAIL(overwriter->close())) { OB_LOG(WARN, "fail to close the overwriter!", K(ret), K(access_type)); } overwriter_ctx_pool_.free(overwriter); + } else if (OB_STORAGE_ACCESS_MULTIPART_WRITER == access_type) { + ObStorageMultiPartWriter *multipart_writer = static_cast(ctx); + if (OB_FAIL(multipart_writer->close())) { + OB_LOG(WARN, "fail to close the multipart writer!", K(ret), K(access_type)); + } + multipart_writer_ctx_pool_.free(multipart_writer); } else { ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "invalid access_type!", K(access_type), K(ret)); @@ -303,11 +366,15 @@ int ObObjectDevice::open(const char *pathname, const int flags, const mode_t mod } else { if (OB_STORAGE_ACCESS_READER == access_type) { ret = open_for_reader(pathname, ctx); + } else if (OB_STORAGE_ACCESS_ADAPTIVE_READER == access_type) { + ret = open_for_adaptive_reader_(pathname, ctx); } else if (OB_STORAGE_ACCESS_APPENDER == access_type || OB_STORAGE_ACCESS_RANDOMWRITER == access_type) { ret = open_for_appender(pathname, opts, ctx); } else if (OB_STORAGE_ACCESS_OVERWRITER == access_type) { ret = open_for_overwriter(pathname, ctx); + } else if (OB_STORAGE_ACCESS_MULTIPART_WRITER == access_type) { + ret = open_for_multipart_writer_(pathname, ctx); } if (OB_FAIL(ret)) { @@ -342,7 +409,7 @@ int ObObjectDevice::close(const ObIOFd &fd) { int ret = OB_SUCCESS; int flag = -1; - void* ctx = NULL; + void *ctx = NULL; fd_mng_.get_fd_flag(fd, flag); if (!fd_mng_.validate_fd(fd, true)) { @@ -356,6 +423,43 @@ int ObObjectDevice::close(const ObIOFd &fd) return ret; } +int ObObjectDevice::seal_for_adaptive(const ObIOFd &fd) +{ + int ret = OB_SUCCESS; + int flag = -1; + void *ctx = NULL; + + fd_mng_.get_fd_flag(fd, flag); + if (!fd_mng_.validate_fd(fd, true)) { + ret = OB_NOT_INIT; + OB_LOG(WARN, "fd is not init!", K(fd.first_id_), K(fd.second_id_)); + } else if (OB_FAIL(fd_mng_.fd_to_ctx(fd, ctx))) { + OB_LOG(WARN, "fail to get ctx accroding fd!", K(ret)); + } else if (OB_ISNULL(ctx)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "fd ctx is null!", K(flag), K(ret)); + } else if (flag == OB_STORAGE_ACCESS_RANDOMWRITER || flag == OB_STORAGE_ACCESS_APPENDER) { + ObStorageAppender *appender = static_cast(ctx); + if (OB_FAIL(appender->seal_for_adaptive())) { + OB_LOG(WARN, "fail to seal!", K(ret), K(flag)); + } + } else { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "unknow access type, not a appender fd!", K(flag), K(ret)); + } + return ret; +} + +int ObObjectDevice::del_unmerged_parts(const char *pathname) +{ + int ret = OB_SUCCESS; + common::ObString uri(pathname); + if (OB_FAIL(util_.del_unmerged_parts(uri))) { + OB_LOG(WARN, "fail to del unmerged parts", K(ret), K(uri)); + } + return ret; +} + int ObObjectDevice::mkdir(const char *pathname, mode_t mode) { UNUSED(mode); @@ -371,24 +475,66 @@ int ObObjectDevice::rmdir(const char *pathname) int ObObjectDevice::unlink(const char *pathname) { + return inner_unlink_(pathname, false/*is_adaptive*/); +} + +int ObObjectDevice::adaptive_unlink(const char *pathname) +{ + const bool is_adaptive = true; + return inner_unlink_(pathname, is_adaptive); +} + +int ObObjectDevice::inner_unlink_(const char *pathname, const bool is_adaptive) +{ + int ret = OB_SUCCESS; common::ObString uri(pathname); - return util_.del_file(uri); + if (OB_FAIL(util_.del_file(uri, is_adaptive))) { + OB_LOG(WARN, "fail to del file", K(ret), K(uri), K(is_adaptive)); + } + return ret; } int ObObjectDevice::exist(const char *pathname, bool &is_exist) { + return inner_exist_(pathname, is_exist, false/*is_adaptive*/); +} + +int ObObjectDevice::adaptive_exist(const char *pathname, bool &is_exist) +{ + const bool is_adaptive = true; + return inner_exist_(pathname, is_exist, is_adaptive); +} + +int ObObjectDevice::inner_exist_(const char *pathname, bool &is_exist, const bool is_adaptive) +{ + int ret = OB_SUCCESS; common::ObString uri(pathname); - return util_.is_exist(uri, is_exist); + if (OB_FAIL(util_.is_exist(uri, is_adaptive, is_exist))) { + OB_LOG(WARN, "fail to check if the file exists", K(ret), K(uri), K(is_adaptive)); + } + return ret; } /*notice: for backup, this interface only return size*/ int ObObjectDevice::stat(const char *pathname, ObIODFileStat &statbuf) +{ + return inner_stat_(pathname, statbuf, false/*is_adaptive*/); +} + +int ObObjectDevice::adaptive_stat(const char *pathname, ObIODFileStat &statbuf) +{ + const bool is_adaptive = true; + return inner_stat_(pathname, statbuf, is_adaptive); +} + +int ObObjectDevice::inner_stat_(const char *pathname, + ObIODFileStat &statbuf, const bool is_adaptive) { int ret = OB_SUCCESS; int64_t length = 0; common::ObString uri(pathname); - if (OB_FAIL(util_.get_file_length(uri, length))) { - OB_LOG(WARN, "fail to get fail length!", K(ret)); + if (OB_FAIL(util_.get_file_length(uri, is_adaptive, length))) { + OB_LOG(WARN, "fail to get file length!", K(ret), K(uri), K(is_adaptive)); } else { statbuf.size_ = length; } @@ -396,15 +542,27 @@ int ObObjectDevice::stat(const char *pathname, ObIODFileStat &statbuf) } int ObObjectDevice::scan_dir(const char *dir_name, common::ObBaseDirEntryOperator &op) +{ + return inner_scan_dir_(dir_name, op, false/*is_adaptive*/); +} + +int ObObjectDevice::adaptive_scan_dir(const char *dir_name, ObBaseDirEntryOperator &op) +{ + const bool is_adaptive = true; + return inner_scan_dir_(dir_name, op, is_adaptive); +} + +int ObObjectDevice::inner_scan_dir_(const char *dir_name, + ObBaseDirEntryOperator &op, const bool is_adaptive) { common::ObString uri(dir_name); int ret = OB_SUCCESS; bool is_dir_scan = false; if (op.is_dir_scan()) { - ret = util_.list_directories(uri, op); + ret = util_.list_directories(uri, is_adaptive, op); is_dir_scan = true; } else { - ret = util_.list_files(uri, op); + ret = util_.list_files(uri, is_adaptive, op); } if (OB_FAIL(ret)) { @@ -425,25 +583,30 @@ int ObObjectDevice::pread(const ObIOFd &fd, const int64_t offset, const int64_t UNUSED(checker); int ret = OB_SUCCESS; int flag = -1; - void* ctx = NULL; + void *ctx = NULL; fd_mng_.get_fd_flag(fd, flag); if (!fd_mng_.validate_fd(fd, true)) { ret = OB_NOT_INIT; OB_LOG(WARN, "fd is not init!", K(fd.first_id_), K(fd.second_id_)); - } else if (flag != OB_STORAGE_ACCESS_READER) { - ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "fd is not a reader fd!", K(flag), K(ret)); } else if (OB_FAIL(fd_mng_.fd_to_ctx(fd, ctx))) { OB_LOG(WARN, "fail to get ctx accroding fd!", K(ret)); - } else { + } else if (OB_ISNULL(ctx)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "fd ctx is null!", K(flag), K(ret)); + } else if (flag == OB_STORAGE_ACCESS_READER) { ObStorageReader *reader = static_cast(ctx); - if (OB_ISNULL(reader)) { - ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "fd ctx is null!", K(flag), K(ret)); - } else if (OB_FAIL(reader->pread((char*)buf, size, offset, read_size))) { - OB_LOG(WARN, "fail to pread!", K(ret)); + if (OB_FAIL(reader->pread((char*)buf, size, offset, read_size))) { + OB_LOG(WARN, "fail to do normal pread!", K(ret), K(flag)); } + } else if (flag == OB_STORAGE_ACCESS_ADAPTIVE_READER) { + ObStorageAdaptiveReader *adaptive_reader = static_cast(ctx); + if (OB_FAIL(adaptive_reader->pread((char*)buf, size, offset, read_size))) { + OB_LOG(WARN, "fail to do adaptive reader pread!", K(ret), K(flag)); + } + } else { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "fd is not a reader fd!", K(flag), K(ret)); } return ret; } @@ -463,16 +626,24 @@ int ObObjectDevice::write(const ObIOFd &fd, const void *buf, const int64_t size, OB_LOG(WARN, "fd is not init!", K(fd.first_id_), K(fd.second_id_), K(ret)); } else if (OB_FAIL(fd_mng_.fd_to_ctx(fd, ctx))) { OB_LOG(WARN, "fail to get ctx accroding fd!", K(ret)); + } else if (OB_ISNULL(ctx)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "fd ctx is null!", K(flag), K(ret)); } else if (flag == OB_STORAGE_ACCESS_OVERWRITER) { - ObStorageWriter* overwriter = static_cast(ctx); + ObStorageWriter *overwriter = static_cast(ctx); if (OB_FAIL(overwriter->write((char*)buf, size))) { OB_LOG(WARN, "fail to do overwrite write!", K(ret)); } } else if (flag == OB_STORAGE_ACCESS_APPENDER) { - ObStorageAppender* appender = static_cast(ctx); + ObStorageAppender *appender = static_cast(ctx); if (OB_FAIL(appender->write((char*)buf, size))) { OB_LOG(WARN, "fail to do append write!", K(ret)); } + } else if (flag == OB_STORAGE_ACCESS_MULTIPART_WRITER) { + ObStorageMultiPartWriter *multipart_writer = static_cast(ctx); + if (OB_FAIL(multipart_writer->write((char*)buf, size))) { + OB_LOG(WARN, "fail to do multipart writer write!", K(ret)); + } } else { ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "unknow access type, not a writable type!", K(flag), K(ret)); @@ -505,11 +676,19 @@ int ObObjectDevice::pwrite(const ObIOFd &fd, const int64_t offset, const int64_t OB_LOG(WARN, "fd is not init!", K(fd.first_id_), K(fd.second_id_)); } else if (OB_FAIL(fd_mng_.fd_to_ctx(fd, ctx))) { OB_LOG(WARN, "fail to get ctx accroding fd!", K(ret)); + } else if (OB_ISNULL(ctx)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "fd ctx is null!", K(flag), K(ret)); } else if (flag == OB_STORAGE_ACCESS_RANDOMWRITER) { - ObStorageAppender* appender = static_cast(ctx); + ObStorageAppender *appender = static_cast(ctx); if (OB_FAIL(appender->pwrite((char*)buf, size, offset))) { OB_LOG(WARN, "fail to do appender pwrite!", K(ret)); } + } else if (flag == OB_STORAGE_ACCESS_MULTIPART_WRITER) { + ObStorageMultiPartWriter *multipart_writer = static_cast(ctx); + if (OB_FAIL(multipart_writer->pwrite((char*)buf, size, offset))) { + OB_LOG(WARN, "fail to do multipart writer pwrite!", K(ret)); + } } else { ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "unknow access type, not a writable type!", K(flag), K(ret)); @@ -550,9 +729,7 @@ int ObObjectDevice::reconfig(const ObIODOpts &opts) int ObObjectDevice::seal_file(const ObIOFd &fd) { - UNUSED(fd); - OB_LOG_RET(WARN, OB_NOT_SUPPORTED, "seal file is not support in object device !", K(fd)); - return OB_NOT_SUPPORTED; + return seal_for_adaptive(fd); } int ObObjectDevice::scan_dir(const char *dir_name, int (*func)(const dirent *entry)) diff --git a/deps/oblib/src/lib/restore/ob_object_device.h b/deps/oblib/src/lib/restore/ob_object_device.h index 01d7c19d1..2103882e7 100644 --- a/deps/oblib/src/lib/restore/ob_object_device.h +++ b/deps/oblib/src/lib/restore/ob_object_device.h @@ -73,15 +73,31 @@ public: //add new virtual int get_config(ObIODOpts &opts) override; + int del_unmerged_parts(const char *pathname); + int seal_for_adaptive(const ObIOFd &fd); + int adaptive_exist(const char *pathname, bool &is_exist); + int adaptive_stat(const char *pathname, ObIODFileStat &statbuf); + int adaptive_unlink(const char *pathname); + int adaptive_scan_dir(const char *dir_name, ObBaseDirEntryOperator &op); + public: common::ObFdSimulator& get_fd_mng() {return fd_mng_;} private: int get_access_type(ObIODOpts *opts, ObStorageAccessType& access_type); int open_for_reader(const char *pathname, void*& ctx); + int open_for_adaptive_reader_(const char *pathname, void *&ctx); int open_for_overwriter(const char *pathname, void*& ctx); int open_for_appender(const char *pathname, ObIODOpts *opts, void*& ctx); + int open_for_multipart_writer_(const char *pathname, void *&ctx); int release_res(void* ctx, const ObIOFd &fd, ObStorageAccessType access_type); + + int inner_exist_(const char *pathname, bool &is_exist, const bool is_adaptive = false); + int inner_stat_(const char *pathname, ObIODFileStat &statbuf, const bool is_adaptive = false); + int inner_unlink_(const char *pathname, const bool is_adaptive = false); + int inner_scan_dir_(const char *dir_name, + ObBaseDirEntryOperator &op, const bool is_adaptive = false); + private: //maybe fd mng can be device level common::ObFdSimulator fd_mng_; @@ -89,8 +105,10 @@ private: ObStorageUtil util_; /*obj ctx pool: use to create fd ctx(reader/writer)*/ common::ObPooledAllocator reader_ctx_pool_; + common::ObPooledAllocator adaptive_reader_ctx_pool_; common::ObPooledAllocator appender_ctx_pool_; common::ObPooledAllocator overwriter_ctx_pool_; + common::ObPooledAllocator multipart_writer_ctx_pool_; common::ObObjectStorageInfo storage_info_; bool is_started_; char storage_info_str_[OB_MAX_URI_LENGTH]; diff --git a/deps/oblib/src/lib/restore/ob_storage.cpp b/deps/oblib/src/lib/restore/ob_storage.cpp index 5ee9e3c02..8182e1329 100644 --- a/deps/oblib/src/lib/restore/ob_storage.cpp +++ b/deps/oblib/src/lib/restore/ob_storage.cpp @@ -24,7 +24,7 @@ namespace oceanbase namespace common { -const char *OB_STORAGE_TYPES_STR[] = {"OSS", "FILE", "COS", "LOCAL"}; +const char *OB_STORAGE_TYPES_STR[] = {"OSS", "FILE", "COS", "LOCAL", "S3"}; void print_access_storage_log( const char *msg, @@ -57,6 +57,7 @@ int validate_uri_type(const common::ObString &uri) int ret = OB_SUCCESS; if (!uri.prefix_match(OB_OSS_PREFIX) && !uri.prefix_match(OB_COS_PREFIX) && + !uri.prefix_match(OB_S3_PREFIX) && !uri.prefix_match(OB_FILE_PREFIX)) { ret = OB_INVALID_BACKUP_DEST; STORAGE_LOG(ERROR, "invalid backup uri", K(ret), K(uri)); @@ -73,11 +74,13 @@ int get_storage_type_from_path(const common::ObString &uri, ObStorageType &type) type = OB_STORAGE_OSS; } else if (uri.prefix_match(OB_COS_PREFIX)) { type = OB_STORAGE_COS; + } else if (uri.prefix_match(OB_S3_PREFIX)) { + type = OB_STORAGE_S3; } else if (uri.prefix_match(OB_FILE_PREFIX)) { type = OB_STORAGE_FILE; } else { ret = OB_INVALID_BACKUP_DEST; - STORAGE_LOG(ERROR, "invlaid backup uri", K(ret), K(uri)); + STORAGE_LOG(ERROR, "invalid backup uri", K(ret), K(uri)); } return ret; } @@ -94,7 +97,7 @@ const char *get_storage_type_str(const ObStorageType &type) bool is_io_error(const int result) { - return OB_IO_ERROR == result || OB_OSS_ERROR == result || OB_COS_ERROR == result; + return OB_IO_ERROR == result || OB_OSS_ERROR == result || OB_COS_ERROR == result || OB_S3_ERROR == result; } int get_storage_type_from_name(const char *type_str, ObStorageType &type) @@ -112,14 +115,144 @@ int get_storage_type_from_name(const char *type_str, ObStorageType &type) return ret; } +// let path end with '/' +int build_full_dir_path(const char *dir_path, char *dir_path_buf, const int64_t dir_path_buf_len) +{ + int ret = OB_SUCCESS; + int64_t pos = 0; + if (OB_ISNULL(dir_path) || OB_ISNULL(dir_path_buf) || + OB_UNLIKELY(strlen(dir_path) <= 0 || dir_path_buf_len <= 0)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid arguments", K(ret), K(dir_path), KP(dir_path_buf), K(dir_path_buf_len)); + } else if (OB_FAIL(databuff_printf(dir_path_buf, dir_path_buf_len, pos, "%s", dir_path))) { + OB_LOG(WARN, "failed to deep copy dir path", K(ret), K(dir_path), K(dir_path_buf_len)); + } else if (!is_end_with_slash(dir_path) + && OB_FAIL(databuff_printf(dir_path_buf, dir_path_buf_len, pos, "/"))) { + OB_LOG(WARN, "failed to make path end with '/'", K(ret), K(dir_path), K(dir_path_buf_len)); + } + return ret; +} + /** - * ------------------------------ObStorageGlobalIns--------------------- + * ------------------------------ListAppendableObjectFragmentOp------------------------------ + */ +int ListAppendableObjectFragmentOp::func(const dirent *entry) +{ + int ret = OB_SUCCESS; + ObString fragment_name(entry->d_name); + ObAppendableFragmentMeta fragment_meta; + if (OB_FAIL(fragment_meta.parse_from(fragment_name))) { + OB_LOG(WARN, "fail to parse fragment name", K(ret), K(entry->d_name)); + } else if (OB_LIKELY(fragment_meta.is_valid())) { + if (fragment_meta.is_format_meta()) { + exist_format_meta_ = true; + } else if (fragment_meta.is_seal_meta()) { + exist_seal_meta_ = true; + } else if (fragment_meta.is_data()) { + if (need_get_file_size()) { + if (get_size() != fragment_meta.get_length()) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "fragment file size is not equal to the fragment name range", K(ret), "file_size", + get_size(), K(fragment_meta)); + } + } + + if (FAILEDx(meta_arr_.push_back(fragment_meta))) { + OB_LOG(WARN, "fail to push back fragment meta", K(ret), K(fragment_meta)); + } + } else { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "unknown fragment meta type", K(ret), K(fragment_meta)); + } + } else { + // just skip this invalid fragment, continue next one. + OB_LOG(WARN, "invalid fragment name", K(ret), K(entry->d_name)); + } + return ret; +} + +int ListAppendableObjectFragmentOp::gen_object_meta(ObStorageObjectMeta &obj_meta) +{ + int ret = OB_SUCCESS; + int64_t i = 0; + int64_t last_end_offset = 0; + obj_meta.reset(); + obj_meta.type_ = ObStorageObjectType::OB_OBJ_SIMULATE_APPEND; + std::sort(meta_arr_.begin(), meta_arr_.end(), ObStorageObjectMeta::fragment_meta_cmp_func); + while (OB_SUCC(ret) && i < meta_arr_.size()) { + ObAppendableFragmentMeta &tmp_meta = meta_arr_[i]; + last_end_offset = tmp_meta.end_; + if (OB_FAIL(obj_meta.fragment_metas_.push_back(tmp_meta))) { + OB_LOG(WARN, "failed to push back fragment meta", K(ret), K(obj_meta.fragment_metas_)); + } + + // find a fragment that intersects with the current fragment(and not covered by the current fragment) + // and can maximize the coverage of the file contents + int64_t j = i + 1; + int64_t selected_idx = i; + int64_t max_end_offset = tmp_meta.end_; + while (OB_SUCC(ret) && j < meta_arr_.size() && meta_arr_[j].start_ <= tmp_meta.end_) { + if (meta_arr_[j].end_ > max_end_offset) { + selected_idx = j; + max_end_offset = meta_arr_[j].end_; + } + j++; + } + + if (OB_FAIL(ret)) { + } else if (selected_idx == i) { + if (i < meta_arr_.size() - 1) { + // do not return error + OB_LOG(WARN, "cannot find a fragment that intersects with the current fragment", + K(i), K(j), K(tmp_meta)); + } + i = j; + } else { + i = selected_idx; + } + } + + if (OB_SUCC(ret)) { + obj_meta.is_exist_ = true; + obj_meta.length_ = last_end_offset; + } + return ret; +} + +/** + * ------------------------------DelAppendableObjectFragmentOp------------------------------ + */ +DelAppendableObjectFragmentOp::DelAppendableObjectFragmentOp( + const common::ObString &uri, ObStorageUtil &util) + : uri_(uri), util_(util) +{} + +int DelAppendableObjectFragmentOp::func(const dirent *entry) +{ + int ret = OB_SUCCESS; + char obj_fragment_path[OB_MAX_URI_LENGTH] = {0}; + const char *delemiter_str = is_end_with_slash(uri_.ptr()) ? "" : "/"; + + if (OB_ISNULL(entry)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid argument", K(ret)); + } else if (OB_FAIL(databuff_printf(obj_fragment_path, sizeof(obj_fragment_path), + "%s%s%s", uri_.ptr(), delemiter_str, entry->d_name))) { + OB_LOG(WARN, "fail to construct appendable obj fragment path", K(ret), K_(uri), K(delemiter_str), K(entry->d_name)); + } else if (OB_FAIL(util_.del_file(obj_fragment_path))) { + OB_LOG(WARN, "fail to del appendable object fragment", K(ret), K(obj_fragment_path)); + } else { + OB_LOG(DEBUG, "succ to del appendable object fragment", K(obj_fragment_path)); + } + return ret; +} + +/** + * ------------------------------ObStorageGlobalIns------------------------------ */ ObStorageGlobalIns::ObStorageGlobalIns() :io_prohibited_(false) -{ - -} +{} ObStorageGlobalIns& ObStorageGlobalIns::get_instance() { @@ -152,9 +285,8 @@ bool ObStorageGlobalIns::is_io_prohibited() const return io_prohibited_; } - /** - * ------------------------------ObExternalIOCounter--------------------- + * ------------------------------ObExternalIOCounter------------------------------ */ int64_t ObExternalIOCounter::flying_io_cnt_ = 0; @@ -174,7 +306,7 @@ int64_t ObExternalIOCounter::get_flying_io_cnt() } /** - * ------------------------------ObExternalIOCounterGuard--------------------- + * ------------------------------ObExternalIOCounterGuard------------------------------ */ ObExternalIOCounterGuard::ObExternalIOCounterGuard() { @@ -186,14 +318,18 @@ ObExternalIOCounterGuard::~ObExternalIOCounterGuard() io_counter_.dec_flying_io(); } - +/** + * ------------------------------ObStorageUtil------------------------------ + */ ObStorageUtil::ObStorageUtil() : file_util_(), oss_util_(), cos_util_(), + s3_util_(), util_(NULL), storage_info_(NULL), - init_state(false) + init_state(false), + device_type_(ObStorageType::OB_STORAGE_MAX_TYPE) { } @@ -201,7 +337,6 @@ ObStorageUtil::ObStorageUtil() int ObStorageUtil::open(common::ObObjectStorageInfo *storage_info) { int ret = OB_SUCCESS; - ObStorageType device_type = OB_STORAGE_MAX_TYPE; if (is_init()) { ret = OB_INIT_TWICE; @@ -209,21 +344,23 @@ int ObStorageUtil::open(common::ObObjectStorageInfo *storage_info) } else if (OB_ISNULL(storage_info) || OB_UNLIKELY(!storage_info->is_valid())) { ret = OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "invalid argument", K(ret), KP(storage_info)); - } else if (OB_FALSE_IT(device_type = storage_info->get_type())) { - } else if (OB_STORAGE_OSS == device_type) { + } else if (OB_FALSE_IT(device_type_ = storage_info->get_type())) { + } else if (OB_STORAGE_OSS == device_type_) { util_ = &oss_util_; - } else if (OB_STORAGE_COS == device_type) { + } else if (OB_STORAGE_COS == device_type_) { util_ = &cos_util_; - } else if (OB_STORAGE_FILE == device_type) { + } else if (OB_STORAGE_S3 == device_type_) { + util_ = &s3_util_; + } else if (OB_STORAGE_FILE == device_type_) { util_ = &file_util_; } else { ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid device type", K(ret), K(device_type)); + STORAGE_LOG(WARN, "Invalid device type", K(ret), K_(device_type)); } if (OB_SUCC(ret) && NULL != util_) { if (OB_FAIL(util_->open(storage_info))) { - STORAGE_LOG(WARN, "failed to open util", K(ret), K(device_type)); + STORAGE_LOG(WARN, "failed to open util", K(ret), K_(device_type)); util_ = NULL; } else { storage_info_ = storage_info; @@ -240,9 +377,600 @@ void ObStorageUtil::close() util_ = NULL; storage_info_ = NULL; init_state = false; + device_type_ = OB_STORAGE_MAX_TYPE; } } +/*---------------------------------- NEW ADAPTIVE INTERFACE ----------------------------------*/ +int ObStorageUtil::detect_storage_obj_meta( + const common::ObString &uri, + const bool is_adaptive, + const bool need_fragment_meta, + ObStorageObjectMeta &obj_meta) +{ + int ret = OB_SUCCESS; + obj_meta.reset(); + bool need_detect_appendable = is_adaptive; + if (OB_UNLIKELY(!is_init())) { + ret = OB_NOT_INIT; + STORAGE_LOG(WARN, "util is not inited", K(ret), K(uri)); + } else if (ObStorageGlobalIns::get_instance().is_io_prohibited()) { + ret = OB_BACKUP_IO_PROHIBITED; + STORAGE_LOG(WARN, "current observer backup io is prohibited", K(ret), K(uri)); + } else if (OB_FAIL(util_->head_object_meta(uri, obj_meta))) { + STORAGE_LOG(WARN, "fail to head object meta", K(ret), K(uri)); + } else if (obj_meta.is_exist_) { + if (ObStorageObjectType::OB_FS_DIR != obj_meta.type_) { + // just return directly + need_detect_appendable = false; + obj_meta.type_ = ObStorageObjectType::OB_OBJ_NORMAL; + } + } + + if (OB_SUCC(ret) && need_detect_appendable) { + obj_meta.reset(); + int64_t pos = 0; + // if need_fragment_meta is TRUE, we need to obtain fragment meta from seal_meta or list_info + if (need_fragment_meta) { + if (OB_FAIL(read_seal_meta_if_needed(uri, obj_meta))) { + OB_LOG(WARN, "fail to read seal meta if needed", K(ret), K(uri)); + } + + // If not exist seal meta, we need to list all files and check format meta exist or not. + // format meta must exist, otherwise need to return error. + if (OB_SUCC(ret) && !obj_meta.is_exist_) { + if (OB_FAIL(list_appendable_file_fragments(uri, obj_meta))) { + OB_LOG(WARN, "fail to list appendable file fragments", K(ret), K(uri)); + } + } + } else { + // if need_fragment_meta is FALSE, just check format_meta exist or not + char format_meta_uri[OB_MAX_URI_LENGTH] = { 0 }; + if (OB_FAIL(construct_fragment_full_name(uri, OB_S3_APPENDABLE_FORMAT_META, + format_meta_uri, sizeof(format_meta_uri)))) { + OB_LOG(WARN, "fail to construct s3 format_meta name", K(ret), K(uri)); + } else if (OB_FAIL(util_->head_object_meta(format_meta_uri, obj_meta))) { + STORAGE_LOG(WARN, "fail to head object meta", K(ret), K(format_meta_uri)); + } else if (obj_meta.is_exist_) { + obj_meta.type_ = ObStorageObjectType::OB_OBJ_SIMULATE_APPEND; + } else { + if (ObStorageObjectType::OB_FS_DIR == obj_meta.type_) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "please set is_adaptive as false when want to check a common dir exist or not", + K(ret), K(uri), K(is_adaptive)); + } + } + } + } + return ret; +} + +int ObStorageUtil::read_seal_meta_if_needed( + const common::ObString &uri, + ObStorageObjectMeta &obj_meta) +{ + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + ObArenaAllocator allocator(APPENDABLE_OBJECT_ALLOCATOR); + int64_t pos = 0; + char *seal_meta_uri = NULL; + + if (OB_ISNULL(seal_meta_uri = static_cast(allocator.alloc(OB_MAX_URI_LENGTH)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "fail to alloc buf for reading seal meta file", K(ret)); + } else if (FALSE_IT(memset(seal_meta_uri, 0, OB_MAX_URI_LENGTH))) { + } else if (OB_FAIL(construct_fragment_full_name(uri, OB_S3_APPENDABLE_SEAL_META, + seal_meta_uri, OB_MAX_URI_LENGTH))) { + OB_LOG(WARN, "fail to construct s3 seal_meta name", K(ret), K(uri)); + } else { + ObStorageReader reader; + int64_t seal_meta_len = 0; + if (OB_FAIL(reader.open(seal_meta_uri, storage_info_))) { + if (OB_BACKUP_FILE_NOT_EXIST == ret) { + obj_meta.is_exist_ = false; + ret = OB_SUCCESS; + } + } else { + // If exist seal meta, directly read it content. + seal_meta_len = reader.get_length(); + if (seal_meta_len > 0) { + int64_t read_size = 0; + char *buf = NULL; + pos = 0; + if (OB_ISNULL(buf = static_cast(allocator.alloc(seal_meta_len + 1)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "fail to alloc buf for reading seal meta file", K(ret), K(seal_meta_uri), K(seal_meta_len)); + } else if (OB_FAIL(reader.pread(buf, seal_meta_len, 0/*offset*/, read_size))) { + OB_LOG(WARN, "failed to read seal meta file content", K(ret), K(seal_meta_uri), K(seal_meta_len)); + } else if (OB_UNLIKELY(seal_meta_len != read_size)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "fail to read seal meta file entire content", + K(ret), K(seal_meta_uri), K(seal_meta_len), K(read_size)); + } else if (OB_FAIL(obj_meta.deserialize(buf, read_size, pos))) { + OB_LOG(WARN, "fail to deserialize storage object meta", K(ret), K(seal_meta_uri), K(read_size), KP(buf)); + } else { + obj_meta.is_exist_ = true; + } + } else { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "the seal meta file is empty", K(ret), K(seal_meta_uri)); + } + + if (OB_TMP_FAIL(reader.close())) { + OB_LOG(WARN, "fail to close reader", K(ret), K(tmp_ret), K(seal_meta_uri)); + } + } + } + return ret; +} + +int ObStorageUtil::list_appendable_file_fragments( + const ObString &uri, + ObStorageObjectMeta &obj_meta) +{ + int ret = OB_SUCCESS; + int64_t appendable_file_length = -1; + ListAppendableObjectFragmentOp op; + obj_meta.reset(); + if (OB_FAIL(list_files(uri, false, op))) { + OB_LOG(WARN, "failed to list appendable object fragments", K(ret), K(uri)); + } else if (OB_UNLIKELY(!op.exist_format_meta())) { + ret = OB_BACKUP_FILE_NOT_EXIST; + OB_LOG(WARN, "not exist valid appendable file, cuz lack of format_meta", K(ret), K(uri)); + } else if (FALSE_IT(obj_meta.is_exist_ = true)) { + } else if (FALSE_IT(obj_meta.type_ = ObStorageObjectType::OB_OBJ_SIMULATE_APPEND)) { + } else if (OB_FAIL(op.gen_object_meta(obj_meta))) { + OB_LOG(WARN, "fail to gen object meta based on the listed meta info", K(ret), K(uri)); + } + return ret; +} + +int ObStorageUtil::is_exist(const common::ObString &uri, const bool is_adaptive, bool &exist) +{ + int ret = OB_SUCCESS; + const int64_t start_ts = ObTimeUtility::current_time(); + exist = false; + ObStorageObjectMeta obj_meta; + +#ifdef ERRSIM + ret = OB_E(EventTable::EN_BACKUP_IO_IS_EXIST) OB_SUCCESS; +#endif + if (OB_FAIL(ret)) { + //do nothing + } else if (OB_FAIL(validate_uri_type(uri))) { + OB_LOG(WARN, "fail to validate uri!", K(ret), K(uri)); + } else if (OB_FAIL(detect_storage_obj_meta(uri, is_adaptive, false/*need_fragment_meta*/, obj_meta))) { + OB_LOG(WARN, "fail to detect storage obj type", K(ret), K(uri), K(is_adaptive)); + } else { + exist = obj_meta.is_exist_; + } + + print_access_storage_log("is_exist", uri, start_ts); + return ret; +} + +int ObStorageUtil::get_file_length(const common::ObString &uri, const bool is_adaptive, int64_t &file_length) +{ + int ret = OB_SUCCESS; + const int64_t start_ts = ObTimeUtility::current_time(); + file_length = -1; + ObStorageObjectMeta obj_meta; + +#ifdef ERRSIM + ret = OB_E(EventTable::EN_BACKUP_IO_GET_FILE_LENGTH) OB_SUCCESS; +#endif + if (OB_FAIL(ret)) { + } else if (OB_FAIL(validate_uri_type(uri))) { + OB_LOG(WARN, "fail to validate uri!", K(ret), K(uri)); + } else if (OB_FAIL(detect_storage_obj_meta(uri, is_adaptive, true/*need_fragment_meta*/, obj_meta))) { + OB_LOG(WARN, "fail to detect storage obj type", K(ret), K(uri), K(is_adaptive)); + } else if (!obj_meta.is_exist_) { + ret = OB_BACKUP_FILE_NOT_EXIST; + OB_LOG(INFO, "cannot get file length for not exist file", K(ret), K(uri)); + } else { + file_length = obj_meta.length_; + } + + if (OB_SUCC(ret)) { + if (file_length == 0) { + OB_LOG(INFO, "this file is empty", K(ret), K(uri), K(file_length)); + } else if (file_length < 0) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "this file length is invalid", K(ret), K(uri), K(file_length)); + } + } + + print_access_storage_log("get_file_length", uri, start_ts); + return ret; +} + +int ObStorageUtil::del_file(const common::ObString &uri, const bool is_adaptive) +{ + int ret = OB_SUCCESS; + const int64_t start_ts = ObTimeUtility::current_time(); + ObStorageObjectMeta obj_meta; + +#ifdef ERRSIM + ret = OB_E(EventTable::EN_BACKUP_IO_BEFORE_DEL_FILE) OB_SUCCESS; +#endif + if (OB_FAIL(ret)) { + } else if (OB_FAIL(validate_uri_type(uri))) { + STORAGE_LOG(WARN, "fail to validate uri!", K(ret), K(uri)); + } else if (OB_FAIL(detect_storage_obj_meta(uri, is_adaptive, false/*need_fragment_meta*/, obj_meta))) { + OB_LOG(WARN, "fail to detect storage obj type", K(ret), K(uri), K(is_adaptive)); + } else if (obj_meta.is_simulate_append_type()) { + if (OB_FAIL(del_appendable_file(uri))) { + OB_LOG(WARN, "fail to delete appendable file", K(ret), K(uri)); + } + } else { + const int max_try_cnt = 5; + int try_cnt = 0; + while (try_cnt < max_try_cnt) { + if (OB_FAIL(util_->del_file(uri))) { + try_cnt++; + OB_LOG(WARN, "fail to delete file", K(ret), K(try_cnt), K(uri)); + } else { + break; + } + } + } + + if (OB_SUCC(ret)) { + OB_LOG(DEBUG, "succ to delete file", K(ret), K(uri)); + } else { + OB_LOG(WARN, "fail to delete file", K(ret), K(uri)); + } + +#ifdef ERRSIM + if (OB_SUCC(ret)) { + ret = OB_E(EventTable::EN_BACKUP_IO_AFTER_DEL_FILE) OB_SUCCESS; + } +#endif + print_access_storage_log("del_file", uri, start_ts); + + if (OB_FAIL(ret)) { + EVENT_INC(ObStatEventIds::BACKUP_IO_DEL_FAIL_COUNT); + } + EVENT_INC(ObStatEventIds::BACKUP_DELETE_COUNT); + EVENT_ADD(ObStatEventIds::BACKUP_DELETE_DELAY, ObTimeUtility::current_time() - start_ts); + return ret; +} + +int ObStorageUtil::del_appendable_file(const ObString &uri) +{ + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + if (OB_UNLIKELY(!is_init())) { + ret = OB_NOT_INIT; + STORAGE_LOG(WARN, "util is not inited", K(ret), K(uri)); + } else if (ObStorageGlobalIns::get_instance().is_io_prohibited()) { + ret = OB_BACKUP_IO_PROHIBITED; + STORAGE_LOG(WARN, "current observer backup io is prohibited", K(ret), K(uri)); + } else if (OB_FAIL(validate_uri_type(uri))) { + STORAGE_LOG(WARN, "fail to validate uri!", K(ret), K(uri)); + } else { + DelAppendableObjectFragmentOp op(uri, *this); + if (OB_FAIL(list_files(uri, false/*is_adaptive*/, op))) { + OB_LOG(WARN, "fail to list appendable object fragments", K(ret), K(uri)); + } + } + return ret; +} + +int ObStorageUtil::list_files( + const common::ObString &uri, + const bool is_adaptive, + common::ObBaseDirEntryOperator &op) +{ + int ret = OB_SUCCESS; + const int64_t start_ts = ObTimeUtility::current_time(); + char uri_buf[OB_MAX_URI_LENGTH] = {0}; + + if (OB_UNLIKELY(!is_init())) { + ret = OB_NOT_INIT; + STORAGE_LOG(WARN, "util is not inited", K(ret), K(uri)); + } else if (ObStorageGlobalIns::get_instance().is_io_prohibited()) { + ret = OB_BACKUP_IO_PROHIBITED; + STORAGE_LOG(WARN, "current observer backup io is prohibited", K(ret), K(uri)); + } else if (OB_FAIL(validate_uri_type(uri))) { + STORAGE_LOG(WARN, "fail to validate uri!", K(ret), K(uri)); + } else if (OB_FAIL(build_full_dir_path(uri.ptr(), uri_buf, sizeof(uri_buf)))) { + OB_LOG(WARN, "fail to make uri end with '/'", K(ret), K(uri)); + } else if (is_adaptive && OB_FAIL(list_adaptive_files(uri_buf, op))) { + STORAGE_LOG(WARN, "fail to list adaptive files", K(ret), K(uri)); + } else if (!is_adaptive && OB_FAIL(util_->list_files(uri_buf, op))) { + STORAGE_LOG(WARN, "fail to list files", K(ret), K(uri)); + } + + print_access_storage_log("list_files", uri, start_ts, 0); + if (OB_FAIL(ret)) { + EVENT_INC(ObStatEventIds::BACKUP_IO_LS_FAIL_COUNT); + } + + EVENT_INC(ObStatEventIds::BACKUP_IO_LS_COUNT); + + return ret; +} + +int ObStorageUtil::list_adaptive_files( + const common::ObString &uri, + common::ObBaseDirEntryOperator &op) +{ + int ret = OB_SUCCESS; + + ObArenaAllocator allocator(APPENDABLE_OBJECT_ALLOCATOR); + ObStorageListObjectsCtx list_obj_ctx; + ObStorageListFilesCtx list_file_ctx; + ObStorageListCtxBase *list_ctx = NULL; + bool is_obj_storage = (OB_STORAGE_FILE != device_type_); + const bool need_get_size = op.need_get_file_size(); + + if (is_obj_storage) { + if (OB_FAIL(list_obj_ctx.init(allocator, OB_STORAGE_LIST_MAX_NUM, need_get_size))) { + OB_LOG(WARN, "fail to init list_obj_ctx", K(ret), K(need_get_size)); + } else { + list_ctx = &list_obj_ctx; + } + } else { + if (OB_FAIL(list_file_ctx.init(allocator, OB_STORAGE_LIST_MAX_NUM, need_get_size))) { + OB_LOG(WARN, "fail to init list_file_ctx", K(ret), K(need_get_size)); + } else { + list_ctx = &list_file_ctx; + } + } + + ObString bucket; + ObString dir_path; + if (OB_SUCC(ret)) { + if (OB_FAIL(build_bucket_and_object_name(allocator, uri, bucket, dir_path))) { + OB_LOG(WARN, "fail to build bucket and object name", K(ret), K(uri)); + } + } + + if (OB_SUCC(ret)) { + do { + list_ctx->rsp_num_ = 0; + list_ctx->has_next_ = false; + if (OB_FAIL(util_->list_files(uri, *list_ctx))) { + OB_LOG(WARN, "fail to list files", K(ret), K(uri), K(list_ctx)); + } else if (list_ctx->rsp_num_ > 0) { + if (is_obj_storage) { + if (OB_FAIL(handle_listed_objs(list_ctx, uri, dir_path, op))) { + OB_LOG(WARN, "fail to handle listed objs", K(ret), K(uri), K(dir_path), K(list_ctx)); + } + } else { + if (OB_FAIL(handle_listed_fs(list_ctx, uri, dir_path, op))) { + OB_LOG(WARN, "fail to handle listed fs", K(ret), K(uri), K(dir_path), K(list_ctx)); + } + } + } + } while (OB_SUCC(ret) && list_ctx->has_next_); + } + return ret; +} + +int ObStorageUtil::handle_listed_objs( + ObStorageListCtxBase *ctx_base, + const common::ObString &uri, + const common::ObString &dir_path, + common::ObBaseDirEntryOperator &op) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(ctx_base)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid argument", K(ret)); + } else { + ObStorageListObjectsCtx *list_ctx = static_cast(ctx_base); + int64_t dir_path_len = dir_path.length(); + int64_t full_dir_path_len = (is_end_with_slash(dir_path.ptr()) ? dir_path_len : (dir_path_len + 1)); + + char cur_appendable_full_path[OB_MAX_URI_LENGTH] = {0}; + bool already_exist_fragment = (strlen(list_ctx->cur_appendable_full_obj_path_) > 0); + for (int64_t i = 0; OB_SUCC(ret) && (i < list_ctx->rsp_num_); ++i) { + ObString cur_obj_path(strlen(list_ctx->name_arr_[i]), list_ctx->name_arr_[i]); + char *contain_idx_ptr = strstr(cur_obj_path.ptr(), OB_S3_APPENDABLE_FRAGMENT_PREFIX); + bool is_fragment_obj = (nullptr != contain_idx_ptr); + if (is_fragment_obj) { + const int64_t appendable_full_path_len = (contain_idx_ptr - cur_obj_path.ptr()); + if (OB_UNLIKELY(appendable_full_path_len <= full_dir_path_len || + appendable_full_path_len >= OB_MAX_URI_LENGTH)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "appendable obj full path len is invalid", K(ret), K(appendable_full_path_len), + K(full_dir_path_len), K(OB_MAX_URI_LENGTH), K(dir_path), K(cur_obj_path)); + } else { + if (!already_exist_fragment) { + MEMCPY(list_ctx->cur_appendable_full_obj_path_, cur_obj_path.ptr(), appendable_full_path_len); + list_ctx->cur_appendable_full_obj_path_[appendable_full_path_len] = '\0'; + already_exist_fragment = true; + } else { + // check current fragment's parent dir is same with the previous one + // if not, we need to handle previous one, and then save current one into list_ctx + MEMCPY(cur_appendable_full_path, cur_obj_path.ptr(), appendable_full_path_len); + cur_appendable_full_path[appendable_full_path_len] = '\0'; + if (0 != STRCMP(list_ctx->cur_appendable_full_obj_path_, cur_appendable_full_path)) { + if (OB_FAIL(handle_listed_appendable_obj(list_ctx, uri, dir_path, op))) { + OB_LOG(WARN, "fail to handle listed appendable obj", K(ret), K(uri), K(dir_path), + K(cur_appendable_full_path), K(list_ctx->cur_appendable_full_obj_path_)); + } else { + MEMCPY(list_ctx->cur_appendable_full_obj_path_, cur_obj_path.ptr(), appendable_full_path_len); + list_ctx->cur_appendable_full_obj_path_[appendable_full_path_len] = '\0'; + } + } + } + } + } else { + // use @op to handle previous appendable object name + if (already_exist_fragment) { + if (OB_FAIL(handle_listed_appendable_obj(list_ctx, uri, dir_path, op))) { + OB_LOG(WARN, "fail to handle listed appendable obj", K(ret), K(uri), K(dir_path)); + } else { + already_exist_fragment = false; + } + } + + // use @op to handle current normal object name + if (OB_SUCC(ret)) { + const int64_t size = op.need_get_file_size() ? list_ctx->size_arr_[i] : -1; + if (OB_FAIL(handle_listed_object(op, cur_obj_path.ptr() + full_dir_path_len, + cur_obj_path.length() - full_dir_path_len, + size))) { + OB_LOG(WARN, "fail to handle listed object", K(ret), K(cur_obj_path), + K(cur_obj_path.length()), K(full_dir_path_len), K(dir_path), K(size)); + } + } + } + } + + if (OB_SUCC(ret)) { + if (already_exist_fragment && strlen(list_ctx->cur_appendable_full_obj_path_) > 0 && !list_ctx->has_next_) { + if (OB_FAIL(handle_listed_appendable_obj(list_ctx, uri, dir_path, op))) { + OB_LOG(WARN, "fail to handle listed appendable obj", K(ret), K(uri), K(dir_path)); + } else { + already_exist_fragment = false; + } + } + } + } + return ret; +} + +int ObStorageUtil::handle_listed_appendable_obj( + ObStorageListObjectsCtx *list_ctx, + const common::ObString &uri, + const common::ObString &dir_path, + common::ObBaseDirEntryOperator &op) +{ + int ret = OB_SUCCESS; + bool need_handle_file = true; + int64_t dir_path_len = dir_path.length(); + int64_t full_dir_path_len = (is_end_with_slash(dir_path.ptr()) ? dir_path_len : (dir_path_len + 1)); + const int64_t appendable_full_path_len = strlen(list_ctx->cur_appendable_full_obj_path_); + int64_t appendable_file_len = 0; + if (op.need_get_file_size()) { + char append_obj_uri[OB_MAX_URI_LENGTH] = {0}; + // uri is equal to "prefix + bucket + '/' + dir_path" + const int64_t uri_prefix_len = uri.length() - dir_path_len; + MEMCPY(append_obj_uri, uri.ptr(), uri_prefix_len); // now append_obj_uri == prefix + bucket + '/' + // cur_appendable_full_obj_path_ == full_dir_path + logic_apendable_obj_name + '/' + // not copy the last character '/' + MEMCPY(append_obj_uri + uri_prefix_len, list_ctx->cur_appendable_full_obj_path_, appendable_full_path_len - 1); + append_obj_uri[uri_prefix_len + appendable_full_path_len - 1] = '\0'; + + // after above operation, append_obj_uri is equal to "prefix + bucket + '/' + full_dir_path + apendable_obj_name" + + if (OB_FAIL(get_adaptive_file_length(append_obj_uri, appendable_file_len))) { + OB_LOG(WARN, "fail to get adaptive file length", K(ret), KP(append_obj_uri)); + } else if (appendable_file_len == 0) { + // For appendable file, if its length is 0, we think that this file should not exist in the listed result, + // thus we will not handle this file. + need_handle_file = false; + } else if (appendable_file_len < 0) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "appendable file len should not be less than 0", + K(ret), K(appendable_file_len), K(append_obj_uri)); + } + } + + if (OB_SUCC(ret) && need_handle_file) { + if (OB_FAIL(handle_listed_object(op, list_ctx->cur_appendable_full_obj_path_ + full_dir_path_len, + appendable_full_path_len - full_dir_path_len - 1, + appendable_file_len))) { + OB_LOG(WARN, "fail to handle listed object", + K(ret), K(full_dir_path_len), K(appendable_full_path_len)); + } else { + list_ctx->cur_appendable_full_obj_path_[0] = '\0'; + } + } + return ret; +} + +int ObStorageUtil::handle_listed_fs( + ObStorageListCtxBase *ctx_base, + const common::ObString &uri, + const common::ObString &dir_path, + common::ObBaseDirEntryOperator &op) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(ctx_base)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid argument", K(ret)); + } else { + ObStorageListFilesCtx *list_ctx = static_cast(ctx_base); + const bool is_slash_end = is_end_with_slash(uri.ptr()); + const char *slash_delimiter = is_slash_end ? "" : "/"; + char tmp_uri[OB_MAX_URI_LENGTH] = { 0 }; + + int64_t size = 0; + // When handling file system list_files, the result will only save file's name(not include dir path) + for (int64_t i = 0; OB_SUCC(ret) && (i < list_ctx->rsp_num_); ++i) { + const int64_t name_len = strlen(list_ctx->name_arr_[i]); + // if the file's suffix is '/', that means this is a 'appendable' file. otherwise it is a normal file + if (list_ctx->name_arr_[i][name_len - 1] == '/') { + list_ctx->name_arr_[i][name_len - 1] = '\0'; + if (op.need_get_file_size()) { + if (OB_FAIL(databuff_printf(tmp_uri, OB_MAX_URI_LENGTH, "%s%s%s", uri.ptr(), slash_delimiter, + list_ctx->name_arr_[i]))) { + STORAGE_LOG(WARN, "fail to build appendable file uri", K(ret), K(i), K(uri), K(is_slash_end), + KP(list_ctx->name_arr_[i])); + } else if (OB_FAIL(get_adaptive_file_length(tmp_uri, size))) { + OB_LOG(WARN, "fail to get adaptive file length", K(ret), KP(tmp_uri)); + } + } + } else { + if (op.need_get_file_size()) { + size = list_ctx->size_arr_[i]; + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(handle_listed_object(op, list_ctx->name_arr_[i], + strlen(list_ctx->name_arr_[i]), + size))) { + OB_LOG(WARN, "fail to handle listed object", + K(ret), K(list_ctx->name_arr_[i]), K(dir_path), K(size)); + } + } + } + } + return ret; +} + +int ObStorageUtil::list_directories( + const common::ObString &uri, + const bool is_adaptive, + common::ObBaseDirEntryOperator &op) +{ + int ret = OB_SUCCESS; + char uri_buf[OB_MAX_URI_LENGTH] = {0}; + const int64_t start_ts = ObTimeUtility::current_time(); + +#ifdef ERRSIM + ret = OB_E(EventTable::EN_BACKUP_IO_LIST_FILE) OB_SUCCESS; +#endif + + if (OB_FAIL(ret)) { + } else if (OB_UNLIKELY(is_adaptive)) { + ret = OB_NOT_SUPPORTED; + OB_LOG(WARN, "not support list directories for adaptive mode", K(ret), K(uri), K(is_adaptive)); + } else if (ObStorageGlobalIns::get_instance().is_io_prohibited()) { + ret = OB_BACKUP_IO_PROHIBITED; + STORAGE_LOG(WARN, "current observer backup io is prohibited", K(ret), K(uri)); + } else if (OB_UNLIKELY(!is_init())) { + ret = OB_NOT_INIT; + STORAGE_LOG(WARN, "util is not inited", K(ret), K(uri)); + } else if (OB_FAIL(validate_uri_type(uri))) { + STORAGE_LOG(WARN, "fail to validate uri!", K(ret), K(uri)); + } else if (OB_FAIL(build_full_dir_path(uri.ptr(), uri_buf, sizeof(uri_buf)))) { + OB_LOG(WARN, "fail to make uri end with '/'", K(ret), K(uri)); + } else if (OB_FAIL(util_->list_directories(uri_buf, op))) { + STORAGE_LOG(WARN, "failed to list_files", K(ret), K(uri), K(uri_buf)); + } + print_access_storage_log("list_files", uri, start_ts, 0); + return ret; +} + +/*---------------------------------- NEW ADAPTIVE INTERFACE ----------------------------------*/ + int ObStorageUtil::is_exist(const common::ObString &uri, bool &exist) { int ret = OB_SUCCESS; @@ -254,14 +982,14 @@ int ObStorageUtil::is_exist(const common::ObString &uri, bool &exist) #endif if (OB_FAIL(ret)) { //do nothing - } else if (!is_init()) { + } else if (OB_UNLIKELY(!is_init())) { ret = OB_NOT_INIT; STORAGE_LOG(WARN, "util is not inited", K(ret), K(uri)); } else if (ObStorageGlobalIns::get_instance().is_io_prohibited()) { ret = OB_BACKUP_IO_PROHIBITED; STORAGE_LOG(WARN, "current observer backup io is prohibited", K(ret), K(uri)); } else if (OB_FAIL(validate_uri_type(uri))) { - STORAGE_LOG(WARN, "fail to validate uri!", K(uri)); + STORAGE_LOG(WARN, "fail to validate uri!", K(ret), K(uri)); } else if (OB_FAIL(util_->is_exist(uri, exist))) { STORAGE_LOG(WARN, "failed to check is exist", K(ret), K(uri)); } @@ -280,14 +1008,14 @@ int ObStorageUtil::get_file_length(const common::ObString &uri, int64_t &file_le ret = OB_E(EventTable::EN_BACKUP_IO_GET_FILE_LENGTH) OB_SUCCESS; #endif if (OB_FAIL(ret)) { - } else if (!is_init()) { + } else if (OB_UNLIKELY(!is_init())) { ret = OB_NOT_INIT; STORAGE_LOG(WARN, "util is not inited", K(ret), K(uri)); } else if (ObStorageGlobalIns::get_instance().is_io_prohibited()) { ret = OB_BACKUP_IO_PROHIBITED; STORAGE_LOG(WARN, "current observer backup io is prohibited", K(ret), K(uri)); } else if (OB_FAIL(validate_uri_type(uri))) { - STORAGE_LOG(WARN, "fail to validate uri!", K(uri)); + STORAGE_LOG(WARN, "fail to validate uri!", K(ret), K(uri)); } else if (OB_FAIL(util_->get_file_length(uri, file_length))) { if (OB_BACKUP_FILE_NOT_EXIST == ret) { STORAGE_LOG(INFO, "cannot get file length for not exist file", K(ret), K(uri)); @@ -295,14 +1023,69 @@ int ObStorageUtil::get_file_length(const common::ObString &uri, int64_t &file_le STORAGE_LOG(WARN, "failed to get_file_length", K(ret), K(uri)); } } - - if (file_length <= 0) { - STORAGE_LOG(INFO, "this file is empty", K(ret), K(uri), K(file_length)); + + if (OB_SUCC(ret)) { + if (file_length == 0) { + STORAGE_LOG(INFO, "this file is empty", K(ret), K(uri), K(file_length)); + } else if (file_length < 0) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "this file length is invalid", K(ret), K(uri), K(file_length)); + } } + print_access_storage_log("get_file_length", uri, start_ts); return ret; } +int ObStorageUtil::get_adaptive_file_length(const common::ObString &uri, int64_t &file_length) +{ + int ret = OB_SUCCESS; + const int64_t start_ts = ObTimeUtility::current_time(); + file_length = 0; + ObStorageObjectMeta obj_meta; +#ifdef ERRSIM + ret = OB_E(EventTable::EN_BACKUP_IO_GET_FILE_LENGTH) OB_SUCCESS; +#endif + if (OB_FAIL(ret)) { + } else if (OB_UNLIKELY(!is_init())) { + ret = OB_NOT_INIT; + STORAGE_LOG(WARN, "util is not inited", K(ret), K(uri)); + } else if (ObStorageGlobalIns::get_instance().is_io_prohibited()) { + ret = OB_BACKUP_IO_PROHIBITED; + STORAGE_LOG(WARN, "current observer backup io is prohibited", K(ret), K(uri)); + } else if (OB_FAIL(validate_uri_type(uri))) { + STORAGE_LOG(WARN, "fail to validate uri!", K(ret), K(uri)); + } else { + if (OB_FAIL(read_seal_meta_if_needed(uri, obj_meta))) { + OB_LOG(WARN, "fail to read seal meta if needed", K(ret), K(uri)); + } + + // If not exist seal meta, we need to list all files and check format meta exist or not. + // format meta must exist, otherwise need to return error. + if (OB_SUCC(ret) && !obj_meta.is_exist_) { + if (OB_FAIL(list_appendable_file_fragments(uri, obj_meta))) { + OB_LOG(WARN, "fail to list appendable file fragments", K(ret), K(uri)); + } + } + + if (OB_SUCC(ret) && obj_meta.is_exist_) { + file_length = obj_meta.length_; + } + } + + if (OB_SUCC(ret)) { + if (file_length == 0) { + STORAGE_LOG(INFO, "this file is empty", K(ret), K(uri), K(file_length)); + } else if (file_length < 0) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "this file length is invalid", K(ret), K(uri), K(file_length)); + } + } + + print_access_storage_log("get_adaptive_file_length", uri, start_ts); + return ret; +} + int ObStorageUtil::del_file(const common::ObString &uri) { int ret = OB_SUCCESS; @@ -312,14 +1095,14 @@ int ObStorageUtil::del_file(const common::ObString &uri) ret = OB_E(EventTable::EN_BACKUP_IO_BEFORE_DEL_FILE) OB_SUCCESS; #endif if (OB_FAIL(ret)) { - } else if (!is_init()) { + } else if (OB_UNLIKELY(!is_init())) { ret = OB_NOT_INIT; STORAGE_LOG(WARN, "util is not inited", K(ret), K(uri)); } else if (ObStorageGlobalIns::get_instance().is_io_prohibited()) { ret = OB_BACKUP_IO_PROHIBITED; STORAGE_LOG(WARN, "current observer backup io is prohibited", K(ret), K(uri)); } else if (OB_FAIL(validate_uri_type(uri))) { - STORAGE_LOG(WARN, "fail to validate uri!", K(uri)); + STORAGE_LOG(WARN, "fail to validate uri!", K(ret), K(uri)); } else { const int max_try_cnt = 5; int try_cnt = 0; @@ -364,14 +1147,14 @@ int ObStorageUtil::mkdir(const common::ObString &uri) ret = OB_E(EventTable::EN_BACKUP_IO_BEFORE_MKDIR) OB_SUCCESS; #endif if (OB_FAIL(ret)) { - } else if (!is_init()) { + } else if (OB_UNLIKELY(!is_init())) { ret = OB_NOT_INIT; STORAGE_LOG(WARN, "util is not inited", K(ret), K(uri)); } else if (ObStorageGlobalIns::get_instance().is_io_prohibited()) { ret = OB_BACKUP_IO_PROHIBITED; STORAGE_LOG(WARN, "current observer backup io is prohibited", K(ret), K(uri)); } else if (OB_FAIL(validate_uri_type(uri))) { - STORAGE_LOG(WARN, "fail to validate uri!", K(uri)); + STORAGE_LOG(WARN, "fail to validate uri!", K(ret), K(uri)); } else if (OB_FAIL(util_->mkdir(uri))) { STORAGE_LOG(WARN, "failed to mkdir", K(ret), K(uri)); } @@ -387,6 +1170,7 @@ int ObStorageUtil::mkdir(const common::ObString &uri) int ObStorageUtil::list_files(const common::ObString &uri, common::ObBaseDirEntryOperator &op) { int ret = OB_SUCCESS; + char uri_buf[OB_MAX_URI_LENGTH] = {0}; const int64_t start_ts = ObTimeUtility::current_time(); #ifdef ERRSIM @@ -394,16 +1178,18 @@ int ObStorageUtil::list_files(const common::ObString &uri, common::ObBaseDirEntr #endif if (OB_FAIL(ret)) { - } else if (!is_init()) { + } else if (OB_UNLIKELY(!is_init())) { ret = OB_NOT_INIT; STORAGE_LOG(WARN, "util is not inited", K(ret), K(uri)); } else if (ObStorageGlobalIns::get_instance().is_io_prohibited()) { ret = OB_BACKUP_IO_PROHIBITED; STORAGE_LOG(WARN, "current observer backup io is prohibited", K(ret), K(uri)); } else if (OB_FAIL(validate_uri_type(uri))) { - STORAGE_LOG(WARN, "fail to validate uri!", K(uri)); - } else if (OB_FAIL(util_->list_files(uri, op))) { - STORAGE_LOG(WARN, "failed to list_files", K(ret), K(uri)); + STORAGE_LOG(WARN, "fail to validate uri!", K(ret), K(uri)); + } else if (OB_FAIL(build_full_dir_path(uri.ptr(), uri_buf, sizeof(uri_buf)))) { + OB_LOG(WARN, "fail to make dir path end with '/'", K(ret), K(uri)); + } else if (OB_FAIL(util_->list_files(uri_buf, op))) { + STORAGE_LOG(WARN, "failed to list_files", K(ret), K(uri), K(uri_buf)); } print_access_storage_log("list_files", uri, start_ts, 0); @@ -425,14 +1211,14 @@ int ObStorageUtil::write_single_file(const common::ObString &uri, const char *bu ret = OB_E(EventTable::EN_BACKUP_IO_BEFORE_WRITE_SINGLE_FILE) OB_SUCCESS; #endif if (OB_FAIL(ret)) { - } else if (!is_init()) { + } else if (OB_UNLIKELY(!is_init())) { ret = OB_NOT_INIT; STORAGE_LOG(WARN, "util is not inited", K(ret), K(uri)); } else if (ObStorageGlobalIns::get_instance().is_io_prohibited()) { ret = OB_BACKUP_IO_PROHIBITED; STORAGE_LOG(WARN, "current observer backup io is prohibited", K(ret), K(uri)); } else if (OB_FAIL(validate_uri_type(uri))) { - STORAGE_LOG(WARN, "fail to validate uri!", K(uri)); + STORAGE_LOG(WARN, "fail to validate uri!", K(ret), K(uri)); } else if (OB_FAIL(util_->write_single_file(uri, buf, size))) { STORAGE_LOG(WARN, "failed to write single file", K(ret), K(uri)); } else { @@ -462,11 +1248,11 @@ int ObStorageUtil::del_dir(const common::ObString &uri) const int64_t start_ts = ObTimeUtility::current_time(); if (OB_FAIL(ret)) { - } else if (!is_init()) { + } else if (OB_UNLIKELY(!is_init())) { ret = OB_NOT_INIT; STORAGE_LOG(WARN, "util is not inited", K(ret), K(uri)); } else if (OB_FAIL(validate_uri_type(uri))) { - STORAGE_LOG(WARN, "fail to validate uri!", K(uri)); + STORAGE_LOG(WARN, "fail to validate uri!", K(ret), K(uri)); } else if (OB_FAIL(util_->del_dir(uri))) { STORAGE_LOG(WARN, "failed to del_file", K(ret), K(uri)); } @@ -477,6 +1263,7 @@ int ObStorageUtil::del_dir(const common::ObString &uri) int ObStorageUtil::list_directories(const common::ObString &uri, common::ObBaseDirEntryOperator &op) { int ret = OB_SUCCESS; + char uri_buf[OB_MAX_URI_LENGTH] = {0}; const int64_t start_ts = ObTimeUtility::current_time(); #ifdef ERRSIM @@ -487,13 +1274,17 @@ int ObStorageUtil::list_directories(const common::ObString &uri, common::ObBaseD } else if (ObStorageGlobalIns::get_instance().is_io_prohibited()) { ret = OB_BACKUP_IO_PROHIBITED; STORAGE_LOG(WARN, "current observer backup io is prohibited", K(ret), K(uri)); - } else if (!is_init()) { + } else if (OB_UNLIKELY(!is_init())) { ret = OB_NOT_INIT; STORAGE_LOG(WARN, "util is not inited", K(ret), K(uri)); - } else if (OB_FAIL(util_->list_directories(uri, op))) { - STORAGE_LOG(WARN, "failed to list_files", K(ret), K(uri)); + } else if (OB_FAIL(validate_uri_type(uri))) { + STORAGE_LOG(WARN, "fail to validate uri!", K(ret), K(uri)); + } else if (OB_FAIL(build_full_dir_path(uri.ptr(), uri_buf, sizeof(uri_buf)))) { + OB_LOG(WARN, "fail to make dir path end with '/'", K(ret), K(uri)); + } else if (OB_FAIL(util_->list_directories(uri_buf, op))) { + STORAGE_LOG(WARN, "failed to list_directories", K(ret), K(uri), K(uri_buf)); } - print_access_storage_log("list_files", uri, start_ts, 0); + print_access_storage_log("list_directories", uri, start_ts, 0); return ret; } @@ -504,11 +1295,14 @@ int ObStorageUtil::is_tagging(const common::ObString &uri, bool &is_tagging) const int64_t start_ts = ObTimeUtility::current_time(); is_tagging = false; if (OB_FAIL(ret)) { + } else if (OB_UNLIKELY(!is_init())) { + ret = OB_NOT_INIT; + STORAGE_LOG(WARN, "util is not inited", K(ret), K(uri)); } else if (ObStorageGlobalIns::get_instance().is_io_prohibited()) { ret = OB_BACKUP_IO_PROHIBITED; STORAGE_LOG(WARN, "current observer backup io is prohibited", K(ret), K(uri)); } else if (OB_FAIL(validate_uri_type(uri))) { - STORAGE_LOG(WARN, "fail to validate uri!", K(uri)); + STORAGE_LOG(WARN, "fail to validate uri!", K(ret), K(uri)); } else if (OB_FAIL(util_->is_tagging(uri, is_tagging))) { STORAGE_LOG(WARN, "failed to check is tagging", K(ret), K(uri)); } @@ -516,12 +1310,37 @@ int ObStorageUtil::is_tagging(const common::ObString &uri, bool &is_tagging) return ret; } +int ObStorageUtil::del_unmerged_parts(const common::ObString &uri) +{ + int ret = OB_NOT_SUPPORTED; + OB_LOG(WARN, "del_unmerged_parts is not supported yet", K(ret), K(uri)); + // const int64_t start_ts = ObTimeUtility::current_time(); + // if (OB_FAIL(ret)) { + // } else if (OB_UNLIKELY(!is_init())) { + // ret = OB_NOT_INIT; + // STORAGE_LOG(WARN, "util is not inited", K(ret), K(uri)); + // } else if (ObStorageGlobalIns::get_instance().is_io_prohibited()) { + // ret = OB_BACKUP_IO_PROHIBITED; + // STORAGE_LOG(WARN, "current observer backup io is prohibited", K(ret), K(uri)); + // } else if (OB_FAIL(validate_uri_type(uri))) { + // STORAGE_LOG(WARN, "fail to validate uri!", K(ret), K(uri)); + // } else if (OB_FAIL(util_->del_unmerged_parts(uri))) { + // STORAGE_LOG(WARN, "fail to del unmerged parts!", K(ret), K(uri)); + // } + // print_access_storage_log("del_unmerged_parts", uri, start_ts, 0); + return ret; +} + +/** + * ------------------------------ObStorageReader------------------------------ + */ ObStorageReader::ObStorageReader() : file_length_(-1), reader_(NULL), file_reader_(), oss_reader_(), cos_reader_(), + s3_reader_(), start_ts_(0) { uri_[0] = '\0'; @@ -559,6 +1378,8 @@ int ObStorageReader::open(const common::ObString &uri, common::ObObjectStorageIn reader_ = &oss_reader_; } else if (OB_STORAGE_COS == type) { reader_ = &cos_reader_; + } else if (OB_STORAGE_S3 == type) { + reader_ = &s3_reader_; } else if (OB_STORAGE_FILE == type) { reader_ = &file_reader_; } else { @@ -583,6 +1404,7 @@ int ObStorageReader::open(const common::ObString &uri, common::ObObjectStorageIn } } + print_access_storage_log("ObStorageReader::open", uri_, start_ts_, 0); return ret; } @@ -616,6 +1438,7 @@ int ObStorageReader::pread(char *buf,const int64_t buf_size, int64_t offset, int EVENT_INC(ObStatEventIds::BACKUP_IO_READ_COUNT); EVENT_ADD(ObStatEventIds::BACKUP_IO_READ_DELAY, ObTimeUtility::current_time() - start_ts); + print_access_storage_log("ObStorageReader::pread", uri_, start_ts, read_size); return ret; } @@ -640,12 +1463,215 @@ int ObStorageReader::close() return ret; } +ObStorageAdaptiveReader::ObStorageAdaptiveReader() + : allocator_(APPENDABLE_OBJECT_ALLOCATOR), + meta_(), + bucket_(), + object_(), + reader_(NULL), + file_reader_(), + oss_reader_(), + cos_reader_(), + s3_reader_(), + start_ts_(0), + storage_info_(NULL) +{ + uri_[0] = '\0'; +} + +ObStorageAdaptiveReader::~ObStorageAdaptiveReader() +{ + if (NULL != reader_) { + STORAGE_LOG_RET(ERROR, common::OB_ERR_UNEXPECTED, "adaptive reader not closed", KCSTRING(uri_)); + } +} + +int ObStorageAdaptiveReader::open(const common::ObString &uri, + common::ObObjectStorageInfo *storage_info) +{ + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + ObStorageType type = OB_STORAGE_MAX_TYPE; + start_ts_ = ObTimeUtility::current_time(); + ObStorageUtil util; + + if (OB_FAIL(ret)) { + } else if (ObStorageGlobalIns::get_instance().is_io_prohibited()) { + ret = OB_BACKUP_IO_PROHIBITED; + STORAGE_LOG(WARN, "current observer backup io is prohibited", K(ret), K(uri)); + } else if (NULL != reader_) { + ret = OB_INIT_TWICE; + STORAGE_LOG(WARN, "cannot open twice", K(ret), K(uri)); + } else if (OB_FAIL(databuff_printf(uri_, sizeof(uri_), "%.*s", uri.length(), uri.ptr()))) { + STORAGE_LOG(WARN, "failed to fill uri", K(ret), K(uri)); + } else if (OB_FAIL(get_storage_type_from_path(uri, type))) { + STORAGE_LOG(WARN, "failed to get type", K(ret), K(uri)); + } else if (OB_STORAGE_OSS == type) { + reader_ = &oss_reader_; + } else if (OB_STORAGE_COS == type) { + reader_ = &cos_reader_; + } else if (OB_STORAGE_S3 == type) { + reader_ = &s3_reader_; + } else if (OB_STORAGE_FILE == type) { + reader_ = &file_reader_; + } else { + ret = OB_ERR_SYS; + STORAGE_LOG(ERROR, "unkown storage type", K(ret), K(uri)); + } + + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(reader_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "reader_ is null", K(ret), K(uri)); + } else if (OB_FAIL(util.open(storage_info))) { + OB_LOG(WARN, "fail to open util", K(ret), K(uri), KPC(storage_info)); + } else if (OB_FAIL(util.detect_storage_obj_meta(uri, true/*is_adaptive*/, + true/*need_fragment_meta*/, meta_))) { + OB_LOG(WARN, "fail to detect obejct meta", K(ret), K(uri), KPC(storage_info)); + } else if (meta_.is_simulate_append_type()) { + // no need to open reader + } else if (meta_.is_object_file_type()) { + if (OB_FAIL(reader_->open(uri, storage_info))) { + OB_LOG(WARN, "fail to open reader", K(ret), K(uri), KPC(storage_info)); + } + } else { + ret = OB_ERR_SYS; + STORAGE_LOG(ERROR, "invalid storage object type", K(ret), K(uri), KPC(storage_info), K_(meta)); + } + + if(OB_FAIL(ret)) { + if (OB_SUCCESS != (tmp_ret = close())) { + STORAGE_LOG(WARN, "failed to close read file", K(ret), K(tmp_ret), K(uri)); + } + } else { + storage_info_ = storage_info; + } + + print_access_storage_log("ObStorageAdaptiveReader::open", uri_, start_ts_, 0); + return ret; +} + +int ObStorageAdaptiveReader::pread(char *buf, + const int64_t buf_size, const int64_t offset, int64_t &read_size) +{ + int ret = OB_SUCCESS; + read_size = 0; + const int64_t start_ts = ObTimeUtility::current_time(); + if (OB_FAIL(ret)) { + } else if (ObStorageGlobalIns::get_instance().is_io_prohibited()) { + ret = OB_BACKUP_IO_PROHIBITED; + OB_LOG(WARN, "current observer backup io is prohibited", K(ret), K(offset), K(buf_size)); + } else if (OB_ISNULL(reader_)) { + ret = OB_NOT_INIT; + OB_LOG(WARN, "adaptive reader not opened", K(ret)); + } else if (OB_ISNULL(buf) || OB_UNLIKELY(buf_size <= 0 || offset < 0)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid args", K(ret), KP(buf), K(buf_size), K(offset), K_(meta)); + } else if (OB_UNLIKELY(offset > meta_.length_)) { + ret = OB_FILE_LENGTH_INVALID; + OB_LOG(WARN, "offset is larger than file length", K(ret), K(offset), K_(meta)); + } else if (offset == meta_.length_) { + ret = OB_SUCCESS; + read_size = 0; + } else if (meta_.is_object_file_type()) { + if (OB_FAIL(reader_->pread(buf, buf_size, offset, read_size))) { + OB_LOG(WARN, "fail to read object", K(ret), K_(meta)); + } + } else if (meta_.is_simulate_append_type()) { + char uri[OB_MAX_URI_LENGTH] = { 0 }; + ObArray fragments_need_to_read; + if (OB_FAIL(meta_.get_needed_fragments(offset, offset + buf_size, + fragments_need_to_read))) { + OB_LOG(WARN, "failed to get fragments need to read", + K(ret), K_(bucket), K_(object), K(offset), K(buf_size), K_(meta)); + } else if (fragments_need_to_read.empty()) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "the subsequent data that is anticipated to be read may not be present", + K(ret), K(offset), K(buf_size)); + } else { + int64_t cur_read_size = 0; + int64_t actual_read_size = -1; + int64_t expected_read_size = -1; + for (int64_t i = 0; OB_SUCC(ret) && i < fragments_need_to_read.size(); i++) { + actual_read_size = -1; + const ObAppendableFragmentMeta &fragment_meta = fragments_need_to_read[i]; + expected_read_size = + std::min({meta_.length_, fragment_meta.end_, offset + buf_size}) + - (offset + cur_read_size); + if (OB_UNLIKELY(!fragment_meta.is_valid())) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "invalid fragment meta", + K(ret), K(fragment_meta), K(cur_read_size)); + } else if (OB_FAIL(construct_fragment_full_name(uri_, + fragment_meta.start_, fragment_meta.end_, + uri, sizeof(uri)))) { + OB_LOG(WARN, "failed to construct fragment name", K(ret), + K(fragment_meta), K(cur_read_size)); + } else if (OB_FAIL(reader_->open(uri, storage_info_, false/*head_meta*/))) { + OB_LOG(WARN, "fail to open reader for fragment", + K(ret), K(uri), K(fragment_meta), K(cur_read_size), K(offset)); + } else if (OB_FAIL(reader_->pread(buf + cur_read_size, + expected_read_size, + (offset + cur_read_size) - fragment_meta.start_, + actual_read_size))) { + OB_LOG(WARN, "failed to read fragment content", K(ret), + K(uri), K(fragment_meta), K(cur_read_size), K(offset), K(expected_read_size)); + } else if (OB_FAIL(reader_->close())) { + OB_LOG(WARN, "fail to close reader", K(ret), K(uri), K(fragment_meta), K(cur_read_size)); + } else if (actual_read_size != expected_read_size) { + ret = OB_IO_ERROR; + OB_LOG(WARN, "failed to read expected size from fragment", K(ret), + K(uri), K(fragment_meta), + K(cur_read_size), K(offset), K(actual_read_size), K(expected_read_size)); + } else { + cur_read_size += actual_read_size; + } + } + + if (OB_SUCC(ret)) { + read_size = cur_read_size; + } + } + } else { + ret = OB_ERR_SYS; + OB_LOG(ERROR, "unkown object type", K(ret), K_(meta)); + } + + // TODO @fangdan: add event + EVENT_ADD(ObStatEventIds::BACKUP_IO_READ_DELAY, ObTimeUtility::current_time() - start_ts); + print_access_storage_log("ObStorageAdaptiveReader::pread", uri_, start_ts, read_size); + return ret; +} + +int ObStorageAdaptiveReader::close() +{ + int ret = OB_SUCCESS; + + if (NULL != reader_) { + print_access_storage_log("storage adaptive reader", uri_, start_ts_, meta_.length_); + } + + if (OB_ISNULL(reader_)) { + ret = OB_NOT_INIT; + STORAGE_LOG(WARN, "not opened", K(ret)); + } else if (OB_FAIL(reader_->close())) { + STORAGE_LOG(WARN, "failed to close reader", K(ret)); + } + reader_ = NULL; + start_ts_ = 0; + uri_[0] = '\0'; + allocator_.clear(); + meta_.reset(); + storage_info_ = NULL; + return ret; +} ObStorageWriter::ObStorageWriter() : writer_(NULL), file_writer_(), oss_writer_(), cos_writer_(), + s3_writer_(), start_ts_(0) { uri_[0] = '\0'; @@ -683,6 +1709,8 @@ int ObStorageWriter::open(const common::ObString &uri, common::ObObjectStorageIn writer_ = &oss_writer_; } else if (OB_STORAGE_COS == type) { writer_ = &cos_writer_; + } else if (OB_STORAGE_S3 == type) { + writer_ = &s3_writer_; } else if (OB_STORAGE_FILE == type) { writer_ = &file_writer_; } else { @@ -704,8 +1732,9 @@ int ObStorageWriter::open(const common::ObString &uri, common::ObObjectStorageIn STORAGE_LOG(WARN, "failed close write file", K(ret), K(tmp_ret), K(uri)); } } - return ret; + print_access_storage_log("ObStorageWriter::open", uri_, start_ts_, 0); + return ret; } int ObStorageWriter::write(const char *buf,const int64_t size) @@ -734,6 +1763,7 @@ int ObStorageWriter::write(const char *buf,const int64_t size) EVENT_INC(ObStatEventIds::BACKUP_IO_WRITE_COUNT); EVENT_ADD(ObStatEventIds::BACKUP_IO_WRITE_DELAY, ObTimeUtility::current_time() - start_ts); + print_access_storage_log("ObStorageWriter::write", uri_, start_ts, size); return ret; } @@ -762,9 +1792,12 @@ ObStorageAppender::ObStorageAppender() file_appender_(), oss_appender_(), cos_appender_(), + s3_appender_(), start_ts_(0), is_opened_(false), - storage_info_() + storage_info_(), + allocator_(APPENDABLE_OBJECT_ALLOCATOR), + type_(OB_STORAGE_MAX_TYPE) { uri_[0] = '\0'; } @@ -774,9 +1807,12 @@ ObStorageAppender::ObStorageAppender(StorageOpenMode mode) file_appender_(mode), oss_appender_(), cos_appender_(), + s3_appender_(), start_ts_(0), is_opened_(false), - storage_info_() + storage_info_(), + allocator_(APPENDABLE_OBJECT_ALLOCATOR), + type_(OB_STORAGE_MAX_TYPE) { uri_[0] = '\0'; } @@ -794,7 +1830,6 @@ int ObStorageAppender::open( { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - ObStorageType type = OB_STORAGE_MAX_TYPE; start_ts_ = ObTimeUtility::current_time(); #ifdef ERRSIM @@ -812,17 +1847,19 @@ int ObStorageAppender::open( STORAGE_LOG(WARN, "invalid argument", K(ret), K(uri), KP(storage_info)); } else if (OB_FAIL(databuff_printf(uri_, sizeof(uri_), "%.*s", uri.length(), uri.ptr()))) { STORAGE_LOG(WARN, "failed to fill uri", K(ret), K(uri)); - } else if (OB_FAIL(get_storage_type_from_path(uri, type))) { + } else if (OB_FAIL(get_storage_type_from_path(uri, type_))) { STORAGE_LOG(WARN, "failed to get type", K(ret), K(uri)); - } else if (OB_STORAGE_OSS == type || OB_STORAGE_COS == type) { + } else if (OB_STORAGE_OSS == type_ || OB_STORAGE_COS == type_ || OB_STORAGE_S3 == type_) { if (OB_FAIL(storage_info_.assign(*storage_info))) { STORAGE_LOG(WARN, "failed to copy storage info", K(ret)); - } else if (OB_STORAGE_OSS == type) { + } else if (OB_STORAGE_OSS == type_) { appender_ = &oss_appender_; - } else if (OB_STORAGE_COS == type) { + } else if (OB_STORAGE_COS == type_) { appender_ = &cos_appender_; + } else if (OB_STORAGE_S3 == type_) { + appender_ = &s3_appender_; } - } else if (OB_STORAGE_FILE == type) { + } else if (OB_STORAGE_FILE == type_) { appender_ = &file_appender_; } else { ret = OB_ERR_SYS; @@ -845,6 +1882,8 @@ int ObStorageAppender::open( STORAGE_LOG(WARN, "failed close write file", K(ret), K(tmp_ret), K(uri)); } } + + print_access_storage_log("ObStorageAppender::open", uri_, start_ts_, 0); return ret; } @@ -874,6 +1913,7 @@ int ObStorageAppender::write(const char *buf,const int64_t size) EVENT_INC(ObStatEventIds::BACKUP_IO_WRITE_COUNT); EVENT_ADD(ObStatEventIds::BACKUP_IO_WRITE_DELAY, ObTimeUtility::current_time() - start_ts); + print_access_storage_log("ObStorageAppender::write", uri_, start_ts, size); return ret; } @@ -927,6 +1967,7 @@ int ObStorageAppender::repeatable_pwrite_(const char *buf, const int64_t size, c int ObStorageAppender::pwrite(const char *buf, const int64_t size, const int64_t offset) { int ret = OB_SUCCESS; + const int64_t start_ts = ObTimeUtility::current_time(); #ifdef ERRSIM ret = OB_E(EventTable::EN_BACKUP_IO_APPENDER_WRITE) OB_SUCCESS; @@ -942,7 +1983,9 @@ int ObStorageAppender::pwrite(const char *buf, const int64_t size, const int64_t STORAGE_LOG(WARN, "failed to write", K(ret)); } - if (OB_BACKUP_PWRITE_OFFSET_NOT_MATCH == ret) { + // no need to adjust the function repeatable_pwrite_ + // because S3 will not return OB_BACKUP_PWRITE_OFFSET_NOT_MATCH + if (OB_BACKUP_PWRITE_OFFSET_NOT_MATCH == ret && appender_ != &s3_appender_) { if (OB_FAIL(repeatable_pwrite_(buf, size, offset))) { STORAGE_LOG(WARN, "failed to repeatable_pwrite", K(ret)); } else { @@ -950,18 +1993,36 @@ int ObStorageAppender::pwrite(const char *buf, const int64_t size, const int64_t } } + print_access_storage_log("ObStorageAppender::pwrite", uri_, start_ts, size); return ret; } int64_t ObStorageAppender::get_length() { int64_t ret_int = -1; + const int64_t start_ts = ObTimeUtility::current_time(); if (OB_ISNULL(appender_)) { STORAGE_LOG_RET(WARN, common::OB_ERR_UNEXPECTED, "appender not opened"); - } else { + } else if (type_ != OB_STORAGE_S3) { ret_int = appender_->get_length(); + } else { + int ret = OB_SUCCESS; + ObStorageUtil util; + ObStorageObjectMeta meta; + ObExternalIOCounterGuard io_guard; + char uri[OB_MAX_URI_LENGTH] = { 0 }; + if (OB_FAIL(util.open(&storage_info_))) { + OB_LOG(WARN, "failed to open util", K(ret)); + } else if (OB_FAIL(util.list_appendable_file_fragments(uri_, meta))) { + OB_LOG(WARN, "failed to list appendable object fragments", K(ret), K_(uri)); + } else { + ret_int = meta.length_; + } + util.close(); } + + print_access_storage_log("ObStorageAppender::get_length", uri_, start_ts, 0); return ret_int; } @@ -986,6 +2047,217 @@ int ObStorageAppender::close() return ret; } +int ObStorageAppender::seal_for_adaptive() +{ + int ret = OB_SUCCESS; + const int64_t start_ts = ObTimeUtility::current_time(); + int64_t handled_size = 0; + + if (OB_NOT_NULL(appender_)) { + print_access_storage_log("storage appender_", uri_, start_ts_, appender_->get_length()); + } + + if (OB_ISNULL(appender_)) { + ret = OB_NOT_INIT; + STORAGE_LOG(WARN, "not opened", K(ret)); + } else if (type_ != OB_STORAGE_S3) { + } else { + char *buf = NULL; + char seal_meta_uri[OB_MAX_URI_LENGTH] = { 0 }; + ObStorageUtil util; + ListAppendableObjectFragmentOp op; + ObStorageObjectMeta appendable_obj_meta; + int64_t pos = 0; // for serializing appendable_obj_meta + int64_t serialize_size = -1; // for serializing appendable_obj_meta + ObExternalIOCounterGuard io_guard; + + if (OB_FAIL(util.open(&storage_info_))) { + OB_LOG(WARN, "failed to open util", K(ret), K_(uri), K_(storage_info)); + } else if (OB_FAIL(util.list_appendable_file_fragments(uri_, appendable_obj_meta))) { + OB_LOG(WARN, "failed to list appendable object fragments", K(ret), K_(uri)); + } else if (FALSE_IT(serialize_size = appendable_obj_meta.get_serialize_size() + 1)) { + } else if (OB_ISNULL(buf = static_cast(allocator_.alloc(serialize_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "failed to alloc memory for appendable object seal meta buf", + K(ret), K(serialize_size), K_(uri), K(appendable_obj_meta)); + } else if (OB_FAIL(appendable_obj_meta.serialize(buf, serialize_size, pos))) { + OB_LOG(WARN, "failed to serialize s3 appendable object meta", + K(ret), K(serialize_size), K_(uri), K(appendable_obj_meta)); + } else if (OB_FAIL(construct_fragment_full_name(uri_, OB_S3_APPENDABLE_SEAL_META, + seal_meta_uri, sizeof(seal_meta_uri)))) { + OB_LOG(WARN, "failed to construct s3 appendable object name for writing seal meta file", + K(ret), K_(uri), K(appendable_obj_meta)); + } else if (OB_FAIL(util.write_single_file(seal_meta_uri, buf, pos))) { + OB_LOG(WARN, "fail to write seal meta file", K(ret), K(seal_meta_uri), K(appendable_obj_meta)); + } else { + // The seal operation includes listing fragments and writing this data to a meta file. + // Here, 'serialize_size' is used as an approximation of the data throughput of the list operation. + handled_size = serialize_size + pos; + OB_LOG(DEBUG, "succeed to write seal meta file", + K_(uri), K(seal_meta_uri), KP(buf), K(pos), K(appendable_obj_meta)); + } + util.close(); + } + + print_access_storage_log("ObStorageAppender::seal_for_adaptive", uri_, start_ts, handled_size); + return ret; +} + +ObStorageMultiPartWriter::ObStorageMultiPartWriter() + : multipart_writer_(NULL), + file_multipart_writer_(), + cos_multipart_writer_(), + oss_multipart_writer_(), + s3_multipart_writer_(), + start_ts_(0), + is_opened_(false), + storage_info_() +{ + uri_[0] = '\0'; +} + +ObStorageMultiPartWriter::~ObStorageMultiPartWriter() +{ + if (is_opened_ && NULL != multipart_writer_) { + STORAGE_LOG_RET(ERROR, common::OB_ERR_UNEXPECTED, "multipart_writer not close"); + } +} + +int ObStorageMultiPartWriter::open( + const common::ObString &uri, + common::ObObjectStorageInfo *storage_info) +{ + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + ObStorageType type = OB_STORAGE_MAX_TYPE; + start_ts_ = ObTimeUtility::current_time(); + if (ObStorageGlobalIns::get_instance().is_io_prohibited()) { + ret = OB_BACKUP_IO_PROHIBITED; + STORAGE_LOG(WARN, "current observer backup io is prohibited", K(ret), K(uri)); + } else if (NULL != multipart_writer_) { + ret = OB_INIT_TWICE; + STORAGE_LOG(WARN, "multipart writer cannot open twice", K(ret), K(uri)); + } else if (OB_UNLIKELY(uri.empty()) || OB_ISNULL(storage_info)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", K(ret), K(uri), KPC(storage_info)); + } else if (OB_FAIL(databuff_printf(uri_, sizeof(uri_), "%.*s", uri.length(), uri.ptr()))) { + STORAGE_LOG(WARN, "failed to fill uri", K(ret), K(uri)); + } else if (OB_FAIL(get_storage_type_from_path(uri, type))) { + STORAGE_LOG(WARN, "failed to get type", K(ret), K(uri)); + } else if (OB_STORAGE_OSS == type || OB_STORAGE_COS == type || OB_STORAGE_S3 == type) { + if (OB_FAIL(storage_info_.assign(*storage_info))) { + STORAGE_LOG(WARN, "failed to copy storage info", K(ret), KPC(storage_info)); + } else if (OB_STORAGE_OSS == type) { + multipart_writer_ = &oss_multipart_writer_; + } else if (OB_STORAGE_COS == type) { + multipart_writer_ = &cos_multipart_writer_; + } else if (OB_STORAGE_S3 == type) { + multipart_writer_ = &s3_multipart_writer_; + } + } else if (OB_STORAGE_FILE == type) { + multipart_writer_ = &file_multipart_writer_; + } else { + ret = OB_ERR_SYS; + STORAGE_LOG(ERROR, "unkown storage type", K(ret), K(uri)); + } + + if (OB_SUCC(ret)) { + if (OB_ISNULL(multipart_writer_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "multipart_writer is null", K(ret), K(uri)); + } else if (OB_FAIL(multipart_writer_->open(uri, storage_info))) { + STORAGE_LOG(WARN, "failed to open multipart writer", K(ret), K(uri), K(storage_info)); + } else { + is_opened_ = true; + } + } + + if (OB_FAIL(ret)) { + if (OB_SUCCESS != (tmp_ret = close())) { + STORAGE_LOG(WARN, "failed close multipart writer", K(ret), K(tmp_ret), K(uri)); + } + } + + print_access_storage_log("ObStorageMultiPartWriter::open", uri_, start_ts_, 0); + return ret; +} + +int ObStorageMultiPartWriter::write(const char *buf, const int64_t size) +{ + int ret = OB_SUCCESS; + const int64_t start_ts = ObTimeUtility::current_time(); + if (ObStorageGlobalIns::get_instance().is_io_prohibited()) { + ret = OB_BACKUP_IO_PROHIBITED; + STORAGE_LOG(WARN, "current observer backup io is prohibited", K(ret), K(size)); + } else if (OB_ISNULL(multipart_writer_)) { + ret = OB_NOT_INIT; + STORAGE_LOG(WARN, "multipart writer not opened", K(ret)); + } else if(OB_FAIL(multipart_writer_->write(buf, size))) { + EVENT_INC(ObStatEventIds::BACKUP_IO_WRITE_FAIL_COUNT); + STORAGE_LOG(WARN, "failed to write", K(ret)); + } else { + EVENT_ADD(ObStatEventIds::BACKUP_IO_WRITE_BYTES, size); + } + EVENT_INC(ObStatEventIds::BACKUP_IO_WRITE_COUNT); + EVENT_ADD(ObStatEventIds::BACKUP_IO_WRITE_DELAY, ObTimeUtility::current_time() - start_ts); + + print_access_storage_log("ObStorageMultiPartWriter::write", uri_, start_ts, size); + return ret; +} + +int ObStorageMultiPartWriter::pwrite(const char *buf, const int64_t size, const int64_t offset) +{ + int ret = OB_SUCCESS; + const int64_t start_ts = ObTimeUtility::current_time(); + if (OB_FAIL(ret)) { + } else if (ObStorageGlobalIns::get_instance().is_io_prohibited()) { + ret = OB_BACKUP_IO_PROHIBITED; + STORAGE_LOG(WARN, "current observer backup io is prohibited", K(ret), K(offset), K(size)); + } else if (OB_ISNULL(multipart_writer_)) { + ret = OB_NOT_INIT; + STORAGE_LOG(WARN, "multipart writer not opened", K(ret)); + } else if (OB_FAIL(multipart_writer_->pwrite(buf, size, offset))) { + STORAGE_LOG(WARN, "failed to write", K(ret)); + } + + print_access_storage_log("ObStorageMultiPartWriter::pwrite", uri_, start_ts, size); + return ret; +} + +int64_t ObStorageMultiPartWriter::get_length() +{ + int64_t ret_int = -1; + const int64_t start_ts = ObTimeUtility::current_time(); + if (OB_ISNULL(multipart_writer_)) { + STORAGE_LOG_RET(WARN, common::OB_ERR_UNEXPECTED, "multipart_writer_ not opened"); + } else { + ret_int = multipart_writer_->get_length(); + } + + print_access_storage_log("ObStorageMultiPartWriter::get_length", uri_, start_ts, 0); + return ret_int; +} + +int ObStorageMultiPartWriter::close() +{ + int ret = OB_SUCCESS; + const int64_t start_ts = ObTimeUtility::current_time(); + if (NULL != multipart_writer_) { + print_access_storage_log("storage multipart writer", uri_, start_ts_, multipart_writer_->get_length()); + } + if (OB_ISNULL(multipart_writer_)) { + ret = OB_NOT_INIT; + STORAGE_LOG(WARN, "not opened", K(ret)); + } else if (OB_FAIL(multipart_writer_->close())) { + STORAGE_LOG(WARN, "failed to close multipart writer", K(ret)); + } + multipart_writer_ = NULL; + start_ts_ = 0; + uri_[0] = '\0'; + is_opened_ = false; + print_access_storage_log("ObStorageMultiPartWriter::close", uri_, start_ts, 0); + return ret; +} } }//oceanbase diff --git a/deps/oblib/src/lib/restore/ob_storage.h b/deps/oblib/src/lib/restore/ob_storage.h index d35493d74..8442be372 100644 --- a/deps/oblib/src/lib/restore/ob_storage.h +++ b/deps/oblib/src/lib/restore/ob_storage.h @@ -16,11 +16,21 @@ #include "ob_storage_file.h" #include "ob_storage_oss_base.h" #include "ob_storage_cos_base.h" +#include "ob_storage_s3_base.h" namespace oceanbase { namespace common { +/* In order to uniform naming format, here we will define the name format about uri/path. + * a. 'uri' represents a full path which has type prefix, like OSS/FILE. + * b. 'raw_dir_path' represents a dir path which does not have suffix '/' + * c. 'dir_path' represents a dir path, but we can't ensure that this path has suffix '/' or not + * d. 'full_dir_path' represents a dir path which has suffix '/' + * e. 'dir_name' represents a directory name, not a path + * f. 'obj_path' represents a object/file path + * g. 'obj_name' represents a object/file name, not a path + */ void print_access_storage_log(const char *msg, const common::ObString &uri, const int64_t start_ts, const int64_t size = 0, bool *is_slow = NULL); @@ -106,6 +116,42 @@ struct ObStorageObjectVersionParam { bool open_object_version_; }; +// If the object is 'SIMULATE_APPEND' type, we will use this operation to list all of its children objects. +class ListAppendableObjectFragmentOp : public common::ObBaseDirEntryOperator +{ +public: + ListAppendableObjectFragmentOp(const bool need_size = true) + : exist_format_meta_(false), exist_seal_meta_(false), meta_arr_(), need_size_(need_size) {} + + virtual ~ListAppendableObjectFragmentOp() { meta_arr_.reset(); } + virtual int func(const dirent *entry) override; + virtual bool need_get_file_size() const { return need_size_; } + int gen_object_meta(ObStorageObjectMeta &obj_meta); + + bool exist_format_meta() const { return exist_format_meta_; } + bool exist_seal_meta() const { return exist_seal_meta_; } + +private: + bool exist_format_meta_; + bool exist_seal_meta_; + ObArray meta_arr_; // save all 'data fragment meta' + bool need_size_; +}; + +// If the object is 'SIMULATE_APPEND' type, we will use this operation to delete all of its children objects. +class ObStorageUtil; +class DelAppendableObjectFragmentOp : public ObBaseDirEntryOperator +{ +public: + DelAppendableObjectFragmentOp(const common::ObString &uri, ObStorageUtil &util); + virtual ~DelAppendableObjectFragmentOp() {} + virtual int func(const dirent *entry) override; + +private: + const common::ObString &uri_; + ObStorageUtil &util_; +}; + class ObStorageUtil { public: @@ -115,26 +161,110 @@ public: virtual ~ObStorageUtil() {} int open(common::ObObjectStorageInfo *storage_info); void close(); + + ////////////////////// READY //// TO //// DROP ///// BELOW //////////////////////////////// int is_exist(const common::ObString &uri, bool &exist); int get_file_length(const common::ObString &uri, int64_t &file_length); int del_file(const common::ObString &uri); + int list_files(const common::ObString &dir_path, common::ObBaseDirEntryOperator &op); + int list_directories(const common::ObString &dir_path, common::ObBaseDirEntryOperator &op); + ////////////////////// READY //// TO //// DROP ///// ABOVE //////////////////////////////// + int mkdir(const common::ObString &uri); int write_single_file(const common::ObString &uri, const char *buf, const int64_t size); - int list_files(const common::ObString &dir_path, common::ObBaseDirEntryOperator &op); int del_dir(const common::ObString &uri); - int list_directories(const common::ObString &dir_path, common::ObBaseDirEntryOperator &op); int is_tagging(const common::ObString &uri, bool &is_tagging); + // This func is to check the object/file/dir exists or not. + // If the uri is a common directory(not a 'SIMULATE_APPEND' object), please set @is_adaptive as FALSE + // If the uri is a normal object, please set @is_adaptive as FALSE + // If the uri is a 'SIMULATE_APPEND' object or we can't ensure that it is a normal object or a + // 'SIMULATE_APPEND' object, please set @is_adaptive as TRUE. + int is_exist(const common::ObString &uri, const bool is_adaptive, bool &exist); + int get_file_length(const common::ObString &uri, const bool is_adaptive, int64_t &file_length); + int list_appendable_file_fragments(const common::ObString &uri, ObStorageObjectMeta &obj_meta); + + int del_file(const common::ObString &uri, const bool is_adaptive); + int del_unmerged_parts(const common::ObString &uri); + + // For one object, if given us the uri(no matter in oss, cos or s3), we can't tell the type of this object. + // It may be a 'single、normal' object. Or it may be a 's3-appendable-object'(like a dir), containing several + // 'single、normal' objects. + // So, this function is for checking the object meta, to get its meta info + // + // @uri, the object full path in object storage. + // @is_adaptive, if FALSE, means it is a normal object absolutely. + // if TRUE, means we don't know it type. We need to check its real type. + // @need_fragment_meta, if TRUE and the type is a 's3-appendable-object', we need to get its child objects meta. + // for example, when using adaptive reader, this param will set as TRUE; when using is_exist(), + // this param will set as FALSE + // @obj_meta the result, which saves the meta info of this object. If the target object not exists, we can check + // obj_meta.is_exist_, not return OB_BACKUP_FILE_NOT_EXIST. + int detect_storage_obj_meta(const common::ObString &uri, const bool is_adaptive, + const bool need_fragment_meta, ObStorageObjectMeta &obj_meta); + + // Due to the 'SIMULATE_APPEND' object and 'NORMAL' object may exist together, thus we can't simply list all objects + // based on the prefix. + // + // For example, + // dir1 + // --file1 + // --file2 + // --dir11 + // --file11 + // --file12 + // --appendable11 + // --@FORMAT_META + // --appendable1 + // --@FORMAT_META + // --@0-100 + // + // ['appendable1' and 'appendable11' are 'SIMULATE_APPEND' type]. If we want to list 'dir1/', we supposed to get the result as flows: + // dir1/file1, dir1/file2 + // dir1/dir11/file11, dir1/dir11/file12, dir1/dir11/appendable11 + // dir1/appendable1 + // Above 6 object paths are the final result. + // + // @is_adaptive If we can ensure that there not exist 'SIMULATE_APPEND' type object in @uri, we can set this param + // as FALSE, otherwise set it as TRUE. + int list_files(const common::ObString &uri, const bool is_adaptive, common::ObBaseDirEntryOperator &op); + int list_directories(const common::ObString &uri, const bool is_adaptive, common::ObBaseDirEntryOperator &op); private: // we does not use storage_info_ to judge init, since for nfs&local, storage_info_ is null - bool is_init() {return init_state;} + bool is_init() { return init_state; } + + // If there only exists common type object in this uri, this function will list all the files. + // If there also exists 'SIMULATE_APPEND' type object in this uri, this function will just list + // this 'appendable-dir' name, not include its children objects' name. + // + // NOTICE: children objects of 'appendable-dir' all have the same prefix(OB_S3_APPENDABLE_FRAGMENT_PREFIX). + // If there exists some children objects not have this prefix, these objects will also be listed. + // Cuz we think these objects are just some common objects. + int list_adaptive_files(const common::ObString &uri, common::ObBaseDirEntryOperator &op); + // ObjectStorage and Filesystem need to handle seperately. + int handle_listed_objs(ObStorageListCtxBase *ctx_base, const common::ObString &uri, + const common::ObString &dir_path, common::ObBaseDirEntryOperator &op); + int handle_listed_appendable_obj(ObStorageListObjectsCtx *list_ctx, const common::ObString &uri, + const common::ObString &dir_path, common::ObBaseDirEntryOperator &op); + int handle_listed_fs(ObStorageListCtxBase *ctx_base, const common::ObString &uri, + const common::ObString &dir_path, common::ObBaseDirEntryOperator &op); + + // For 'SIMULATE_APPEND' type file, if we want to get its file length, we can't get its length from object meta directly. + // <1> First, we need to check if there exists SEAL_META, if exists, read its content and get the file length + // <2> If not exists, we need to list all its children objects and get the file length + int get_adaptive_file_length(const common::ObString &uri, int64_t &file_length); + int read_seal_meta_if_needed(const common::ObString &uri, ObStorageObjectMeta &obj_meta); + + int del_appendable_file(const common::ObString &uri); ObStorageFileUtil file_util_; ObStorageOssUtil oss_util_; ObStorageCosUtil cos_util_; + ObStorageS3Util s3_util_; ObIStorageUtil* util_; common::ObObjectStorageInfo* storage_info_; bool init_state; + ObStorageType device_type_; DISALLOW_COPY_AND_ASSIGN(ObStorageUtil); }; @@ -143,36 +273,69 @@ class ObStorageReader public: ObStorageReader(); virtual ~ObStorageReader(); - int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info); - int pread(char *buf,const int64_t buf_size, int64_t offset, int64_t &read_size); + virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info); + int pread(char *buf, const int64_t buf_size, int64_t offset, int64_t &read_size); int close(); int64_t get_length() const { return file_length_; } -private: + +protected: int64_t file_length_; ObIStorageReader *reader_; ObStorageFileReader file_reader_; ObStorageOssReader oss_reader_; ObStorageCosReader cos_reader_; + ObStorageS3Reader s3_reader_; int64_t start_ts_; char uri_[OB_MAX_URI_LENGTH]; +private: DISALLOW_COPY_AND_ASSIGN(ObStorageReader); }; +// The most important meaning of this class is to read SIMULATE_APPEND file. +// But, if we use this class to read a normal object/file, it should also work well +class ObStorageAdaptiveReader +{ +public: + ObStorageAdaptiveReader(); + ~ObStorageAdaptiveReader(); + int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info); + int pread(char *buf, const int64_t buf_size, const int64_t offset, int64_t &read_size); + int close(); + int64_t get_length() const { return meta_.length_; } + +private: + ObArenaAllocator allocator_; + ObStorageObjectMeta meta_; + ObString bucket_; + ObString object_; + ObIStorageReader *reader_; + ObStorageFileReader file_reader_; + ObStorageOssReader oss_reader_; + ObStorageCosReader cos_reader_; + ObStorageS3Reader s3_reader_; + int64_t start_ts_; + char uri_[OB_MAX_URI_LENGTH]; + ObObjectStorageInfo *storage_info_; + DISALLOW_COPY_AND_ASSIGN(ObStorageAdaptiveReader); +}; + class ObStorageWriter { public: ObStorageWriter(); virtual ~ObStorageWriter(); - int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info); + virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info); int write(const char *buf,const int64_t size); int close(); -private: +protected: ObIStorageWriter *writer_; ObStorageFileWriter file_writer_; ObStorageOssWriter oss_writer_; ObStorageCosWriter cos_writer_; + ObStorageS3Writer s3_writer_; int64_t start_ts_; char uri_[OB_MAX_URI_LENGTH]; +private: DISALLOW_COPY_AND_ASSIGN(ObStorageWriter); }; @@ -190,28 +353,58 @@ public: }; int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info); - int write(const char *buf,const int64_t size); + int write(const char *buf, const int64_t size); int pwrite(const char *buf, const int64_t size, const int64_t offset); int close(); bool is_opened() const { return is_opened_; } int64_t get_length(); void set_open_mode(StorageOpenMode mode) {file_appender_.set_open_mode(mode);} - TO_STRING_KV(KP(appender_), K_(start_ts), K_(is_opened),KCSTRING_(uri)); + int seal_for_adaptive(); + TO_STRING_KV(KP_(appender), K_(start_ts), K_(is_opened), KCSTRING_(uri)); + private: ObIStorageWriter *appender_; ObStorageFileAppender file_appender_; ObStorageOssAppendWriter oss_appender_; ObStorageCosAppendWriter cos_appender_; + ObStorageS3AppendWriter s3_appender_; int64_t start_ts_; bool is_opened_; char uri_[OB_MAX_URI_LENGTH]; common::ObObjectStorageInfo storage_info_; ObArenaAllocator allocator_; + ObStorageType type_; int repeatable_pwrite_(const char *buf, const int64_t size, const int64_t offset); DISALLOW_COPY_AND_ASSIGN(ObStorageAppender); }; +class ObStorageMultiPartWriter +{ +public: + ObStorageMultiPartWriter(); + virtual ~ObStorageMultiPartWriter(); + virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info); + int write(const char *buf, const int64_t size); + int pwrite(const char *buf, const int64_t size, const int64_t offset); + int close(); + bool is_opened() const {return is_opened_;} + int64_t get_length(); + TO_STRING_KV(KP_(multipart_writer), K_(start_ts), K_(is_opened), KCSTRING_(uri)); + +protected: + ObIStorageWriter *multipart_writer_; + ObStorageFileMultiPartWriter file_multipart_writer_; + ObStorageCosMultiPartWriter cos_multipart_writer_; + ObStorageOssMultiPartWriter oss_multipart_writer_; + ObStorageS3MultiPartWriter s3_multipart_writer_; + int64_t start_ts_; + bool is_opened_; + char uri_[OB_MAX_URI_LENGTH]; + common::ObObjectStorageInfo storage_info_; + DISALLOW_COPY_AND_ASSIGN(ObStorageMultiPartWriter); +}; + }//common }//oceanbase diff --git a/deps/oblib/src/lib/restore/ob_storage_cos_base.cpp b/deps/oblib/src/lib/restore/ob_storage_cos_base.cpp index 3ba1c2d0a..2685d3fa2 100644 --- a/deps/oblib/src/lib/restore/ob_storage_cos_base.cpp +++ b/deps/oblib/src/lib/restore/ob_storage_cos_base.cpp @@ -52,65 +52,64 @@ void fin_cos_env() struct CosListFilesCbArg { - common::ObIAllocator &allocator; - ObString &dir_path; - ObBaseDirEntryOperator &list_op; + common::ObIAllocator &allocator_; + ObString &dir_path_; + ObBaseDirEntryOperator &list_op_; CosListFilesCbArg( - common::ObIAllocator &alloc, + common::ObIAllocator &allocator, ObString &dir, ObBaseDirEntryOperator &op) - : allocator(alloc), - dir_path(dir), - list_op(op) {} + : allocator_(allocator), + dir_path_(dir), + list_op_(op) {} ~CosListFilesCbArg() {} }; -static int execute_list_callback( - ObBaseDirEntryOperator &op, - qcloud_cos::ObCosWrapper::CosListObjPara ¶, - ObString &file_name) +struct CosListFilesCtx { - int ret = OB_SUCCESS; - int64_t dname_size = sizeof(para.last_container_name_.d_name); - if (dname_size >= (file_name.length() + 1)) { - if (OB_FAIL(databuff_printf(para.last_container_name_.d_name, dname_size, "%s", file_name.ptr()))) { - OB_LOG(WARN, "fail to copy file name to entry buf!", K(ret), K(dname_size)); - } else if (OB_FAIL(op.func(&(para.last_container_name_)))) { - OB_LOG(WARN, "fail to execute list callback!", K(ret), KCSTRING(para.last_container_name_.d_name), - K(para.last_container_name_.d_type), K(DT_REG), K(dname_size)); - } - } else { - ret = OB_BUF_NOT_ENOUGH; - OB_LOG(WARN, "file name len is too long!", K(ret), K(file_name.length()), K(sizeof(dirent))); - } - return ret; -} + common::ObIAllocator &allocator_; + ObString &dir_path_; + ObStorageListObjectsCtx &list_ctx_; + + CosListFilesCtx( + common::ObIAllocator &allocator, + ObString &dir, + ObStorageListObjectsCtx &ctx) + : allocator_(allocator), + dir_path_(dir), + list_ctx_(ctx) {} + + ~CosListFilesCtx() {} + + TO_STRING_KV(K_(dir_path), K_(list_ctx)); +}; static int handle_object_name_cb(qcloud_cos::ObCosWrapper::CosListObjPara ¶) { int ret = OB_SUCCESS; - para.next_flag_ = true; if (OB_ISNULL(para.arg_)) { ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "arg is empty", K(ret)); - } else if (OB_ISNULL(para.cur_full_path_slice_name_) || 0 >= para.full_path_size_) { + } else if (OB_ISNULL(para.cur_obj_full_path_)) { ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "current object_name is empty", K(ret)); + OB_LOG(WARN, "current object full path is empty", K(ret), K(para.cur_obj_full_path_)); } else if (qcloud_cos::ObCosWrapper::CosListObjPara::CosListType::COS_LIST_CB_ARG == para.type_) { CosListFilesCbArg *ctx = static_cast(para.arg_); // Returned object name is the whole object path, but we donot need the prefix dir_path. - // So, we trim the dir_name from object_name path. - const int dir_name_str_len = strlen(ctx->dir_path.ptr()); - const int object_name_len = para.full_path_size_ - dir_name_str_len; - const int object_name_start_pos = dir_name_str_len; - //tmp_file_name like slice name - ObString tmp_file_name(object_name_len, object_name_len, - para.cur_full_path_slice_name_ + object_name_start_pos); - if (OB_FAIL(execute_list_callback(ctx->list_op, para, tmp_file_name))) { - OB_LOG(WARN, "fail to execute list callback!", K(ret)); + // So, we trim the object full path to get object name + const int dir_name_str_len = strlen(ctx->dir_path_.ptr()); + int64_t object_size = -1; + if (OB_FAIL(c_str_to_int(para.cur_object_size_str_, object_size))) { + OB_LOG(WARN, "fail to get listed cos object size", K(ret), K(para.cur_object_size_str_)); + } else if (OB_FAIL(handle_listed_object(ctx->list_op_, + para.cur_obj_full_path_ + dir_name_str_len, + para.full_path_size_ - dir_name_str_len, + object_size))) { + OB_LOG(WARN, "fail to handle listed cos object", K(ret), K(para.cur_obj_full_path_), + K(dir_name_str_len), K(para.full_path_size_), K(object_size)); } } else { ret = OB_INVALID_ARGUMENT; @@ -120,39 +119,55 @@ static int handle_object_name_cb(qcloud_cos::ObCosWrapper::CosListObjPara ¶) return ret; } +static int handle_list_object_ctx(qcloud_cos::ObCosWrapper::CosListObjPara ¶) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(para.arg_)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "arg is empty", K(ret)); + } else if (qcloud_cos::ObCosWrapper::CosListObjPara::CosListType::COS_PART_LIST_CTX == para.type_) { + CosListFilesCtx *ctx = static_cast(para.arg_); + if (para.finish_part_list_) { + if (OB_FAIL(ctx->list_ctx_.set_next_token(para.next_flag_, + para.next_token_, + para.next_token_size_))) { + OB_LOG(WARN, "fail to set list ctx next token", + K(ret), K(para.next_flag_), K(para.next_token_), K(para.next_token_size_)); + } + } else { + int64_t object_size = -1; + if (OB_FAIL(c_str_to_int(para.cur_object_size_str_, object_size))) { + OB_LOG(WARN, "fail to get listed cos object size", K(ret), K(para.cur_object_size_str_)); + } else if (OB_FAIL(ctx->list_ctx_.handle_object(para.cur_obj_full_path_, + para.full_path_size_, + object_size))) { + OB_LOG(WARN, "fail to add listed cos obejct meta into list ctx", + K(ret), K(para.cur_obj_full_path_), K(para.full_path_size_), K(object_size)); + } + } + } else { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "not supported type", K(ret), K(para.type_)); + } + return ret; +} + static int handle_directory_name_cb( void *arg, const qcloud_cos::ObCosWrapper::CosListObjPara::CosListType type, - const char *object_name, - int64_t object_size) + const char *dir_name, + int64_t dir_name_len) { int ret = OB_SUCCESS; - const char delimiter = '/'; if (OB_ISNULL(arg)) { ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "arg is empty", K(ret), KP(arg)); - } else if (OB_ISNULL(object_name) || 0 >= object_size) { - ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "object_name is empty", K(ret), KP(object_name), K(object_size)); - } else if (delimiter != object_name[object_size - 1]) { - ret = OB_ERR_UNEXPECTED; - OB_LOG(WARN, "object name is unexpected", K(ret), KCSTRING(object_name), K(object_size)); } else { - const int64_t name_length = object_size - 1; - if (name_length <= 0) { - ret = OB_ERR_UNEXPECTED; - OB_LOG(WARN, "object is not exist", K(ret), KCSTRING(object_name), K(object_size)); - } else if (qcloud_cos::ObCosWrapper::CosListObjPara::CosListType::COS_LIST_CB_ARG == type) { + if (qcloud_cos::ObCosWrapper::CosListObjPara::CosListType::COS_LIST_CB_ARG == type) { CosListFilesCbArg *ctx = static_cast(arg); - ObString directory_name; - dirent dir_name_entry; - dir_name_entry.d_type = DT_DIR; - ObString tmp_directory_name(name_length, name_length, object_name); - if (OB_FAIL(databuff_printf(dir_name_entry.d_name, sizeof(dir_name_entry.d_name), "%.*s", - tmp_directory_name.length(), tmp_directory_name.ptr()))) { - OB_LOG(WARN, "fail to set dir name entry", K(ret), K(tmp_directory_name)); - } else if (OB_FAIL(ctx->list_op.func(&dir_name_entry))) { - OB_LOG(WARN, "fail to call list directories operation callback", K(ret), K(tmp_directory_name)); + if (OB_FAIL(handle_listed_directory(ctx->list_op_, dir_name, dir_name_len))) { + OB_LOG(WARN, "fail to handle cos directory name", + K(ret), K(dir_name), K(dir_name_len)); } } else { ret = OB_INVALID_ARGUMENT; @@ -198,37 +213,44 @@ void ObStorageCosUtil::close() int ObStorageCosUtil::is_exist(const ObString &uri, bool &is_exist) { int ret = OB_SUCCESS; - ObExternalIOCounterGuard io_guard; is_exist = false; - int64_t file_length = 0; - if (OB_UNLIKELY(!is_opened_)) { - ret = OB_COS_ERROR; - OB_LOG(WARN, "cos util not opened", K(ret)); - } else if (OB_UNLIKELY(uri.empty())) { - ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "invalid argument", K(ret), K(uri)); - } else if (OB_FAIL(get_object_meta_(uri, is_exist, file_length))) { - OB_LOG(WARN, "fail to get object meta", K(ret), K(uri)); + ObStorageObjectMetaBase obj_meta; + if (OB_FAIL(head_object_meta(uri, obj_meta))) { + OB_LOG(WARN, "fail to head object meta", K(ret), K(uri)); + } else { + is_exist = obj_meta.is_exist_; } return ret; } int ObStorageCosUtil::get_file_length(const ObString &uri, int64_t &file_length) +{ + int ret = OB_SUCCESS; + file_length = 0; + ObStorageObjectMetaBase obj_meta; + if (OB_FAIL(head_object_meta(uri, obj_meta))) { + OB_LOG(WARN, "fail to head object meta", K(ret), K(uri)); + } else if (!obj_meta.is_exist_) { + ret = OB_BACKUP_FILE_NOT_EXIST; + OB_LOG(WARN, "backup file is not exist", K(ret), K(uri)); + } else { + file_length = obj_meta.length_; + } + return ret; +} + +int ObStorageCosUtil::head_object_meta(const ObString &uri, ObStorageObjectMetaBase &obj_meta) { int ret = OB_SUCCESS; ObExternalIOCounterGuard io_guard; - bool is_file_exist = false; if (OB_UNLIKELY(!is_opened_)) { ret = OB_COS_ERROR; OB_LOG(WARN, "cos util not opened", K(ret)); } else if (OB_UNLIKELY(uri.empty())) { ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "invalid argument", K(ret), K(uri)); - } else if (OB_FAIL(get_object_meta_(uri, is_file_exist, file_length))) { + } else if (OB_FAIL(get_object_meta_(uri, obj_meta.is_exist_, obj_meta.length_))) { OB_LOG(WARN, "fail to get object meta", K(ret), K(uri)); - } else if (!is_file_exist) { - ret = OB_BACKUP_FILE_NOT_EXIST; - OB_LOG(WARN, "backup file is not exist", K(ret), K(uri)); } return ret; } @@ -346,8 +368,9 @@ int ObStorageCosUtil::list_files( ObBaseDirEntryOperator &op) { int ret = OB_SUCCESS; + ObStorageCosBase cos_base; ObExternalIOCounterGuard io_guard; - ObArenaAllocator allocator; + ObArenaAllocator allocator(ObModIds::BACKUP); if (OB_UNLIKELY(!is_opened_)) { ret = OB_COS_ERROR; @@ -355,61 +378,107 @@ int ObStorageCosUtil::list_files( } else if (OB_UNLIKELY(uri.empty())) { ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "invalid argument", K(ret), K(uri)); + } else if (OB_FAIL(cos_base.open(uri, storage_info_))) { + OB_LOG(WARN, "fail to open cos base", K(ret), K(uri), KPC_(storage_info)); } else { - char dir_path_array[OB_MAX_URI_LENGTH] = {0}; - const char *ptr = NULL; // point to actual dir path - const int min_dir_path_len = 2; // cuz dir_path end with '\0' - const char *separator = "/"; - const char end_marker = '\0'; + const char *full_dir_path = cos_base.get_handle().get_object_name().ptr(); + const int64_t full_dir_path_len = cos_base.get_handle().get_object_name().length(); + ObString full_dir_path_str(full_dir_path_len, full_dir_path); - ObStorageCosBase cos_base; - if (OB_FAIL(cos_base.open(uri, storage_info_))) { - OB_LOG(WARN, "fail to open cos base", K(ret), K(uri)); - } else { - const ObString &dir_path_str = cos_base.get_handle().get_object_name(); - const ObString::obstr_size_t dir_path_len = dir_path_str.length(); - int32_t actual_dir_path_len = dir_path_len; - if (dir_path_len < min_dir_path_len) { - ret = OB_ERR_UNEXPECTED; - OB_LOG(WARN, "dir_path_str length is unepxected", K(ret), K(dir_path_str), K(dir_path_len)); - } else { - // Let dir path end with '/' - if (dir_path_str.ptr()[dir_path_len - 1] != end_marker) { - ret = OB_ERR_UNEXPECTED; - OB_LOG(WARN, "dir_path_str not end with '\0'"); - } else if (dir_path_str.ptr()[dir_path_len - 2] == *separator) { - ptr = dir_path_str.ptr(); - } else if (OB_FAIL(databuff_printf(dir_path_array, OB_MAX_URI_LENGTH, "%s/", dir_path_str.ptr()))) { - OB_LOG(WARN, "fail to deep copy dir path", K(ret), K(dir_path_len), K(OB_MAX_URI_LENGTH)); - } else { - ptr = dir_path_array; - actual_dir_path_len += 1; - } - } - - if (OB_SUCC(ret)) { - ObString tmp_dir; - tmp_dir.assign(const_cast(ptr), actual_dir_path_len); - - // Construct list object callback arg - CosListFilesCbArg arg(allocator, tmp_dir, op); - if (OB_FAIL(cos_base.list_objects(uri, tmp_dir, separator, arg))) { - OB_LOG(WARN, "fail to list object in cos_base", K(ret), K(uri), K(tmp_dir)); - } - } + // Construct list object callback arg + CosListFilesCbArg arg(allocator, full_dir_path_str, op); + if (OB_UNLIKELY(!is_end_with_slash(full_dir_path))) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "uri is not terminated with '/'", K(ret), K(uri), K(full_dir_path)); + } else if (OB_FAIL(cos_base.list_objects(uri, full_dir_path_str, arg))) { + OB_LOG(WARN, "fail to list object in cos_base", K(ret), K(uri), K(full_dir_path_str)); } - cos_base.reset(); } + + cos_base.reset(); + return ret; +} + +int ObStorageCosUtil::list_files( + const ObString &uri, + ObStorageListCtxBase &ctx_base) +{ + int ret = OB_SUCCESS; + ObStorageCosBase cos_base; + ObExternalIOCounterGuard io_guard; + ObArenaAllocator allocator(ObModIds::BACKUP); + ObStorageListObjectsCtx &list_ctx = static_cast(ctx_base); + + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_COS_ERROR; + OB_LOG(WARN, "cos util not opened", K(ret)); + } else if (OB_UNLIKELY(uri.empty() || !list_ctx.is_valid())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid argument", K(ret), K(uri), K(list_ctx)); + } else if (OB_FAIL(cos_base.open(uri, storage_info_))) { + OB_LOG(WARN, "fail to open cos base", K(ret), K(uri), KPC_(storage_info)); + } else { + const char *full_dir_path = cos_base.get_handle().get_object_name().ptr(); + const int64_t full_dir_path_len = cos_base.get_handle().get_object_name().length(); + ObString full_dir_path_str(full_dir_path_len, full_dir_path); + + // Construct list object context + CosListFilesCtx arg(allocator, full_dir_path_str, list_ctx); + if (OB_UNLIKELY(!is_end_with_slash(full_dir_path))) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "uri is not terminated with '/'", K(ret), K(uri), K(full_dir_path)); + } else if (OB_FAIL(cos_base.list_objects(uri, full_dir_path_str, list_ctx.next_token_, arg))) { + OB_LOG(WARN, "fail to list object in cos_base", + K(ret), K(list_ctx), K(uri), K(full_dir_path_str)); + } + } + + cos_base.reset(); return ret; } int ObStorageCosUtil::list_directories( const ObString &uri, ObBaseDirEntryOperator &op) +{ + int ret = OB_SUCCESS; + ObStorageCosBase cos_base; + ObExternalIOCounterGuard io_guard; + ObArenaAllocator allocator(ObModIds::BACKUP); + + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_COS_ERROR; + OB_LOG(WARN, "cos util not opened", K(ret)); + } else if (OB_UNLIKELY(uri.empty())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid argument", K(ret), K(uri)); + } else if (OB_FAIL(cos_base.open(uri, storage_info_))) { + OB_LOG(WARN, "fail to open cos base", K(ret), K(uri), KPC_(storage_info)); + } else { + const char *delimiter_string = "/"; + const char *next_marker_string = ""; + const char *full_dir_path = cos_base.get_handle().get_object_name().ptr(); + const int64_t full_dir_path_len = cos_base.get_handle().get_object_name().length(); + ObString full_dir_path_str(full_dir_path_len, full_dir_path); + + CosListFilesCbArg arg(allocator, full_dir_path_str, op); + if (OB_UNLIKELY(!is_end_with_slash(full_dir_path))) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "uri is not terminated with '/'", K(ret), K(uri), K(full_dir_path)); + } else if (OB_FAIL(cos_base.list_directories(uri, full_dir_path_str, + next_marker_string, delimiter_string, arg))) { + OB_LOG(WARN, "fail to list directories in cos_base", K(ret), K(uri), K(full_dir_path_str)); + } + } + + cos_base.reset(); + return ret; +} + +int ObStorageCosUtil::del_unmerged_parts(const ObString &uri) { int ret = OB_SUCCESS; ObExternalIOCounterGuard io_guard; - ObArenaAllocator allocator; if (OB_UNLIKELY(!is_opened_)) { ret = OB_COS_ERROR; @@ -418,47 +487,13 @@ int ObStorageCosUtil::list_directories( ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "invalid argument", K(ret), K(uri)); } else { - char dir_path_array[OB_MAX_URI_LENGTH] = {0}; - const char *ptr = NULL; // point to actual dir path - const int min_dir_path_len = 2; // cuz dir_path end with '\0' - const char *delimiter_string = "/"; - const char *next_marker_string = ""; - const char end_marker = '\0'; - ObStorageCosBase cos_base; if (OB_FAIL(cos_base.open(uri, storage_info_))) { OB_LOG(WARN, "fail to open cos base", K(ret), K(uri)); + } else if (OB_FAIL(cos_base.del_unmerged_parts(uri))) { + OB_LOG(WARN, "fail to del unmerged parts", K(ret), K(uri)); } else { - const ObString &dir_path_str = cos_base.get_handle().get_object_name(); - const ObString::obstr_size_t dir_path_len = dir_path_str.length(); - int32_t actual_dir_path_len = dir_path_len; - if (dir_path_len < min_dir_path_len) { - ret = OB_ERR_UNEXPECTED; - OB_LOG(WARN, "dir_path_str length is unepxected", K(ret), K(dir_path_str), K(dir_path_len)); - } else { - // Let dir path end with '/' - if (dir_path_str.ptr()[dir_path_len - 1] != end_marker) { - ret = OB_ERR_UNEXPECTED; - OB_LOG(WARN, "dir_path_str not end with '\0'"); - } else if (dir_path_str.ptr()[dir_path_len - 2] == *delimiter_string) { - ptr = dir_path_str.ptr(); - } else if (OB_FAIL(databuff_printf(dir_path_array, OB_MAX_URI_LENGTH, "%s/", dir_path_str.ptr()))) { - OB_LOG(WARN, "fail to deep copy dir path", K(ret), K(dir_path_len), K(OB_MAX_URI_LENGTH)); - } else { - ptr = dir_path_array; - actual_dir_path_len += 1; - } - } - - if (OB_SUCC(ret)) { - ObString tmp_dir; - tmp_dir.assign(const_cast(ptr), actual_dir_path_len); - - CosListFilesCbArg arg(allocator, tmp_dir, op); - if (OB_FAIL(cos_base.list_directories(uri, tmp_dir, next_marker_string, delimiter_string, arg))) { - OB_LOG(WARN, "fail to list directories in cos_base", K(ret), K(uri), K(tmp_dir)); - } - } + OB_LOG(DEBUG, "succ to delete object", K(uri)); } cos_base.reset(); } @@ -479,7 +514,9 @@ ObStorageCosBase::~ObStorageCosBase() void ObStorageCosBase::reset() { - handle_.reset(); + if (is_opened_) { + handle_.reset(); + } is_opened_ = false; } @@ -566,22 +603,44 @@ int ObStorageCosBase::delete_object(const ObString &uri) int ObStorageCosBase::list_objects( const ObString &uri, - const ObString &dir_name_str, - const char *separator, + const ObString &full_dir_path_str, CosListFilesCbArg &arg) { int ret = OB_SUCCESS; - if (OB_ISNULL(separator)) { + if (OB_LIKELY(is_valid())) { + qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer( + handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length()); + qcloud_cos::CosStringBuffer full_dir_path(full_dir_path_str.ptr(), full_dir_path_str.length()); + if (OB_FAIL(qcloud_cos::ObCosWrapper::list_objects(handle_.get_ptr(), + bucket_name, full_dir_path, handle_object_name_cb, (void *)(&arg)))) { + OB_LOG(WARN, "fail to list objects", K(ret), K(uri)); + } + } else { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "cos wrapper handle not init or create", K(ret)); + } + return ret; +} + +int ObStorageCosBase::list_objects( + const ObString &uri, + const ObString &full_dir_path_str, + const char *next_token, + CosListFilesCtx &ctx) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(next_token)) { ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "invalid argument", K(ret)); } else if (OB_LIKELY(is_valid())) { qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer( handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length()); - qcloud_cos::CosStringBuffer dir_name(dir_name_str.ptr(), dir_name_str.length()); - qcloud_cos::CosStringBuffer next_marker(separator, strlen(separator) + 1); - if (OB_FAIL(qcloud_cos::ObCosWrapper::list_objects(handle_.get_ptr(), bucket_name, - dir_name, next_marker, handle_object_name_cb, (void *)(&arg)))) { - OB_LOG(WARN, "fail to list objects", K(ret), K(uri)); + qcloud_cos::CosStringBuffer full_dir_path(full_dir_path_str.ptr(), full_dir_path_str.length()); + qcloud_cos::CosStringBuffer next_marker(next_token, strlen(next_token)); + + if (OB_FAIL(qcloud_cos::ObCosWrapper::list_part_objects(handle_.get_ptr(), bucket_name, + full_dir_path, next_marker, handle_list_object_ctx, (void *)(&ctx)))) { + OB_LOG(WARN, "fail to list part objects", K(ret), K(uri), K(next_token), K(ctx)); } } else { ret = OB_ERR_UNEXPECTED; @@ -592,7 +651,7 @@ int ObStorageCosBase::list_objects( int ObStorageCosBase::list_directories( const ObString &uri, - const ObString &dir_name_str, + const ObString &full_dir_path_str, const char *next_marker_str, const char *delimiter_str, CosListFilesCbArg &arg) @@ -604,12 +663,14 @@ int ObStorageCosBase::list_directories( } else if (OB_LIKELY(is_valid())) { qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer( handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length()); - qcloud_cos::CosStringBuffer dir_name(dir_name_str.ptr(), dir_name_str.length()); + qcloud_cos::CosStringBuffer full_dir_path(full_dir_path_str.ptr(), full_dir_path_str.length()); qcloud_cos::CosStringBuffer next_marker(next_marker_str, strlen(next_marker_str) + 1); qcloud_cos::CosStringBuffer delimiter(delimiter_str, strlen(delimiter_str) + 1); - if (OB_FAIL(qcloud_cos::ObCosWrapper::list_directories(handle_.get_ptr(), bucket_name, dir_name, - next_marker, delimiter, handle_directory_name_cb, (void *)(&arg)))) { - OB_LOG(WARN, "failed to list directories", K(ret), K(uri)); + + if (OB_FAIL(qcloud_cos::ObCosWrapper::list_directories(handle_.get_ptr(), bucket_name, + full_dir_path, next_marker, delimiter, handle_directory_name_cb, (void *)(&arg)))) { + OB_LOG(WARN, "failed to list directories", + K(ret), K(uri), K(next_marker_str), K(delimiter_str)); } } else { ret = OB_ERR_UNEXPECTED; @@ -639,10 +700,29 @@ int ObStorageCosBase::is_object_tagging( return ret; } +int ObStorageCosBase::del_unmerged_parts(const ObString &uri) +{ + int ret = OB_SUCCESS; + if (OB_LIKELY(is_valid())) { + qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer( + handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length()); + qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer( + handle_.get_object_name().ptr(), handle_.get_object_name().length()); + if (OB_FAIL(qcloud_cos::ObCosWrapper::del_unmerged_parts(handle_.get_ptr(), + bucket_name, object_name))) { + OB_LOG(WARN, "fail to del unmerged parts", K(ret), K(uri)); + } + } else { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "cos wrapper handle not init or create", K(ret)); + } + return ret; +} + /*--------------------------------ObStorageCosReader---------------------------*/ ObStorageCosReader::ObStorageCosReader() - : ObStorageCosBase(), file_length_(-1) + : ObStorageCosBase(), has_meta_(false), file_length_(-1) { } @@ -650,7 +730,8 @@ ObStorageCosReader::~ObStorageCosReader() { } -int ObStorageCosReader::open(const ObString &uri, ObObjectStorageInfo *storage_info) +int ObStorageCosReader::open(const ObString &uri, + ObObjectStorageInfo *storage_info, const bool head_meta) { int ret = OB_SUCCESS; ObExternalIOCounterGuard io_guard; @@ -661,22 +742,28 @@ int ObStorageCosReader::open(const ObString &uri, ObObjectStorageInfo *storage_i } else if (OB_FAIL(ObStorageCosBase::open(uri, storage_info))) { OB_LOG(WARN, "fail to open in cos_base", K(ret), K(uri)); } else { - const ObString &bucket_str = handle_.get_bucket_name(); - const ObString &object_str = handle_.get_object_name(); - qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer( - bucket_str.ptr(), bucket_str.length()); - qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer( - object_str.ptr(), object_str.length()); - bool is_file_exist = false; - qcloud_cos::CosObjectMeta obj_meta; - if (OB_FAIL(qcloud_cos::ObCosWrapper::head_object_meta(handle_.get_ptr(), bucket_name, - object_name, is_file_exist, obj_meta))) { - OB_LOG(WARN, "fail to get object meta", K(ret), K(bucket_str), K(object_str)); - } else if (!is_file_exist) { - ret = OB_BACKUP_FILE_NOT_EXIST; - OB_LOG(WARN, "backup file is not exist", K(ret), K(bucket_str), K(object_str)); - } else { - file_length_ = obj_meta.file_length_; + if (head_meta) { + const ObString &bucket_str = handle_.get_bucket_name(); + const ObString &object_str = handle_.get_object_name(); + qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer( + bucket_str.ptr(), bucket_str.length()); + qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer( + object_str.ptr(), object_str.length()); + bool is_file_exist = false; + qcloud_cos::CosObjectMeta obj_meta; + if (OB_FAIL(qcloud_cos::ObCosWrapper::head_object_meta(handle_.get_ptr(), bucket_name, + object_name, is_file_exist, obj_meta))) { + OB_LOG(WARN, "fail to get object meta", K(ret), K(bucket_str), K(object_str)); + } else if (!is_file_exist) { + ret = OB_BACKUP_FILE_NOT_EXIST; + OB_LOG(WARN, "backup file is not exist", K(ret), K(bucket_str), K(object_str)); + } else { + file_length_ = obj_meta.file_length_; + has_meta_ = true; + } + } + + if (OB_SUCC(ret)) { is_opened_ = true; } } @@ -695,7 +782,7 @@ int ObStorageCosReader::pread( if (!is_opened_) { ret = OB_COS_ERROR; OB_LOG(WARN, "cos reader cannot read before it is opened", K(ret)); - } else if (NULL == buf || buf_size <= 0 || offset < 0) { + } else if (OB_ISNULL(buf) || OB_UNLIKELY(buf_size <= 0 || offset < 0)) { ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "invalid argument", K(ret), KP(buf), K(buf_size), K(offset)); } else { @@ -853,9 +940,9 @@ int ObStorageCosAppendWriter::pwrite(const char *buf, const int64_t size, const if(OB_UNLIKELY(!is_opened_)) { ret = OB_COS_ERROR; OB_LOG(WARN, "cos append writer cannot write before it is not opened", K(ret)); - } else if(NULL == buf || size < 0 || offset < 0) { + } else if(NULL == buf || size <= 0 || offset < 0) { ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "buf is NULL or size is invalid", KP(buf), K(size), K(ret)); + OB_LOG(WARN, "invalid arguments", KP(buf), K(size), K(ret), K(offset)); } else if (OB_FAIL(do_write(buf, size, offset, is_pwrite))) { OB_LOG(WARN, "failed to do write", K(ret), KP(buf), K(size), K(offset)); } @@ -930,5 +1017,219 @@ int ObStorageCosAppendWriter::do_write( return ret; } +/*------------------------------ObStorageCosMultiPartWriter---------------------------*/ +ObStorageCosMultiPartWriter::ObStorageCosMultiPartWriter() + : ObStorageCosBase(), + mod_(ObModIds::BACKUP), + allocator_(ModuleArena::DEFAULT_PAGE_SIZE, mod_), + base_buf_(NULL), + base_buf_pos_(0), + upload_id_(NULL), + partnum_(0), + file_length_(-1) +{} + +ObStorageCosMultiPartWriter::~ObStorageCosMultiPartWriter() +{ + destroy(); +} + +void ObStorageCosMultiPartWriter::reuse() +{ + if (is_opened_) { + if (nullptr != upload_id_) { + handle_.free_mem(static_cast(upload_id_)); + } + if (nullptr != base_buf_) { + handle_.free_mem(static_cast(base_buf_)); + } + } + upload_id_ = nullptr; + base_buf_ = nullptr; + partnum_ = 0; + file_length_ = -1; + ObStorageCosBase::reset(); +} + +int ObStorageCosMultiPartWriter::open(const ObString &uri, common::ObObjectStorageInfo *storage_info) +{ + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + if (OB_UNLIKELY(is_opened_)) { + ret = OB_COS_ERROR; + OB_LOG(WARN, "already open, cannot open again", K(ret)); + } else if (OB_UNLIKELY(uri.empty())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "uri is empty", K(ret), K(uri)); + } else if (OB_FAIL(ObStorageCosBase::open(uri, storage_info))) { + OB_LOG(WARN, "fail to open in cos_base", K(ret), K(uri)); + } else { + const ObString &bucket_name_str = handle_.get_bucket_name(); + const ObString &object_name_str = handle_.get_object_name(); + qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer( + bucket_name_str.ptr(), handle_.get_bucket_name().length()); + qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer( + object_name_str.ptr(), handle_.get_object_name().length()); + + if (OB_FAIL(qcloud_cos::ObCosWrapper::init_multipart_upload(handle_.get_ptr(), + bucket_name, object_name, upload_id_))) { + OB_LOG(WARN, "fail to init multipartupload", K(ret), K(bucket_name_str), K(object_name_str)); + } else if (OB_ISNULL(upload_id_)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "upload_id should not be null", K(ret)); + } else if (OB_ISNULL(base_buf_ = static_cast(handle_.alloc_mem(COS_MULTIPART_UPLOAD_BUF_SIZE)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "fail to alloc buffer for cos multipartupload", K(ret)); + } else { + is_opened_ = true; + base_buf_pos_ = 0; + file_length_ = 0; + } + } + return ret; +} + +int ObStorageCosMultiPartWriter::write(const char * buf, const int64_t size) +{ + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + int64_t fill_size = 0; + int64_t buf_pos = 0; + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_COS_ERROR; + OB_LOG(WARN, "write cos should open first", K(ret)); + } else if (OB_ISNULL(buf) || OB_UNLIKELY(size < 0)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "buf is NULL or size is invalid", K(ret), KP(buf), K(size)); + } + + while (OB_SUCC(ret) && buf_pos != size) { + fill_size = std::min(COS_MULTIPART_UPLOAD_BUF_SIZE - base_buf_pos_, size - buf_pos); + memcpy(base_buf_ + base_buf_pos_, buf + buf_pos, fill_size); + base_buf_pos_ += fill_size; + buf_pos += fill_size; + if (base_buf_pos_ == COS_MULTIPART_UPLOAD_BUF_SIZE) { + if (OB_FAIL(write_single_part())) { + OB_LOG(WARN, "fail to write part into cos", K(ret)); + } else { + base_buf_pos_ = 0; + } + } + } + + // actually, current file size may be smaller than @size. Cuz we may not finish + // the complete multipartupload. + if (OB_SUCC(ret)) { + file_length_ += size; + } + return ret; +} + +int ObStorageCosMultiPartWriter::pwrite(const char *buf, const int64_t size, const int64_t offset) +{ + UNUSED(offset); + return write(buf, size); +} + +int ObStorageCosMultiPartWriter::write_single_part() +{ + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + + ++partnum_; + if (partnum_ > COS_MAX_PART_NUM) { + ret = OB_OUT_OF_ELEMENT; + OB_LOG(WARN, "Out of cos element ", K(ret), K_(partnum)); + } else if (OB_UNLIKELY(!is_opened_)) { + ret = OB_COS_ERROR; + OB_LOG(WARN, "write cos should open first", K(ret)); + } else { + qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer( + handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length()); + qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer( + handle_.get_object_name().ptr(), handle_.get_object_name().length()); + qcloud_cos::CosStringBuffer upload_id_str = qcloud_cos::CosStringBuffer( + upload_id_, strlen(upload_id_)); + if (OB_FAIL(qcloud_cos::ObCosWrapper::upload_part_from_buffer(handle_.get_ptr(), bucket_name, + object_name, upload_id_str, partnum_, base_buf_, base_buf_pos_))) { + OB_LOG(WARN, "fail to upload part to cos", K(ret), KP_(upload_id)); + if (OB_TMP_FAIL(cleanup())) { + OB_LOG(WARN, "fail to abort multiupload", K(ret), K(tmp_ret), KP_(upload_id)); + } + } + } + return ret; +} + +int ObStorageCosMultiPartWriter::close() +{ + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + const int64_t start_time = ObTimeUtility::current_time(); + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_COS_ERROR; + OB_LOG(WARN, "cos writer cannot close before it is opened", K(ret)); + } else if (0 != base_buf_pos_) { + if(OB_SUCCESS != (ret = write_single_part())) { + OB_LOG(WARN, "fail to write the last size to cos", K(ret), K_(base_buf_pos)); + if (OB_TMP_FAIL(cleanup())) { + OB_LOG(WARN, "fail to abort multiupload", K(ret), K(tmp_ret), KP_(upload_id)); + } + ret = OB_COS_ERROR; + } else { + base_buf_pos_ = 0; + } + } + + if (OB_SUCC(ret)) { + qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer( + handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length()); + qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer( + handle_.get_object_name().ptr(), handle_.get_object_name().length()); + qcloud_cos::CosStringBuffer upload_id_str = qcloud_cos::CosStringBuffer( + upload_id_, strlen(upload_id_)); + + if (OB_FAIL(qcloud_cos::ObCosWrapper::complete_multipart_upload(handle_.get_ptr(), bucket_name, + object_name, upload_id_str))) { + OB_LOG(WARN, "fail to complete multipart upload", K(ret), KP_(upload_id)); + if (OB_TMP_FAIL(cleanup())) { + OB_LOG(WARN, "fail to abort multiupload", K(ret), K(tmp_ret), KP_(upload_id)); + } + } + } + + reuse(); + + const int64_t total_cost_time = ObTimeUtility::current_time() - start_time; + if (total_cost_time > 3 * 1000 * 1000) { + OB_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "cos writer close cost too much time", K(total_cost_time), K(ret)); + } + return ret; +} + +int ObStorageCosMultiPartWriter::cleanup() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_COS_ERROR; + OB_LOG(WARN, "cos multipart writer cannot cleanup before it is opened", K(ret)); + } else { + qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer( + handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length()); + qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer( + handle_.get_object_name().ptr(), handle_.get_object_name().length()); + qcloud_cos::CosStringBuffer upload_id_str = qcloud_cos::CosStringBuffer( + upload_id_, strlen(upload_id_)); + + if (OB_FAIL(qcloud_cos::ObCosWrapper::abort_multipart_upload(handle_.get_ptr(), bucket_name, + object_name, upload_id_str))) { + OB_LOG(WARN, "fail to abort multipart upload", K(ret), KP_(upload_id)); + } + } + return ret; +} + } //common } //oceanbase diff --git a/deps/oblib/src/lib/restore/ob_storage_cos_base.h b/deps/oblib/src/lib/restore/ob_storage_cos_base.h index 667985712..5d2caff69 100644 --- a/deps/oblib/src/lib/restore/ob_storage_cos_base.h +++ b/deps/oblib/src/lib/restore/ob_storage_cos_base.h @@ -26,6 +26,7 @@ namespace common { class ObStorageCosBase; struct CosListFilesCbArg; +struct CosListFilesCtx; // Before using cos, you need to initialize cos enviroment. // Thread safe guaranteed by user. @@ -44,6 +45,7 @@ public: virtual void close(); virtual int is_exist(const common::ObString &uri, bool &is_exist); virtual int get_file_length(const common::ObString &uri, int64_t &file_length); + virtual int head_object_meta(const common::ObString &uri, ObStorageObjectMetaBase &obj_meta); virtual int write_single_file(const common::ObString &uri, const char *buf, const int64_t size); @@ -51,9 +53,11 @@ public: virtual int mkdir(const common::ObString &uri); virtual int del_file(const common::ObString &uri); virtual int list_files(const common::ObString &uri, common::ObBaseDirEntryOperator &op); + virtual int list_files(const common::ObString &uri, ObStorageListCtxBase &list_ctx); virtual int del_dir(const common::ObString &uri); virtual int list_directories(const common::ObString &uri, common::ObBaseDirEntryOperator &op); virtual int is_tagging(const common::ObString &uri, bool &is_tagging); + virtual int del_unmerged_parts(const ObString &uri) override; private: int get_object_meta_(const common::ObString &uri, bool &is_file_exist, int64_t &file_length); @@ -75,11 +79,14 @@ public: // some cos function int get_cos_file_meta(bool &is_file_exist, common::qcloud_cos::CosObjectMeta &obj_meta); int delete_object(const common::ObString &uri); + int list_objects(const common::ObString &uri, + const common::ObString &dir_name_str, common::CosListFilesCbArg &arg); int list_objects(const common::ObString &uri, const common::ObString &dir_name_str, - const char *separator, common::CosListFilesCbArg &arg); + const char *next_token, common::CosListFilesCtx &ctx); int list_directories(const common::ObString &uri, const common::ObString &dir_name_str, const char *next_marker_str, const char *delimiter_str, common::CosListFilesCbArg &arg); int is_object_tagging(const common::ObString &uri, bool &is_tagging); + int del_unmerged_parts(const ObString &uri); private: int init_handle(const common::ObObjectStorageInfo &storage_info); @@ -114,13 +121,16 @@ class ObStorageCosReader: public ObStorageCosBase, public ObIStorageReader public: ObStorageCosReader(); virtual ~ObStorageCosReader(); - virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info) override; - int pread(char *buf, const int64_t buf_size, int64_t offset, int64_t &read_size); + virtual int open(const common::ObString &uri, + common::ObObjectStorageInfo *storage_info, const bool head_meta = true) override; + virtual int pread(char *buf, + const int64_t buf_size, const int64_t offset, int64_t &read_size) override; int close(); int64_t get_length() const { return file_length_; } bool is_opened() const { return is_opened_; } private: + bool has_meta_; int64_t file_length_; DISALLOW_COPY_AND_ASSIGN(ObStorageCosReader); @@ -149,6 +159,42 @@ private: DISALLOW_COPY_AND_ASSIGN(ObStorageCosAppendWriter); }; +// part size is in [1MB, 5GB], exclude the last part +// max part num 10000 +class ObStorageCosMultiPartWriter: public ObStorageCosBase, public ObIStorageWriter +{ +public: + ObStorageCosMultiPartWriter(); + virtual ~ObStorageCosMultiPartWriter(); + int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info); + int write(const char *buf,const int64_t size); + int pwrite(const char *buf, const int64_t size, const int64_t offset); + int close(); + int cleanup(); + int64_t get_length() const { return file_length_; } + virtual bool is_opened() const { return is_opened_; } + +private: + void reuse(); + void destroy() { reuse(); } + int write_single_part(); + +private: + const static int64_t COS_MAX_PART_NUM = 10000; + const static int64_t COS_MULTIPART_UPLOAD_BUF_SIZE = 8 * 1024 * 1024L; + +private: + common::ModulePageAllocator mod_; + common::ModuleArena allocator_; + char *base_buf_; + int64_t base_buf_pos_; + char *upload_id_; + int partnum_; + int64_t file_length_; + + DISALLOW_COPY_AND_ASSIGN(ObStorageCosMultiPartWriter); +}; + } //common } //oceanbase #endif \ No newline at end of file diff --git a/deps/oblib/src/lib/restore/ob_storage_file.cpp b/deps/oblib/src/lib/restore/ob_storage_file.cpp index c0fd56bae..04de692c3 100644 --- a/deps/oblib/src/lib/restore/ob_storage_file.cpp +++ b/deps/oblib/src/lib/restore/ob_storage_file.cpp @@ -51,10 +51,8 @@ int get_file_path(const common::ObString &uri, char *buf, const int64_t buf_size } else if (!uri.prefix_match(OB_FILE_PREFIX)) { ret = OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "invalid uri", K(ret), K(uri)); - } else if (OB_FAIL(databuff_printf( - buf, buf_size, "%.*s", - static_cast(uri.length() - offset), - uri.ptr() + offset))) { + } else if (OB_FAIL(databuff_printf(buf, buf_size, "%.*s", + static_cast(uri.length() - offset), uri.ptr() + offset))) { STORAGE_LOG(WARN, "failed to fill path", K(ret), K(uri)); } else if (strlen(buf) <= 0 && buf[0] != '/') { ret = OB_INVALID_ARGUMENT; @@ -170,6 +168,39 @@ int ObStorageFileUtil::get_file_length(const common::ObString &uri, int64_t &fil return ret; } +int ObStorageFileUtil::head_object_meta(const common::ObString &uri, ObStorageObjectMetaBase &obj_meta) +{ + int ret = OB_SUCCESS; + char path[OB_MAX_URI_LENGTH]; + char errno_buf[OB_MAX_ERROR_MSG_LEN] = ""; + struct stat64 file_info; + obj_meta.reset(); + + if (uri.empty()) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid args", K(ret), K(uri)); + } else if (OB_FAIL(get_file_path(uri, path, sizeof(path)))) { + STORAGE_LOG(WARN, "failed to fill path", K(ret), K(uri)); + } else if (0 != ::stat64(path, &file_info)) { + if (ENOENT == errno || ENOTDIR == errno) { + } else { + convert_io_error(errno, ret); + STORAGE_LOG(WARN, "file not exist", + K(ret), K(path), K(errno), "errno", strerror_r(errno, errno_buf, sizeof(errno_buf))); + } + } else if (S_ISDIR(file_info.st_mode)) { + obj_meta.is_exist_ = true; + obj_meta.length_ = -1; + obj_meta.type_ = ObStorageObjectType::OB_FS_DIR; + } else { + obj_meta.is_exist_ = true; + obj_meta.length_ = file_info.st_size; + obj_meta.type_ = ObStorageObjectType::OB_FS_FILE; + } + + return ret; +} + int ObStorageFileUtil::del_file(const common::ObString &uri) { int ret = OB_SUCCESS; @@ -226,23 +257,33 @@ int ObStorageFileUtil::mkdir(const common::ObString &uri) // find the first not exist dir int64_t pos = 0; bool found_exist_dir = false; - for (pos = strlen(path) - 1; OB_SUCC(ret) && pos >= 0 && !found_exist_dir; --pos) { - if (path[pos] == '/') { - path[pos] = '\0'; + const int64_t end_pos = strlen(path) - 1; + for (pos = end_pos; OB_SUCC(ret) && pos >= 0 && !found_exist_dir; --pos) { + if (path[pos] == '/' || pos == end_pos) { + if (pos != end_pos) { + path[pos] = '\0'; + } + if (0 == ::access(path, F_OK)) { STORAGE_LOG(INFO, "path exist", K(pos), KCSTRING(path)); found_exist_dir = true; + } else if (ENOTDIR == errno) { + ret = OB_FILE_ALREADY_EXIST; + STORAGE_LOG(WARN, "already exist the same name file", K(ret), KCSTRING(path), + K(errno), "errno", strerror_r(errno, errno_buf, sizeof(errno_buf))); } else if (ENOENT != errno) { ret = OB_IO_ERROR; STORAGE_LOG(WARN, "check is parent dir exist", K(ret), KCSTRING(path), K(errno), "errno", strerror_r(errno, errno_buf, sizeof(errno_buf))); } - path[pos] = '/'; + + if (pos != end_pos) { + path[pos] = '/'; + } } } // create dirs - int64_t end_pos = strlen(path) - 1; for (int64_t i = pos + 1; OB_SUCC(ret) && i <= end_pos; ++i) { if (path[i] == '/' || end_pos == i) { if (end_pos != i) { @@ -287,7 +328,6 @@ int ObStorageFileUtil::list_files(const common::ObString &uri, common::ObBaseDir struct dirent *result; DIR *open_dir = NULL; char errno_buf[OB_MAX_ERROR_MSG_LEN] = ""; - ObString tmp_string; ObString file_name; char dir_path[OB_MAX_URI_LENGTH] = ""; char sub_dir_path[OB_MAX_URI_LENGTH] = ""; @@ -308,7 +348,6 @@ int ObStorageFileUtil::list_files(const common::ObString &uri, common::ObBaseDir } while (OB_SUCC(ret) && NULL != open_dir) { - tmp_string.reset(); file_name.reset(); is_file = false; size = 0; @@ -360,6 +399,142 @@ int ObStorageFileUtil::list_files(const common::ObString &uri, common::ObBaseDir return ret; } +int ObStorageFileUtil::list_files(const common::ObString &uri, ObStorageListCtxBase &ctx_base) +{ + int ret = OB_SUCCESS; + char dir_path[OB_MAX_URI_LENGTH] = ""; + ObStorageListFilesCtx &list_ctx = static_cast(ctx_base); + + if (OB_UNLIKELY(uri.empty() || !list_ctx.is_valid())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid argument", K(ret), K(list_ctx), KCSTRING(dir_path)); + } else if (OB_FAIL(get_file_path(uri, dir_path, sizeof(dir_path)))) { + STORAGE_LOG(WARN, "failed to fill path", K(ret), K(uri)); + } else { + list_ctx.rsp_num_ = 0; + list_ctx.has_next_ = false; + + struct dirent *result; + char errno_buf[OB_MAX_ERROR_MSG_LEN] = ""; + ObString file_name; + char sub_dir_path[OB_MAX_URI_LENGTH] = ""; + bool is_file = false; + bool is_appendable_file = false; + int64_t size = 0; + + if (!list_ctx.already_open_dir_) { + if (OB_ISNULL(list_ctx.open_dir_ = ::opendir(dir_path))) { + if (ENOENT != errno) { + ret = OB_IO_ERROR; + OB_LOG(WARN, "fail to open dir", K(ret), KCSTRING(dir_path), + KCSTRING(strerror_r(errno, errno_buf, sizeof(errno_buf)))); + } + } else { + list_ctx.already_open_dir_ = true; + } + } + + while (OB_SUCC(ret) && NULL != list_ctx.open_dir_) { + file_name.reset(); + is_file = false; + is_appendable_file = false; + size = 0; + if (list_ctx.rsp_num_ >= list_ctx.max_list_num_) { + list_ctx.has_next_ = true; + break; + } + + if (0 != ::readdir_r(list_ctx.open_dir_, &(list_ctx.next_entry_), &result)) { + ret = OB_IO_ERROR; + OB_LOG(WARN, "read dir error", K(ret), + KCSTRING(strerror_r(errno, errno_buf, sizeof(errno_buf)))); + } else if (NULL != result) { + if (0 == strcmp(list_ctx.next_entry_.d_name, ".") || 0 == strcmp(list_ctx.next_entry_.d_name, "..")) { + // not a file + } else if (DT_DIR == list_ctx.next_entry_.d_type) { + if (OB_FAIL(check_is_appendable(uri, list_ctx.next_entry_, is_appendable_file))) { + OB_LOG(WARN, "fail to check is_appendable", K(ret), K(uri)); + } else if (is_appendable_file) { + is_file = true; + } + } else if (DT_REG == list_ctx.next_entry_.d_type && !list_ctx.need_size_) { + is_file = true; + } else if ((DT_REG == list_ctx.next_entry_.d_type && list_ctx.need_size_) || DT_UNKNOWN == list_ctx.next_entry_.d_type) { + int pret = snprintf(sub_dir_path, OB_MAX_URI_LENGTH, "%s/%s", dir_path, list_ctx.next_entry_.d_name); + if (pret < 0 || pret >= OB_MAX_URI_LENGTH) { + ret = OB_BUF_NOT_ENOUGH; + OB_LOG(WARN, "format dir path fail", K(ret), KCSTRING(dir_path)); + } else { + struct stat sb; + if (-1 == ::stat(sub_dir_path, &sb)) { + ret = OB_IO_ERROR; + OB_LOG(WARN, "stat fail", K(ret), KCSTRING(strerror_r(errno, errno_buf, sizeof(errno_buf)))); + } else if (!S_ISREG(sb.st_mode)) { + // not a file + } else { + is_file = true; + size = static_cast(sb.st_size); + } + } + } + + if (OB_SUCC(ret) && is_file) { + const int64_t name_len = strlen(list_ctx.next_entry_.d_name); + if (name_len >= list_ctx.max_name_len_) { + ret = OB_SIZE_OVERFLOW; + OB_LOG(WARN, "can't hold the dentry name", K(ret), K(list_ctx.max_name_len_), K(name_len)); + } else { + MEMCPY(list_ctx.name_arr_[list_ctx.rsp_num_], list_ctx.next_entry_.d_name, name_len); + if (is_appendable_file) { + list_ctx.name_arr_[list_ctx.rsp_num_][name_len] = '/'; + list_ctx.name_arr_[list_ctx.rsp_num_][name_len + 1] = '\0'; + } else { + list_ctx.name_arr_[list_ctx.rsp_num_][name_len] = '\0'; + } + if (list_ctx.need_size_) { + list_ctx.size_arr_[list_ctx.rsp_num_] = size; + } + ++list_ctx.rsp_num_; + } + } + } else { + list_ctx.has_next_ = false; + break; // end file + } + } + // close dir + if (!list_ctx.has_next_ && list_ctx.already_open_dir_ && NULL != list_ctx.open_dir_) { + ::closedir(list_ctx.open_dir_); + } + } + + return ret; +} + +int ObStorageFileUtil::check_is_appendable( + const common::ObString &uri, + struct dirent &cur_entry, + bool &is_appendable_file) +{ + int ret = OB_SUCCESS; + ObStorageObjectMetaBase obj_meta; + char tmp_uri_buf[OB_MAX_URI_LENGTH] = ""; + int pos = snprintf(tmp_uri_buf, OB_MAX_URI_LENGTH, "%s/%s/%s%s", uri.ptr(), cur_entry.d_name, + OB_S3_APPENDABLE_FRAGMENT_PREFIX, OB_S3_APPENDABLE_FORMAT_META); + if (pos < 0 || pos >= OB_MAX_URI_LENGTH) { + ret = OB_BUF_NOT_ENOUGH; + OB_LOG(WARN, "fail to build format meta file path", K(ret), K(pos), K(uri), KCSTRING(cur_entry.d_name)); + } else { + common::ObString format_meta_uri(pos, tmp_uri_buf); + if (OB_FAIL(head_object_meta(format_meta_uri, obj_meta))) { + OB_LOG(WARN, "fail to head object meta", K(ret), K(format_meta_uri)); + } else { + is_appendable_file = obj_meta.is_exist_; + } + } + return ret; +} + int ObStorageFileUtil::del_dir(const common::ObString &uri) { int ret = OB_SUCCESS; @@ -518,6 +693,13 @@ int ObStorageFileUtil::is_tagging(const common::ObString &uri, bool &is_tagging) return ret; } +int ObStorageFileUtil::del_unmerged_parts(const ObString &uri) +{ + UNUSED(uri); + int ret = OB_NOT_SUPPORTED; + return ret; +} + ObStorageFileReader::ObStorageFileReader() : fd_(-1), is_opened_(false), @@ -537,7 +719,8 @@ ObStorageFileReader::~ObStorageFileReader() } } -int ObStorageFileReader::open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info) +int ObStorageFileReader::open(const common::ObString &uri, + common::ObObjectStorageInfo *storage_info, const bool head_meta) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -555,19 +738,22 @@ int ObStorageFileReader::open(const common::ObString &uri, common::ObObjectStora STORAGE_LOG(WARN, "failed to open read file", K(ret), KCSTRING(path_), K(errno), "errno", strerror_r(errno, errno_buf, sizeof(errno_buf))); } else { - is_opened_ = true; - if (0 != ::fstat64(fd_, &file_info)) { - ret = OB_IO_ERROR; - STORAGE_LOG(WARN, "file not exist", - K(ret), K(fd_), KCSTRING(path_), K(errno), "errno", strerror_r(errno, errno_buf, sizeof(errno_buf))); - } else if (S_ISDIR(file_info.st_mode)) { - ret = OB_IO_ERROR; - STORAGE_LOG(WARN, "uri is a dir", K(ret), KCSTRING(path_)); - } else { - file_length_ = file_info.st_size; + if (head_meta) { + if (0 != ::fstat64(fd_, &file_info)) { + convert_io_error(errno, ret); + STORAGE_LOG(WARN, "file not exist", + K(ret), K(fd_), KCSTRING(path_), K(errno), "errno", strerror_r(errno, errno_buf, sizeof(errno_buf))); + } else if (S_ISDIR(file_info.st_mode)) { + ret = OB_IO_ERROR; + STORAGE_LOG(WARN, "uri is a dir", K(ret), KCSTRING(path_)); + } else { + file_length_ = file_info.st_size; + } } - if (OB_FAIL(ret)) { + if (OB_SUCC(ret)) { + is_opened_ = true; + } else { if (OB_SUCCESS != (tmp_ret = close())) { STORAGE_LOG(WARN, "failed to close", K(ret), K(tmp_ret), KCSTRING(path_)); } @@ -577,7 +763,7 @@ int ObStorageFileReader::open(const common::ObString &uri, common::ObObjectStora } int ObStorageFileReader::pread( - char *buf,const int64_t buf_size, int64_t offset, int64_t &read_size) + char *buf, const int64_t buf_size, const int64_t offset, int64_t &read_size) { int ret = OB_SUCCESS; char errno_buf[OB_MAX_ERROR_MSG_LEN] = ""; @@ -614,11 +800,8 @@ int ObStorageFileReader::close() int ret = OB_SUCCESS; char errno_buf[OB_MAX_ERROR_MSG_LEN] = ""; - if (!is_opened_) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "not opened", K(ret), K(fd_)); - } else if (0 != ::close(fd_)) { - ret = OB_IO_ERROR; + if (is_opened_ && 0 != ::close(fd_)) { + convert_io_error(errno, ret); STORAGE_LOG(WARN, "failed to close read file", K(ret), KCSTRING(path_), K(errno), "errno", strerror_r(errno, errno_buf, sizeof(errno_buf))); } @@ -851,6 +1034,13 @@ int ObStorageFileWriter::close() #endif if (has_error_) { STORAGE_LOG(WARN, "writer has error, skip rename file", KCSTRING(path_), KCSTRING(real_path_)); + + // has error, try delete file regardless of whether the temporary file exists + if (0 != ::remove(path_)) { + tmp_ret = OB_IO_ERROR; + STORAGE_LOG(WARN, "failed to remove file", K(tmp_ret), KCSTRING(path_), + K(errno), "errno", strerror_r(errno, errno_buf, sizeof(errno_buf))); + } } else if (0 != ::rename(path_, real_path_)) { ret = OB_IO_ERROR; STORAGE_LOG(WARN, "failed to rename meta file", @@ -865,6 +1055,7 @@ int ObStorageFileWriter::close() STORAGE_LOG(INFO, "succeed to rename file after close", KCSTRING(path_), KCSTRING(real_path_)); } } + has_error_ = false; return ret; } @@ -976,5 +1167,11 @@ int ObStorageFileAppender::get_open_flag_and_mode_(int &flag, bool &need_lock) return ret; } +int ObStorageFileMultiPartWriter::pwrite(const char *buf, const int64_t size, const int64_t offset) +{ + UNUSED(offset); + return write(buf, size); +} + }//common }//oceanbase diff --git a/deps/oblib/src/lib/restore/ob_storage_file.h b/deps/oblib/src/lib/restore/ob_storage_file.h index fdc5772cd..7ffa996a0 100644 --- a/deps/oblib/src/lib/restore/ob_storage_file.h +++ b/deps/oblib/src/lib/restore/ob_storage_file.h @@ -34,15 +34,19 @@ public: virtual int is_exist(const common::ObString &uri, bool &exist); virtual int get_file_length(const common::ObString &uri, int64_t &file_length); + virtual int head_object_meta(const common::ObString &uri, ObStorageObjectMetaBase &obj_meta); virtual int del_file(const common::ObString &uri); virtual int write_single_file(const common::ObString &uri, const char *buf, const int64_t size); virtual int mkdir(const common::ObString &uri); virtual int list_files(const common::ObString &uri, common::ObBaseDirEntryOperator &op); + virtual int list_files(const common::ObString &uri, ObStorageListCtxBase &list_ctx); virtual int del_dir(const common::ObString &uri); virtual int list_directories(const common::ObString &uri, common::ObBaseDirEntryOperator &op); virtual int is_tagging(const common::ObString &uri, bool &is_tagging); + virtual int del_unmerged_parts(const ObString &uri) override; private: int get_tmp_file_format_timestamp(const char *file_name, bool &is_tmp_file, int64_t ×tamp); + int check_is_appendable(const common::ObString &uri, struct dirent &entry, bool &is_appendable_file); private: DISALLOW_COPY_AND_ASSIGN(ObStorageFileUtil); @@ -53,11 +57,13 @@ class ObStorageFileReader: public ObIStorageReader public: ObStorageFileReader(); virtual ~ObStorageFileReader(); - virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info = NULL); - virtual int pread(char *buf,const int64_t buf_size, int64_t offset, int64_t &read_size); - virtual int close(); - virtual int64_t get_length() const { return file_length_; } - virtual bool is_opened() const { return is_opened_; } + virtual int open(const common::ObString &uri, + common::ObObjectStorageInfo *storage_info = NULL, const bool head_meta = true) override; + virtual int pread(char *buf, + const int64_t buf_size, const int64_t offset, int64_t &read_size) override; + virtual int close() override; + virtual int64_t get_length() const override { return file_length_; } + virtual bool is_opened() const override { return is_opened_; } private: int fd_; bool is_opened_; @@ -121,6 +127,17 @@ private: DISALLOW_COPY_AND_ASSIGN(ObStorageFileAppender); }; +class ObStorageFileMultiPartWriter : public ObStorageFileWriter +{ +public: + ObStorageFileMultiPartWriter() {} + virtual ~ObStorageFileMultiPartWriter() {} + virtual int pwrite(const char *buf, const int64_t size, const int64_t offset) override; + +private: + DISALLOW_COPY_AND_ASSIGN(ObStorageFileMultiPartWriter); +}; + }//common }//oceanbase diff --git a/deps/oblib/src/lib/restore/ob_storage_info.cpp b/deps/oblib/src/lib/restore/ob_storage_info.cpp index adb63d99e..34c677911 100644 --- a/deps/oblib/src/lib/restore/ob_storage_info.cpp +++ b/deps/oblib/src/lib/restore/ob_storage_info.cpp @@ -87,9 +87,10 @@ ObStorageType ObObjectStorageInfo::get_type() const // oss:host=xxxx&access_id=xxx&access_key=xxx // cos:host=xxxx&access_id=xxx&access_key=xxxappid=xxx +// s3:host=xxxx&access_id=xxx&access_key=xxx&s3_region=xxx int ObObjectStorageInfo::set(const common::ObStorageType device_type, const char *storage_info) { - bool has_appid = false; + bool has_needed_extension = false; int ret = OB_SUCCESS; if (is_valid()) { ret = OB_INIT_TWICE; @@ -103,16 +104,19 @@ int ObObjectStorageInfo::set(const common::ObStorageType device_type, const char ret = OB_INVALID_BACKUP_DEST; LOG_WARN("storage info is empty", K(ret), K_(device_type)); } - } else if (OB_FAIL(parse_storage_info_(storage_info, has_appid))) { + } else if (OB_FAIL(parse_storage_info_(storage_info, has_needed_extension))) { LOG_WARN("parse storage info failed", K(ret)); } else if (OB_STORAGE_FILE != device_type && (0 == strlen(endpoint_) || 0 == strlen(access_id_) || 0 == strlen(access_key_))) { ret = OB_INVALID_BACKUP_DEST; LOG_WARN("backup device is not nfs, endpoint/access_id/access_key do not allow to be empty", K(ret), K_(device_type), K_(endpoint), K_(access_id)); - } else if (OB_STORAGE_COS == device_type && !has_appid) { + } else if (OB_STORAGE_COS == device_type && !has_needed_extension) { ret = OB_INVALID_BACKUP_DEST; LOG_WARN("invalid cos info, appid do not allow to be empty", K(ret), K_(extension)); + } else if (OB_STORAGE_S3 == device_type && !has_needed_extension) { + ret = OB_INVALID_BACKUP_DEST; + LOG_WARN("invalid s3 info, region do not allow to be empty", K(ret), K_(extension)); } else if (OB_STORAGE_FILE == device_type && (0 != strlen(endpoint_) || 0 != strlen(access_id_) || 0 != strlen(access_key_))) { ret = OB_INVALID_BACKUP_DEST; @@ -143,10 +147,10 @@ int ObObjectStorageInfo::set(const char *uri, const char *storage_info) return ret; } -int ObObjectStorageInfo::parse_storage_info_(const char *storage_info, bool &has_appid) +int ObObjectStorageInfo::parse_storage_info_(const char *storage_info, bool &has_needed_extension) { int ret = OB_SUCCESS; - + has_needed_extension = false; if (OB_ISNULL(storage_info) || strlen(storage_info) >= OB_MAX_BACKUP_STORAGE_INFO_LENGTH) { ret = OB_INVALID_BACKUP_DEST; LOG_WARN("storage info is invalid", K(ret), K(storage_info), K(strlen(storage_info))); @@ -163,9 +167,14 @@ int ObObjectStorageInfo::parse_storage_info_(const char *storage_info, bool &has token = ::strtok_r(str, "&", &saved_ptr); if (NULL == token) { break; + } else if (0 == strncmp(REGION, token, strlen(REGION))) { + has_needed_extension = (OB_STORAGE_S3 == device_type_); + if (OB_FAIL(set_storage_info_field_(token, extension_, sizeof(extension_)))) { + LOG_WARN("failed to set region", K(ret), K(token)); + } } else if (0 == strncmp(HOST, token, strlen(HOST))) { if (OB_FAIL(set_storage_info_field_(token, endpoint_, sizeof(endpoint_)))) { - LOG_WARN("failed to set endpoint",K(ret), K(token)); + LOG_WARN("failed to set endpoint", K(ret), K(token)); } } else if (0 == strncmp(ACCESS_ID, token, strlen(ACCESS_ID))) { if (OB_FAIL(set_storage_info_field_(token, access_id_, sizeof(access_id_)))) { @@ -176,7 +185,7 @@ int ObObjectStorageInfo::parse_storage_info_(const char *storage_info, bool &has LOG_WARN("failed to set access key", K(ret), K(token)); } } else if (OB_STORAGE_FILE != device_type_ && 0 == strncmp(APPID, token, strlen(APPID))) { - has_appid = true; + has_needed_extension = (OB_STORAGE_COS == device_type_); if (OB_FAIL(set_storage_info_field_(token, extension_, sizeof(extension_)))) { LOG_WARN("failed to set appid", K(ret), K(token)); } @@ -267,8 +276,7 @@ int ObObjectStorageInfo::get_storage_info_str(char *storage_info, const int64_t } if (OB_SUCC(ret) && 0 != strlen(extension_) && info_len > strlen(storage_info)) { - // if OB_STORAGE_FILE's extension is not empty - // delimiter should be included + // if OB_STORAGE_FILE's extension is not empty, delimiter should be included int64_t str_len = strlen(storage_info); if (str_len > 0 && OB_FAIL(databuff_printf(storage_info, info_len, str_len, "&"))) { LOG_WARN("failed to add delimiter to storage info", K(ret), K(info_len), K(str_len)); diff --git a/deps/oblib/src/lib/restore/ob_storage_info.h b/deps/oblib/src/lib/restore/ob_storage_info.h index 91d00614f..258bfca2d 100644 --- a/deps/oblib/src/lib/restore/ob_storage_info.h +++ b/deps/oblib/src/lib/restore/ob_storage_info.h @@ -38,6 +38,7 @@ const char *const ACCESS_KEY = "access_key="; const char *const HOST = "host="; const char *const APPID = "appid="; const char *const DELETE_MODE = "delete_mode="; +const char *const REGION = "s3_region="; class ObObjectStorageInfo { diff --git a/deps/oblib/src/lib/restore/ob_storage_oss_base.cpp b/deps/oblib/src/lib/restore/ob_storage_oss_base.cpp index 361764694..b2124c204 100644 --- a/deps/oblib/src/lib/restore/ob_storage_oss_base.cpp +++ b/deps/oblib/src/lib/restore/ob_storage_oss_base.cpp @@ -269,29 +269,6 @@ int get_bucket_object_name_for_list(const ObString &uri, ObString &bucket, ObStr return ret; } -int build_md5_str(MD5_CTX *c, char *buf, int64_t size) -{ - int ret = OB_SUCCESS; - unsigned char md5[MD5_DIGEST_LENGTH]; - if (0 == MD5_Final(md5, c)) { - ret = OB_FINAL_MD5_ERROR; - OB_LOG(WARN, "final MD5_CTX error", K(ret)); - } else { - int64_t pos = 0; - for (int i = 0; OB_SUCC(ret) && i < MD5_DIGEST_LENGTH; ++i) { - int n = snprintf(buf + pos, size - pos, "%02X", md5[i]); - if (n < 0 || n >= size - pos) { - ret = OB_SIZE_OVERFLOW; - OB_LOG (WARN, "md5_buf_size not enough", K(size), K(i), K(pos), K(n), K(ret)); - } else { - pos += n; - } - } - } - OB_LOG(INFO, "The Whole File MD5 ", KCSTRING(buf)); - return ret; -} - ObOssEnvIniter::ObOssEnvIniter() : lock_(common::ObLatchIds::OBJECT_DEVICE_LOCK), is_global_inited_(false) @@ -668,7 +645,6 @@ ObStorageOssMultiPartWriter::ObStorageOssMultiPartWriter() is_opened_(false), file_length_(-1) { - MEMSET(&whole_file_md5_, 0, sizeof(whole_file_md5_)); upload_id_.len = -1; upload_id_.data = NULL; } @@ -717,67 +693,16 @@ int ObStorageOssMultiPartWriter::open(const ObString &uri, common::ObObjectStora if(OB_ISNULL(base_buf_ = static_cast(allocator_.alloc(OSS_BASE_BUFFER_SIZE)))) { ret = OB_ALLOCATE_MEMORY_FAILED; OB_LOG(WARN, "fail to alloc memory", K(OSS_BASE_BUFFER_SIZE), K(ret)); - } else if(0 == MD5_Init(&whole_file_md5_)) {//init MD5_CTX,return 0 for failed - ret = OB_INIT_MD5_ERROR; - OB_LOG(WARN, "init MD5_CTX error,uri=%s, ret=%d", to_cstring(uri), ret); } else { is_opened_ = true; base_buf_pos_ = 0; + file_length_ = 0; } } } return ret; } -int ObStorageOssMultiPartWriter::upload_data(const char *buf, const int64_t size, char *&upload_buf, int64_t &upload_size) -{ - ObMemBuf *writer_buf = GET_TSI_MULT(ObMemBuf, 1); - - int ret = OB_SUCCESS; - ObExternalIOCounterGuard io_guard; - upload_buf = NULL; - upload_size = 0; - - if (!is_inited()) { - ret = OB_NOT_INIT; - OB_LOG(WARN, "oss client not inited", K(ret)); - } else if (NULL == buf || size < 0 || OB_ISNULL(writer_buf)) { - ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "argument is invalid", KP(buf), K(size), K(writer_buf), K(ret)); - } else { - int64_t max_size = 0; - int64_t max_over_flow_size = 0; - int64_t compress_size = 0; - int64_t pos = 0; - if (OB_FAIL(ObStorageOssStaticVar::get_instance().get_oss_compressor()->get_max_overflow_size(size, max_over_flow_size))) { - OB_LOG(WARN, "fail to get max over flow size", K(ret)); - } else { - max_size = sizeof(int64_t) + size + max_over_flow_size; - if (OB_FAIL(writer_buf->ensure_space(max_size, ObModIds::BACKUP))) { - OB_LOG(WARN, "fail to allocation memory", K(max_size), K(ret)); -// } else if (OB_FAIL(serialization::encode_i8(writer_buf.data(), max_size, pos, compress_type_))) { -// OB_LOG(WARN, "fail to serialization compress type", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (ObCompressorType::NONE_COMPRESSOR != ObStorageOssStaticVar::get_instance().get_compressor_type() && 0 != size) { - if (OB_FAIL(ObStorageOssStaticVar::get_instance().get_oss_compressor()->compress(buf, - size, writer_buf->get_buffer() + pos, max_size, compress_size))) { - OB_LOG(WARN, "fail to compress data", K(ret)); - } else { - upload_size = compress_size + pos; - } - } else if (ObCompressorType::NONE_COMPRESSOR == ObStorageOssStaticVar::get_instance().get_compressor_type() || 0 == size) { - MEMCPY(writer_buf->get_buffer() + pos, buf, size); - upload_size = size + pos; - } - upload_buf = writer_buf->get_buffer(); - } - } - return ret; -} - int ObStorageOssMultiPartWriter::write(const char * buf,const int64_t size) { int ret = OB_SUCCESS; @@ -817,21 +742,18 @@ int ObStorageOssMultiPartWriter::write(const char * buf,const int64_t size) int ObStorageOssMultiPartWriter::pwrite(const char *buf, const int64_t size, const int64_t offset) { - int ret = OB_NOT_SUPPORTED; - UNUSEDx(buf, size, offset); - return ret; + UNUSED(offset); + return write(buf, size); } int ObStorageOssMultiPartWriter::write_single_part() { int ret = OB_SUCCESS; ObExternalIOCounterGuard io_guard; - char *upload_buf = NULL; - int64_t upload_size = 0; ++partnum_; //cal part num begin from 0 - if(partnum_ >= OSS_MAX_PART_NUM) { + if(partnum_ > OSS_MAX_PART_NUM) { ret = OB_OUT_OF_ELEMENT; OB_LOG(WARN, "Out of oss element ", K(partnum_), K(OSS_MAX_PART_NUM), K(ret)); } else if(!is_inited()) { @@ -840,11 +762,6 @@ int ObStorageOssMultiPartWriter::write_single_part() } else if(!is_opened_) { ret = OB_OSS_ERROR; OB_LOG(WARN, "write oss should open first", K(ret)); - } else if(0 == MD5_Update(&whole_file_md5_, base_buf_, base_buf_pos_)) { - ret = OB_UPDATE_MD5_ERROR; - OB_LOG(WARN, "MD5 update error", K(ret)); - } else if (OB_FAIL(upload_data(base_buf_, base_buf_pos_, upload_buf, upload_size))) { - OB_LOG(WARN, "fail to compress upload data", K(ret)); } if(OB_SUCCESS == ret) { @@ -860,7 +777,7 @@ int ObStorageOssMultiPartWriter::write_single_part() aos_list_init(&buffer); const int64_t start_time = ObTimeUtility::current_time(); - if (OB_ISNULL(content = aos_buf_pack(aos_pool_, upload_buf, static_cast(upload_size)))) { + if (OB_ISNULL(content = aos_buf_pack(aos_pool_, base_buf_, static_cast(base_buf_pos_)))) { ret = OB_OSS_ERROR; OB_LOG(WARN, "fail to pack buf", K(content), K(ret)); } else { @@ -870,12 +787,12 @@ int ObStorageOssMultiPartWriter::write_single_part() partnum_, &buffer, &resp_headers)) || !aos_status_is_ok(aos_ret)) { ret = OB_OSS_ERROR; OB_LOG(WARN, "fail to upload one part from buffer", - K(upload_size), K(bucket_), K(object_), K(ret)); + K_(base_buf_pos), K_(bucket), K_(object), K(ret)); print_oss_info(resp_headers, aos_ret); cleanup(); } bool is_slow = false; - print_access_storage_log("oss upload one part ", object_, start_time, upload_size, &is_slow); + print_access_storage_log("oss upload one part ", object_, start_time, base_buf_pos_, &is_slow); int tmp_ret = OB_SUCCESS; if (OB_SUCC(ret) && is_slow) { print_oss_info(resp_headers, aos_ret); @@ -894,16 +811,16 @@ int ObStorageOssMultiPartWriter::close() int ret = OB_SUCCESS; ObExternalIOCounterGuard io_guard; const int64_t start_time = ObTimeUtility::current_time(); - if(!is_inited()) { + if(OB_UNLIKELY(!is_inited())) { ret = OB_NOT_INIT; OB_LOG(WARN, "oss client not inited", K(ret)); - } else if(!is_opened_) { + } else if(OB_UNLIKELY(!is_opened_)) { ret = OB_OSS_ERROR; - OB_LOG(WARN, "oss writer cannot close before it is opened"); + OB_LOG(WARN, "oss writer cannot close before it is opened", K(ret)); } else if(0 != base_buf_pos_) {//base_buf has data if(OB_SUCCESS != (ret = write_single_part())) { OB_LOG(WARN, "write the last size to oss error", - K(base_buf_pos_), K(bucket_), K(object_),K(ret)); + K_(base_buf_pos), K_(bucket), K_(object), K(ret)); cleanup(); ret = OB_OSS_ERROR; } else { @@ -926,72 +843,61 @@ int ObStorageOssMultiPartWriter::close() oss_complete_part_content_t *complete_part_content = NULL; //add all parts to complete_part_list - if(NULL == (params = oss_create_list_upload_part_params(aos_pool_))) { + if(OB_ISNULL(params = oss_create_list_upload_part_params(aos_pool_))) { ret = OB_OSS_ERROR; - OB_LOG(WARN, "fail to create oss upload params", K(bucket_), K(object_), K(ret)); - } else if(NULL == (aos_ret = oss_list_upload_part(oss_option_, &bucket, &object, &upload_id_, - params, &resp_headers)) || !aos_status_is_ok(aos_ret)) { - ret = OB_OSS_ERROR; - OB_LOG(WARN, "fail to list oss upload parts", K(bucket_), K(object_), K(ret)); - print_oss_info(resp_headers, aos_ret); - cleanup(); + OB_LOG(WARN, "fail to create oss upload params", K_(bucket), K_(object), K(ret)); } else { - aos_list_for_each_entry(oss_list_part_content_t, part_content, ¶ms->part_list, node) { - if (NULL == (complete_part_content = oss_create_complete_part_content(aos_pool_))) { - ret = OB_OSS_ERROR; - OB_LOG(WARN, "fail to create complete part content", K(bucket_), K(object_), K(ret)); - break; + params->max_ret = OB_STORAGE_LIST_MAX_NUM; + do { + if(OB_ISNULL(aos_ret = oss_list_upload_part(oss_option_, &bucket, &object, &upload_id_, + params, &resp_headers)) || !aos_status_is_ok(aos_ret)) { + convert_io_error(aos_ret, ret); + OB_LOG(WARN, "fail to list oss upload parts", K_(bucket), K_(object), K(ret)); + print_oss_info(resp_headers, aos_ret); + cleanup(); } else { - aos_str_set(&complete_part_content->part_number, part_content->part_number.data); - aos_str_set(&complete_part_content->etag, part_content->etag.data); - aos_list_add_tail(&complete_part_content->node, &complete_part_list); + aos_list_for_each_entry(oss_list_part_content_t, part_content, ¶ms->part_list, node) { + if (OB_ISNULL(complete_part_content = oss_create_complete_part_content(aos_pool_))) { + ret = OB_OSS_ERROR; + OB_LOG(WARN, "fail to create complete part content", K_(bucket), K_(object), K(ret)); + break; + } else { + aos_str_set(&complete_part_content->part_number, part_content->part_number.data); + aos_str_set(&complete_part_content->etag, part_content->etag.data); + aos_list_add_tail(&complete_part_content->node, &complete_part_list); + } + } + + if (OB_SUCC(ret) && AOS_TRUE == params->truncated) { + const char *next_part_number_marker = NULL; + if (OB_ISNULL(next_part_number_marker = + apr_psprintf(aos_pool_, "%.*s", + params->next_part_number_marker.len, + params->next_part_number_marker.data))) { + ret = OB_OSS_ERROR; + OB_LOG(WARN, "next part number marker is NULL", K(ret), + K(params->next_part_number_marker.data), K(params->next_part_number_marker.len)); + } else { + aos_str_set(¶ms->part_number_marker, next_part_number_marker); + aos_list_init(¶ms->part_list); + } + } } - } + } while (OB_SUCC(ret) && AOS_TRUE == params->truncated); } //complete multipart upload - if(OB_SUCCESS == ret) { - if(NULL == (aos_ret = oss_complete_multipart_upload(oss_option_, &bucket, &object, &upload_id_, + if(OB_SUCC(ret)) { + if(OB_ISNULL(aos_ret = oss_complete_multipart_upload(oss_option_, &bucket, &object, &upload_id_, &complete_part_list, complete_headers, &resp_headers)) || !aos_status_is_ok(aos_ret)) { - ret = OB_OSS_ERROR; - OB_LOG(WARN, "fail to complete multipart upload", K(bucket_), K(object_), K(ret)); + convert_io_error(aos_ret, ret); + OB_LOG(WARN, "fail to complete multipart upload", K_(bucket), K_(object), K(ret)); print_oss_info(resp_headers, aos_ret); cleanup(); } } } - if(OB_SUCCESS == ret) { - //The oss c client verifies the integrity of each part every time it uploads, but the md5 value of the entire file is not saved, - //Here you need to upload md5 by oss_copy_object. - char md5_str[MD5_STR_LENGTH + 1] = {0};//must end with '\0' - if(OB_SUCCESS != (ret = build_md5_str(&whole_file_md5_, md5_str, sizeof(md5_str)))) { - OB_LOG(WARN, "fail to build md5 ", K(bucket_), K(object_), K(ret)); - } else { - aos_string_t bucket; - aos_string_t object; - aos_str_set(&bucket, bucket_.ptr()); - aos_str_set(&object, object_.ptr()); - aos_table_t *resp_headers = NULL; - aos_status_t *aos_ret = NULL; - aos_table_t *headers = NULL; - - if(NULL == (headers = aos_table_make(aos_pool_, AOS_TABLE_INIT_SIZE))) { - ret = OB_OSS_ERROR; - OB_LOG(WARN, "fail to make aos table", K(ret)); - } else { - apr_table_set(headers, OSS_META_MD5, md5_str); - if(NULL == (aos_ret = oss_copy_object(oss_option_, &bucket, &object, &bucket, &object, - headers, &resp_headers)) || !aos_status_is_ok(aos_ret)) { - ret = OB_OSS_ERROR; - OB_LOG(WARN, "fail to copy object with md5", K(bucket_), K(object_), K(ret)); - print_oss_info(resp_headers, aos_ret); - cleanup(); - } - } - } - } - reset(); const int64_t total_cost_time = ObTimeUtility::current_time() - start_time; @@ -1033,7 +939,8 @@ ObStorageOssReader::ObStorageOssReader(): object_(), file_length_(-1), is_opened_(false), - allocator_(ObModIds::BACKUP) + has_meta_(false), + allocator_(OB_STORAGE_OSS_ALLOCATOR) { } @@ -1041,7 +948,8 @@ ObStorageOssReader::~ObStorageOssReader() { } -int ObStorageOssReader::open(const ObString &uri, common::ObObjectStorageInfo *storage_info) +int ObStorageOssReader::open(const ObString &uri, + common::ObObjectStorageInfo *storage_info, const bool head_meta) { int ret = OB_SUCCESS; ObExternalIOCounterGuard io_guard; @@ -1059,21 +967,29 @@ int ObStorageOssReader::open(const ObString &uri, common::ObObjectStorageInfo *s OB_LOG(WARN, "failed to init oss", K(ret)); } else if (OB_SUCCESS != (ret = get_bucket_object_name(uri, bucket_, object_, allocator_))) { OB_LOG(WARN, "bucket name of object name is empty", K(ret)); - } else if (OB_SUCCESS != (ret = get_oss_file_meta(bucket_, object_, is_file_exist, - remote_md5, file_length))) { - OB_LOG(WARN, "fail to get file meta", K(bucket_), K(object_), K(ret)); - } else if (!is_file_exist) { - ret = OB_BACKUP_FILE_NOT_EXIST; - OB_LOG(WARN, "backup file is not exist", K(bucket_), K(object_), K(ret)); } else { - file_length_ = file_length; - is_opened_ = true; + if (head_meta) { + if (OB_SUCCESS != (ret = get_oss_file_meta(bucket_, object_, is_file_exist, + remote_md5, file_length))) { + OB_LOG(WARN, "fail to get file meta", K(bucket_), K(object_), K(ret)); + } else if (!is_file_exist) { + ret = OB_BACKUP_FILE_NOT_EXIST; + OB_LOG(WARN, "backup file is not exist", K(bucket_), K(object_), K(ret)); + } else { + file_length_ = file_length; + has_meta_ = true; + } + } + + if (OB_SUCC(ret)) { + is_opened_ = true; + } } return ret; } int ObStorageOssReader::pread( - char *buf,const int64_t buf_size, int64_t offset, int64_t &read_size) + char *buf,const int64_t buf_size, const int64_t offset, int64_t &read_size) { int ret = OB_SUCCESS; ObExternalIOCounterGuard io_guard; @@ -1083,9 +999,9 @@ int ObStorageOssReader::pread( if (!is_inited()) { ret = OB_NOT_INIT; OB_LOG(WARN, "oss client not inited", K(ret)); - } else if (NULL == buf || buf_size <= 0) { + } else if (OB_ISNULL(buf) || OB_UNLIKELY(buf_size <= 0 || offset < 0)) { ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "buf is NULL or buf_size is invalid", K(buf_size), K(ret)); + OB_LOG(WARN, "buf is NULL or buf_size is invalid", K(buf_size), K(ret), KP(buf), K(offset)); } else if (!is_opened_) { ret = OB_OSS_ERROR; OB_LOG(WARN, "oss reader cannot read before it is opened", K(ret)); @@ -1095,15 +1011,15 @@ int ObStorageOssReader::pread( ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "aos pool or oss option is NULL", K(aos_pool), K(oss_option), K(ret)); } else { - if (file_length_ == offset) { + if (has_meta_ && file_length_ == offset) { read_size = 0; - } else if ( file_length_ < offset) { + } else if (has_meta_ && file_length_ < offset) { ret = OB_FILE_LENGTH_INVALID; OB_LOG(WARN, "File lenth is invilid", K(file_length_), K(offset), K(bucket_), K(object_), K(ret)); - } else if (file_length_ - offset > 0) { + } else { const int64_t start_time = ObTimeUtility::current_time(); - int64_t get_data_size = std::min(buf_size, file_length_ - offset); + int64_t get_data_size = buf_size; aos_string_t bucket; aos_string_t object; aos_str_set(&bucket, bucket_.ptr()); @@ -1143,7 +1059,7 @@ int ObStorageOssReader::pread( aos_list_for_each_entry(aos_buf_t, content, &buffer, node) { len += aos_buf_size(content); } - if(len != get_data_size) { + if(len > get_data_size) { ret = OB_OSS_ERROR; OB_LOG(WARN, "get data size error", K(get_data_size), K(len), K(bucket_), K(object_), K(ret)); @@ -1161,12 +1077,14 @@ int ObStorageOssReader::pread( } } - read_size = get_data_size; + if (OB_SUCC(ret)) { + read_size = len; + } } } } bool is_slow = false; - print_access_storage_log("oss read one part ", object_, start_time, get_data_size, &is_slow); + print_access_storage_log("oss read one part ", object_, start_time, read_size, &is_slow); if (is_slow) { print_oss_info(resp_headers, aos_ret); } @@ -1183,21 +1101,12 @@ int ObStorageOssReader::close() { int ret = OB_SUCCESS; ObExternalIOCounterGuard io_guard; - //1 for store the last '\0' - if(!is_inited()) { - ret = OB_NOT_INIT; - OB_LOG(WARN, "oss client not inited", K(ret)); - } else if(!is_opened_) { - ret = OB_OSS_ERROR; - OB_LOG(WARN, "oss reader cannot close before it is opened", K(ret)); - } else { - is_opened_ = false; - file_length_ = -1; - // Release memory - allocator_.clear(); - - reset(); - } + is_opened_ = false; + has_meta_ = false; + file_length_ = -1; + // Release memory + allocator_.clear(); + reset(); return ret; } @@ -1228,58 +1137,48 @@ int ObStorageOssUtil::open(common::ObObjectStorageInfo *storage_info) void ObStorageOssUtil::close() { - if (!is_opened_) { - OB_LOG_RET(WARN, OB_ERR_UNEXPECTED, "oss util cannot close before it is opened"); - } else { - is_opened_ = false; - storage_info_ = NULL; - } + is_opened_ = false; + storage_info_ = NULL; } int ObStorageOssUtil::is_exist(const ObString &uri, bool &exist) { int ret = OB_SUCCESS; - ObExternalIOCounterGuard io_guard; exist = false; - ObString bucket_ob_string; - ObString object_ob_string; - common::ObArenaAllocator allocator(ObModIds::RESTORE); - char *remote_md5 = NULL; - int64_t file_length = 0; - ObStorageOssBase oss_base; - - if (uri.empty()) { - ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "name is empty", K(ret)); - } else if (!is_opened_) { - ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "oss util is not inited", K(ret), K(uri)); - } else if (OB_FAIL(oss_base.init_with_storage_info(storage_info_))) { - OB_LOG(WARN, "fail to init oss base with account", K(ret), K(uri)); - } else if (OB_FAIL(get_bucket_object_name(uri, bucket_ob_string, - object_ob_string, allocator))) { - OB_LOG(WARN, "bucket or object name is empty", - K(bucket_ob_string), K(object_ob_string), K(ret)); - } else if (OB_FAIL(oss_base.get_oss_file_meta(bucket_ob_string, object_ob_string, exist, - remote_md5, file_length))) { - OB_LOG(WARN, "fail to get file meta info", - K(bucket_ob_string), K(object_ob_string), K(ret)); + ObStorageObjectMetaBase obj_meta; + if (OB_FAIL(head_object_meta(uri, obj_meta))) { + OB_LOG(WARN, "fail to head object meta", K(uri), K(ret)); + } else { + exist = obj_meta.is_exist_; } - return ret; } int ObStorageOssUtil::get_file_length(const common::ObString &uri, int64_t &file_length) +{ + int ret = OB_SUCCESS; + file_length = 0; + ObStorageObjectMetaBase obj_meta; + if (OB_FAIL(head_object_meta(uri, obj_meta))) { + OB_LOG(WARN, "fail to head object meta", K(uri), K(ret)); + } else if (!obj_meta.is_exist_) { + ret = OB_BACKUP_FILE_NOT_EXIST; + OB_LOG(WARN, "backup file is not exist", K(uri), K(ret)); + } else { + file_length = obj_meta.length_; + } + return ret; +} + +int ObStorageOssUtil::head_object_meta(const common::ObString &uri, ObStorageObjectMetaBase &obj_meta) { int ret = OB_SUCCESS; ObExternalIOCounterGuard io_guard; - bool exist = false; ObString bucket_ob_string; ObString object_ob_string; - common::ObArenaAllocator allocator; + common::ObArenaAllocator allocator(OB_STORAGE_OSS_ALLOCATOR); char *remote_md5 = NULL; ObStorageOssBase oss_base; - file_length = 0; if (uri.empty()) { ret = OB_INVALID_ARGUMENT; @@ -1289,17 +1188,11 @@ int ObStorageOssUtil::get_file_length(const common::ObString &uri, int64_t &file OB_LOG(WARN, "oss util is not inited", K(ret), K(uri)); } else if (OB_FAIL(oss_base.init_with_storage_info(storage_info_))) { OB_LOG(WARN, "fail to init oss base with account", K(ret), K(uri)); - } else if (OB_SUCCESS != (ret = get_bucket_object_name(uri, bucket_ob_string, - object_ob_string, allocator))) { - OB_LOG(WARN, "bucket or object name is empty", - K(bucket_ob_string), K(object_ob_string), K(ret)); - } else if (OB_FAIL(oss_base.get_oss_file_meta(bucket_ob_string, object_ob_string, exist, - remote_md5, file_length))) { - OB_LOG(WARN, "fail to get file meta info", - K(bucket_ob_string), K(object_ob_string), K(ret)); - } else if (!exist) { - ret = OB_BACKUP_FILE_NOT_EXIST; - OB_LOG(WARN, "backup file is not exist", K(bucket_ob_string), K(object_ob_string), K(ret)); + } else if (OB_SUCCESS != (ret = get_bucket_object_name(uri, bucket_ob_string, object_ob_string, allocator))) { + OB_LOG(WARN, "bucket or object name is empty", K(bucket_ob_string), K(object_ob_string), K(ret)); + } else if (OB_FAIL(oss_base.get_oss_file_meta(bucket_ob_string, object_ob_string, obj_meta.is_exist_, + remote_md5, obj_meta.length_))) { + OB_LOG(WARN, "fail to get file meta info", K(bucket_ob_string), K(object_ob_string), K(ret)); } return ret; } @@ -1336,7 +1229,7 @@ int ObStorageOssUtil::is_tagging( int ret = OB_SUCCESS; const int64_t OB_MAX_TAGGING_STR_LENGTH = 16; ObExternalIOCounterGuard io_guard; - common::ObArenaAllocator allocator; + common::ObArenaAllocator allocator(OB_STORAGE_OSS_ALLOCATOR); ObString bucket_str; ObString object_str; ObStorageOssBase oss_base; @@ -1446,7 +1339,7 @@ int ObStorageOssUtil::del_file(const common::ObString &uri) { int ret = OB_SUCCESS; ObExternalIOCounterGuard io_guard; - common::ObArenaAllocator allocator; + common::ObArenaAllocator allocator(OB_STORAGE_OSS_ALLOCATOR); ObString bucket_str; ObString object_str; ObStorageOssBase oss_base; @@ -1475,16 +1368,19 @@ int ObStorageOssUtil::del_file(const common::ObString &uri) return ret; } -int handle_oss_file_name(const char* file_name_data, int file_name_len, - common::ObBaseDirEntryOperator &op) +int handle_oss_file_name( + const char* file_name_data, + int file_name_len, + common::ObBaseDirEntryOperator &op) { int ret = OB_SUCCESS; dirent entry; entry.d_type = DT_REG; - if (NAME_MAX < file_name_len) { + if (OB_ISNULL(file_name_data) || OB_UNLIKELY(sizeof(entry.d_name) <= file_name_len)) { ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "file name is too long", K(file_name_len)); + OB_LOG(WARN, "file name is too long", + K(ret), KP(file_name_data), K(file_name_len), K(sizeof(entry.d_name))); } else { MEMCPY(entry.d_name, file_name_data, file_name_len); entry.d_name[file_name_len] = '\0'; // set str end @@ -1495,122 +1391,132 @@ int handle_oss_file_name(const char* file_name_data, int file_name_len, return ret; } -int ObStorageOssUtil::list_files(const common::ObString &dir_path, +int ObStorageOssUtil::do_list_(ObStorageOssBase &oss_base, + const ObString &bucket_str, const char *full_dir_path, + const int64_t max_ret, const char *delimiter, + const char *next_marker, oss_list_object_params_t *¶ms) +{ + int ret = OB_SUCCESS; + aos_string_t bucket; + aos_status_t *aos_ret = NULL; + aos_table_t *resp_headers = NULL; + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_NOT_INIT; + OB_LOG(WARN, "oss util is not inited", K(ret)); + } else if (OB_UNLIKELY(!oss_base.is_inited() + || bucket_str.empty() || !is_end_with_slash(full_dir_path) + || max_ret <= 0 || max_ret > OB_STORAGE_LIST_MAX_NUM)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid arguments", K(ret), + K(oss_base.is_inited()), K(bucket_str), K(max_ret), K(full_dir_path)); + } else { + if (NULL == params) { + if (OB_ISNULL(params = oss_create_list_object_params(oss_base.aos_pool_))) { + ret = OB_OSS_ERROR; + OB_LOG(WARN, "fail to create list object params", K(ret), K(bucket_str), K(full_dir_path)); + } + } else { + // reuse params + aos_list_init(¶ms->object_list); + aos_list_init(¶ms->common_prefix_list); + } + + if (OB_SUCC(ret)) { + params->max_ret = max_ret; + aos_str_set(&bucket, bucket_str.ptr()); + aos_str_set(¶ms->prefix, full_dir_path); + if (NULL != next_marker && strlen(next_marker) > 0) { + aos_str_set(¶ms->marker, next_marker); + } + if (NULL != delimiter && strlen(delimiter) > 0) { + aos_str_set(¶ms->delimiter, delimiter); + } + } + + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(aos_ret = oss_list_object(oss_base.oss_option_, &bucket, + params, &resp_headers)) + || !aos_status_is_ok(aos_ret)) { + convert_io_error(aos_ret, ret); + OB_LOG(WARN, "fail to list oss objects", K(ret), K(bucket_str), K(full_dir_path)); + oss_base.print_oss_info(resp_headers, aos_ret); + } + } + return ret; +} + +int ObStorageOssUtil::list_files( + const common::ObString &uri, common::ObBaseDirEntryOperator &op) { int ret = OB_SUCCESS; ObExternalIOCounterGuard io_guard; - common::ObArenaAllocator tmp_allocator; + common::ObArenaAllocator tmp_allocator(OB_STORAGE_OSS_ALLOCATOR); ObString bucket_str; ObString object_str; - char object_dir_str[OB_MAX_URI_LENGTH] = {0}; - const char *object_dir_ptr = NULL; - aos_status_t *aos_ret = NULL; - oss_list_object_params_t *params = NULL; - const char *next_marker = "/"; - const char end_marker = '\0'; - //object_str end with '\0', so object_str.length() need > 2 - const int64_t min_object_str_len = 2; ObStorageOssBase oss_base; + const char *full_dir_path = NULL; - if (dir_path.empty()) { + if (OB_UNLIKELY(uri.empty())) { ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "name is empty", K(ret), K(dir_path)); - } else if (!is_opened_) { + OB_LOG(WARN, "uri is invalid", K(ret), K(uri)); + } else if (OB_UNLIKELY(!is_opened_)) { ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "oss util is not inited", K(ret), K(dir_path)); + OB_LOG(WARN, "oss util is not inited", K(ret), K(uri)); } else if (OB_FAIL(oss_base.init_with_storage_info(storage_info_))) { - OB_LOG(WARN, "fail to init oss base with account", K(ret), K(dir_path)); - } else if (OB_FAIL(get_bucket_object_name(dir_path, bucket_str, object_str, tmp_allocator))) { - OB_LOG(WARN, "bucket or object name is empty", K(ret), K(dir_path), K(bucket_str), K(object_str)); - } else if (object_str.length() < min_object_str_len) { - ret = OB_ERR_UNEXPECTED; - OB_LOG(WARN, "object str length is unepxected", K(ret), K(object_str), "length", object_str.length()); + OB_LOG(WARN, "fail to init oss base with account", K(ret), K(uri), KPC_(storage_info)); + } else if (OB_FAIL(get_bucket_object_name(uri, bucket_str, object_str, tmp_allocator))) { + OB_LOG(WARN, "bucket or object name is empty", K(ret), K(uri), K(bucket_str), K(object_str)); + } else if (FALSE_IT(full_dir_path = object_str.ptr())) { + } else if (OB_UNLIKELY(!is_end_with_slash(full_dir_path))) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "uri is not terminated with '/'", K(ret), K(uri), K(full_dir_path)); } else { - //make path end with '/' - if (object_str.ptr()[object_str.length() - 1] != end_marker) { - ret = OB_ERR_UNEXPECTED; - OB_LOG(WARN, "object str not end with '\0'", K(ret), K(object_str)); - } else if (object_str.ptr()[object_str.length() - 2] == *next_marker) { - object_dir_ptr = object_str.ptr(); - } else { - int n = snprintf(object_dir_str, OB_MAX_URI_LENGTH, "%s/", object_str.ptr()); - if (n <= 0 || n >= OB_MAX_URI_LENGTH) { - ret = OB_SIZE_OVERFLOW; - OB_LOG(WARN, "fail to deep copy object", K(object_str), K(object_str.length()), - K(n), K(OB_MAX_URI_LENGTH), K(ret)); - } else { - object_dir_ptr = object_dir_str; - } - } - } - - if (OB_FAIL(ret)) { - } else if (OB_ISNULL(params = oss_create_list_object_params(oss_base.aos_pool_))) { - ret = OB_OSS_ERROR; - OB_LOG(WARN, "fail to create list object params", K(ret), K(dir_path)); - } else if (OB_ISNULL(object_dir_ptr)) { - ret = OB_ERR_UNEXPECTED; - OB_LOG(WARN, "object_dir_ptr is null", K(ret), K(dir_path)); - } else { - aos_str_set(¶ms->prefix, object_dir_ptr); - aos_str_set(¶ms->marker, next_marker); - aos_string_t bucket; - aos_str_set(&bucket, bucket_str.ptr()); + ObString object_path; + const char *next_marker = ""; + const int64_t full_dir_path_len = strlen(full_dir_path); oss_list_object_content_t *content = NULL; - ObString tmp_string; - aos_table_t *resp_headers = NULL; - - + oss_list_object_params_t *params = NULL; do { - tmp_string.reset(); - if (OB_ISNULL(aos_ret = oss_list_object(oss_base.oss_option_, &bucket, params, &resp_headers)) - || !aos_status_is_ok(aos_ret)) { - convert_io_error(aos_ret, ret); - OB_LOG(WARN, "fail to list all object", K(ret)); - oss_base.print_oss_info(resp_headers, aos_ret); + if (OB_FAIL(do_list_(oss_base, bucket_str, full_dir_path, OB_STORAGE_LIST_MAX_NUM, + NULL/*delimiter*/, next_marker, params))) { + OB_LOG(WARN, "fail to list oss objects", + K(ret), K(bucket_str), K(full_dir_path), K(next_marker)); } else { + object_path.reset(); aos_list_for_each_entry(oss_list_object_content_t, content, ¶ms->object_list, node) { //key.data has prefix object_str, we only get the file name - tmp_string.assign(content->key.data, content->key.len); - const int64_t object_len = strlen(object_dir_ptr); - if (0 != MEMCMP(tmp_string.ptr(), object_dir_ptr, object_len)) { + object_path.assign(content->key.data, content->key.len); + int64_t object_size = -1; + + if (OB_UNLIKELY(false == object_path.prefix_match(full_dir_path))) { ret = OB_OSS_ERROR; - OB_LOG(WARN, "the date has no object prefix", K(ret), K(tmp_string), - K(object_str), K(tmp_string.length()), K(object_len)); - } else { - const int64_t file_name_pos = object_len; - const int32_t name_length = static_cast(content->key.len - file_name_pos); - ObString tmp_file_name(name_length, name_length, content->key.data + file_name_pos); - if (op.need_get_file_size()) { - const int64_t object_size = aos_atoi64(content->size.data); - if (0 > object_size) { - ObString tmp_size_string(content->size.data); - ret = OB_OSS_ERROR; - OB_LOG(WARN, "invalid object size", K(ret), K(object_size), K(tmp_size_string), K(tmp_file_name)); - } else { - op.set_size(object_size); - } - } - if (OB_FAIL(handle_oss_file_name(tmp_file_name.ptr(), tmp_file_name.length(), op))) { - OB_LOG(WARN, "fail to handle oss file name", K(ret), K(tmp_file_name)); - } else { - OB_LOG(DEBUG, "get file name", K(tmp_string)); - } + OB_LOG(WARN, "returned object prefix not match", + K(ret), K(object_path), K(full_dir_path), K(uri)); + } else if (OB_UNLIKELY(object_path.length() == full_dir_path_len)) { + // skip + OB_LOG(INFO, "exist object path length is same with dir path length", + K(object_path), K(full_dir_path), K(full_dir_path_len)); + } else if (OB_FAIL(c_str_to_int(content->size.data, object_size))) { + OB_LOG(WARN, "fail to get object size", K(ret), K(content->size.data), K(object_path)); + } else if (OB_FAIL(handle_listed_object(op, content->key.data + full_dir_path_len, + content->key.len - full_dir_path_len, + object_size))) { + OB_LOG(WARN, "fail to handle oss file name", + K(ret), K(object_path), K(full_dir_path), K(content->size.data), K(object_size)); } + if (OB_FAIL(ret)) { break; } - } + } // end aos_list_for_each_entry } - if (OB_SUCC(ret)) { - if (NULL == (next_marker = apr_psprintf(oss_base.aos_pool_, "%.*s", params->next_marker.len, params->next_marker.data))) { + if (OB_SUCC(ret) && AOS_TRUE == params->truncated) { + if (NULL == (next_marker = apr_psprintf(oss_base.aos_pool_, "%.*s", + params->next_marker.len, + params->next_marker.data))) { ret = OB_OSS_ERROR; - OB_LOG(WARN, "next marker is NULL", K(ret), KP(next_marker)); - } else { - aos_str_set(¶ms->marker, next_marker); - aos_list_init(¶ms->object_list); - aos_list_init(¶ms->common_prefix_list); + OB_LOG(WARN, "next marker is NULL", K(ret), KP(next_marker), K(params->next_marker.data)); } } } while (AOS_TRUE == params->truncated && OB_SUCC(ret)); @@ -1619,6 +1525,87 @@ int ObStorageOssUtil::list_files(const common::ObString &dir_path, return ret; } +int ObStorageOssUtil::list_files( + const common::ObString &uri, + ObStorageListCtxBase &ctx_base) +{ + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + common::ObArenaAllocator tmp_allocator(OB_STORAGE_OSS_ALLOCATOR); + ObString bucket_str; + ObString object_str; + ObStorageOssBase oss_base; + const char *full_dir_path = NULL; + ObStorageListObjectsCtx &list_ctx = static_cast(ctx_base); + + if (OB_UNLIKELY(uri.empty() || !list_ctx.is_valid())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid arguments", K(ret), K(uri), K(list_ctx)); + } else if (OB_UNLIKELY(!is_opened_)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "oss util is not inited", K(ret), K(uri)); + } else if (OB_FAIL(oss_base.init_with_storage_info(storage_info_))) { + OB_LOG(WARN, "fail to init oss base with account", K(ret), K(uri), KPC_(storage_info)); + } else if (OB_FAIL(get_bucket_object_name(uri, bucket_str, object_str, tmp_allocator))) { + OB_LOG(WARN, "bucket or object name is empty", K(ret), K(uri), K(bucket_str), K(object_str)); + } else if (FALSE_IT(full_dir_path = object_str.ptr())) { + } else if (OB_UNLIKELY(!is_end_with_slash(full_dir_path))) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "uri is not terminated with '/'", K(ret), K(uri), K(full_dir_path)); + } else { + ObString object_path; + const int64_t full_dir_path_len = strlen(full_dir_path); + oss_list_object_content_t *content = NULL; + oss_list_object_params_t *params = NULL; + const char *next_marker = ""; + if (list_ctx.next_token_ != NULL && list_ctx.next_token_[0] != '\0') { + next_marker = list_ctx.next_token_; + } + const int64_t max_ret = MIN(OB_STORAGE_LIST_MAX_NUM, list_ctx.max_list_num_); + + if (OB_FAIL(do_list_(oss_base, bucket_str, full_dir_path, max_ret, + NULL/*delimiter*/, next_marker, params))) { + OB_LOG(WARN, "fail to list oss objects", + K(ret), K(bucket_str), K(full_dir_path), K(max_ret), K(next_marker)); + } else if (OB_FAIL(list_ctx.set_next_token(AOS_TRUE == params->truncated, + params->next_marker.data, + params->next_marker.len))) { + OB_LOG(WARN, "fail to set next token when listing oss objects", + K(ret), K(params->truncated), K(params->next_marker.data)); + } else { + object_path.reset(); + aos_list_for_each_entry(oss_list_object_content_t, content, ¶ms->object_list, node) { + //key.data has prefix object_str, we only get the file name + object_path.assign(content->key.data, content->key.len); + int64_t object_size = -1; + + if (OB_UNLIKELY(false == object_path.prefix_match(full_dir_path))) { + ret = OB_OSS_ERROR; + OB_LOG(WARN, "returned object prefix not match", + K(ret), K(object_path), K(full_dir_path), K(uri)); + } else if (OB_UNLIKELY(object_path.length() == full_dir_path_len)) { + // skip + OB_LOG(INFO, "exist object path length is same with dir path length", + K(object_path), K(full_dir_path), K(full_dir_path_len)); + } else if (OB_FAIL(c_str_to_int(content->size.data, object_size))) { + OB_LOG(WARN, "fail to get object size", K(ret), K(content->size.data), K(object_path)); + } else if (OB_FAIL(list_ctx.handle_object(content->key.data, + content->key.len, + object_size))) { + OB_LOG(WARN, "fail to add listed oss obejct meta into ctx", + K(ret), K(object_path), K(full_dir_path), K(content->size.data), K(object_size)); + } + + if (OB_FAIL(ret)) { + break; + } + } // end aos_list_for_each_entry + } + } + + return ret; +} + int ObStorageOssUtil::del_dir(const common::ObString &uri) { int ret = OB_SUCCESS; @@ -1647,148 +1634,180 @@ int ObStorageOssUtil::strtotime(const char *date_time, int64_t &time) } int ObStorageOssUtil::list_directories( - const common::ObString &dir_path, - common::ObBaseDirEntryOperator &op) + const common::ObString &uri, + common::ObBaseDirEntryOperator &op) { int ret = OB_SUCCESS; ObExternalIOCounterGuard io_guard; - common::ObArenaAllocator tmp_allocator; + common::ObArenaAllocator tmp_allocator(OB_STORAGE_OSS_ALLOCATOR); ObString bucket_str; ObString object_str; - char object_dir_str[OB_MAX_URI_LENGTH] = {0}; - const char *object_dir_ptr = NULL; - aos_status_t *aos_ret = NULL; - oss_list_object_params_t *params = NULL; - const char *next_marker = ""; - const char end_marker = '\0'; - const char *delimiter = "/"; - //object_str end with '\0', so object_str.length() need > 2 - const int64_t min_object_str_len = 2; ObStorageOssBase oss_base; + const char *full_dir_path = NULL; - if (dir_path.empty()) { + if (OB_UNLIKELY(uri.empty())) { ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "name is empty", K(ret), K(dir_path)); - } else if (!is_opened_) { + OB_LOG(WARN, "name is empty", K(ret), K(uri)); + } else if (OB_UNLIKELY(!is_opened_)) { ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "oss util is not inited", K(ret), K(dir_path)); + OB_LOG(WARN, "oss util is not inited", K(ret), K(uri)); } else if (OB_FAIL(oss_base.init_with_storage_info(storage_info_))) { - OB_LOG(WARN, "fail to init oss base with account", K(ret), K(dir_path)); - } else if (OB_FAIL(get_bucket_object_name(dir_path, bucket_str, object_str, tmp_allocator))) { - OB_LOG(WARN, "bucket or object name is empty", K(ret), K(dir_path), K(bucket_str), K(object_str)); - } else if (object_str.length() < min_object_str_len) { - ret = OB_ERR_UNEXPECTED; - OB_LOG(WARN, "object str length is unepxected", K(ret), K(object_str), "length", object_str.length()); + OB_LOG(WARN, "fail to init oss base with account", K(ret), K(uri)); + } else if (OB_FAIL(get_bucket_object_name(uri, bucket_str, object_str, tmp_allocator))) { + OB_LOG(WARN, "bucket or object name is empty", K(ret), K(uri), K(bucket_str), K(object_str)); + } else if (FALSE_IT(full_dir_path = object_str.ptr())) { + } else if (OB_UNLIKELY(!is_end_with_slash(full_dir_path))) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "uri is not terminated with '/'", K(ret), K(uri), K(full_dir_path)); } else { - //make path end with '/' - if (object_str.ptr()[object_str.length() - 1] != end_marker) { - ret = OB_ERR_UNEXPECTED; - OB_LOG(WARN, "object str not end with '\0'", K(ret), K(object_str)); - } else if (object_str.ptr()[object_str.length() - 2] == *delimiter) { - object_dir_ptr = object_str.ptr(); - } else { - int n = snprintf(object_dir_str, OB_MAX_URI_LENGTH, "%s/", object_str.ptr()); - if (n <= 0 || n >= OB_MAX_URI_LENGTH) { - ret = OB_SIZE_OVERFLOW; - OB_LOG(WARN, "fail to deep copy object", K(object_str), K(object_str.length()), - K(n), K(OB_MAX_URI_LENGTH), K(ret)); - } else { - object_dir_ptr = object_dir_str; - } - } - } - - if (OB_FAIL(ret)) { - } else if (OB_ISNULL(params = oss_create_list_object_params(oss_base.aos_pool_))) { - ret = OB_OSS_ERROR; - OB_LOG(WARN, "fail to create list object params", K(ret), K(dir_path)); - } else if (OB_ISNULL(object_dir_ptr)) { - ret = OB_ERR_UNEXPECTED; - OB_LOG(WARN, "object_dir_ptr is null", K(ret), K(dir_path)); - } else { - aos_str_set(¶ms->prefix, object_dir_ptr); - aos_str_set(¶ms->marker, next_marker); - aos_str_set(¶ms->delimiter, delimiter); - aos_string_t bucket; - aos_str_set(&bucket, bucket_str.ptr()); - aos_table_t *resp_headers; + ObString listed_dir_full_path; + const char *next_marker = ""; + const char *delimiter = "/"; + const int64_t full_dir_path_len = strlen(full_dir_path); oss_list_object_common_prefix_t *common_prefix = NULL; - ObString tmp_string; - ObString directory_name; + oss_list_object_params_t *params = NULL; do { - tmp_string.reset(); - directory_name.reset(); - if (OB_ISNULL(aos_ret = oss_list_object(oss_base.oss_option_, &bucket, params, &resp_headers)) || !aos_status_is_ok(aos_ret)) { - convert_io_error(aos_ret, ret); - OB_LOG(WARN, "fail to list all object", K(ret)); - oss_base.print_oss_info(resp_headers, aos_ret); + if (OB_FAIL(do_list_(oss_base, bucket_str, full_dir_path, OB_STORAGE_LIST_MAX_NUM, + delimiter, next_marker, params))) { + OB_LOG(WARN, "fail to list s3 objects", K(ret), K(uri), K(delimiter), K(next_marker)); } else { aos_list_for_each_entry(oss_list_object_common_prefix_t, common_prefix, ¶ms->common_prefix_list, node) { - //key.data has prefix object_str, we only get the file name - tmp_string.assign(common_prefix->prefix.data, common_prefix->prefix.len); - const int64_t object_len = strlen(object_dir_ptr); - const int64_t prefix_len = strlen(common_prefix->prefix.data); - if (prefix_len < object_len) { + // For example, + // dir1 + // --file1 + // --dir11 + // --file11 + // if we list directories in 'dir1', then full_dir_path == 'dir1/' + // and listed_dir_full_path == 'dir1/dir11/', which represents the full directory path of 'dir11' + listed_dir_full_path.assign(common_prefix->prefix.data, common_prefix->prefix.len); + if (OB_UNLIKELY(false == listed_dir_full_path.prefix_match(full_dir_path))) { ret = OB_OSS_ERROR; - OB_LOG(WARN, "the object prefix len should not be smaller than directory len", K(ret), K(tmp_string), - K(object_str), K(tmp_string.length()), K(object_len), K(object_len), K(prefix_len)); - } else if (0 != MEMCMP(tmp_string.ptr(), object_dir_ptr, object_len)) { - ret = OB_OSS_ERROR; - OB_LOG(WARN, "the date has no object prefix", K(ret), K(tmp_string), - K(object_str), K(tmp_string.length()), K(object_len)); - } else if (*delimiter != tmp_string.ptr()[common_prefix->prefix.len - 1]) { - ret = OB_OSS_ERROR; - OB_LOG(WARN, "the date has no directory", K(ret), K(tmp_string), - K(object_str), K(tmp_string.length()), K(object_len)); - } else { - const int64_t directory_name_pos = object_len; - const int32_t name_length = static_cast(common_prefix->prefix.len - directory_name_pos) - 1; //remove '/' - if (name_length <= 0) { - ret = OB_OSS_ERROR; - OB_LOG(WARN, "the object do not exist", K(ret), K(tmp_string), - K(object_str), K(tmp_string.length()), K(object_len), K(name_length), K(directory_name_pos)); - } else { - ObString tmp_directory_name(name_length, name_length, common_prefix->prefix.data + directory_name_pos); - dirent entry; - entry.d_type = DT_DIR; - if (OB_FAIL(databuff_printf(entry.d_name, sizeof(entry.d_name), - "%.*s", tmp_directory_name.length(), tmp_directory_name.ptr()))) { - OB_LOG(WARN, "fail to copy name to entry", K(ret), K(tmp_directory_name)); - } else if (OB_FAIL(op.func(&entry))) { - OB_LOG(WARN, "fail to call operation callback", K(ret), K(tmp_directory_name)); - } - } - OB_LOG(DEBUG, "get directory name", K(tmp_string), K(directory_name)); + OB_LOG(WARN, "returned object prefix not match", + K(ret), K(listed_dir_full_path), K(full_dir_path), K(uri)); + } else if (OB_UNLIKELY(!is_end_with_slash(listed_dir_full_path.ptr()))) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "the data has no directory", + K(ret), K(full_dir_path), K(listed_dir_full_path), K(uri)); + } else if (OB_FAIL(handle_listed_directory(op, + common_prefix->prefix.data + full_dir_path_len, + common_prefix->prefix.len - 1 - full_dir_path_len))) { + // common_prefix->prefix.len is the length of listed_dir_full_path, including the trailing '/' character. + // Therefore, the length for dir_name needs to exclude both the prefix full_dir_path and the trailing '/'. + OB_LOG(WARN, "fail to handle oss directory name", + K(ret), K(listed_dir_full_path), K(full_dir_path), K(full_dir_path_len)); } + if (OB_FAIL(ret)) { break; } - } + } // end aos_list_for_each_entry } - if (OB_SUCC(ret)) { - if (NULL == (next_marker = apr_psprintf(oss_base.aos_pool_, "%.*s", params->next_marker.len, params->next_marker.data))) { + if (OB_SUCC(ret) && AOS_TRUE == params->truncated) { + if (NULL == (next_marker = apr_psprintf(oss_base.aos_pool_, "%.*s", + params->next_marker.len, + params->next_marker.data))) { ret = OB_OSS_ERROR; - OB_LOG(WARN, "next marker is NULL", K(ret), KP(next_marker)); - } else { - aos_str_set(¶ms->marker, next_marker); - aos_list_init(¶ms->object_list); - aos_list_init(¶ms->common_prefix_list); + OB_LOG(WARN, "next marker is NULL", K(ret), KP(next_marker), K(params->next_marker.data)); } } } while (AOS_TRUE == params->truncated && OB_SUCC(ret)); } - OB_LOG(INFO, "list directory count", K(dir_path)); + OB_LOG(INFO, "list directory count", K(uri)); return ret; } +int ObStorageOssUtil::del_unmerged_parts(const ObString &uri) +{ + int ret = OB_SUCCESS; + oss_list_multipart_upload_params_t *params = NULL; + const char *next_key_marker = ""; + const char *next_upload_id_marker = ""; + ObString bucket_str; + ObString object_str; + ObStorageOssBase oss_base; + ObArenaAllocator allocator(OB_STORAGE_OSS_ALLOCATOR); + ObExternalIOCounterGuard io_guard; + + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "oss util is not inited", K(ret), K(uri)); + } else if (OB_UNLIKELY(uri.empty())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "uri is empty", K(ret), K(uri)); + } else if (OB_FAIL(oss_base.init_with_storage_info(storage_info_))) { + OB_LOG(WARN, "fail to init oss base with account", K(ret), K(uri), KPC_(storage_info)); + } else if (OB_FAIL(get_bucket_object_name(uri, bucket_str, object_str, allocator))) { + OB_LOG(WARN, "bucket or object name is empty", K(ret), K(uri), K(bucket_str), K(object_str)); + } else if (OB_ISNULL(params = oss_create_list_multipart_upload_params(oss_base.aos_pool_))) { + ret = OB_OSS_ERROR; + OB_LOG(WARN, "fail to create list oss multipart upload params", K(ret), K(bucket_str)); + } else { + aos_str_set(¶ms->prefix, object_str.ptr()); + aos_str_set(¶ms->next_key_marker, next_key_marker); + aos_str_set(¶ms->next_upload_id_marker, next_upload_id_marker); + params->max_ret = OB_STORAGE_LIST_MAX_NUM; + aos_string_t bucket; + aos_string_t object; + aos_str_set(&bucket, bucket_str.ptr()); + oss_list_multipart_upload_content_t *content = NULL; + aos_table_t *resp_headers = NULL; + aos_status_t *aos_ret = NULL; + + do { + if (OB_ISNULL(aos_ret = oss_list_multipart_upload(oss_base.oss_option_, &bucket, + params, &resp_headers)) + || !aos_status_is_ok(aos_ret)) { + convert_io_error(aos_ret, ret); + OB_LOG(WARN, "fail to list oss multipart uploads", K(ret), K(bucket_str)); + oss_base.print_oss_info(resp_headers, aos_ret); + } else { + aos_list_for_each_entry(oss_list_multipart_upload_content_t, content, ¶ms->upload_list, node) { + if (OB_ISNULL(aos_ret = oss_abort_multipart_upload(oss_base.oss_option_, &bucket, &(content->key), + &(content->upload_id), &resp_headers)) + || !aos_status_is_ok(aos_ret)) { + convert_io_error(aos_ret, ret); + OB_LOG(WARN, "fail to abort oss multipart upload", + K(ret), K(bucket_str), K(content->key.data), K(content->upload_id.data)); + oss_base.print_oss_info(resp_headers, aos_ret); + break; + } else { + OB_LOG(INFO, "succeed abort oss multipart upload", + K(bucket_str), K(content->key.data), K(content->upload_id.data)); + } + } + } + + if (OB_SUCC(ret) && AOS_TRUE == params->truncated) { + if (OB_ISNULL(next_key_marker = + apr_psprintf(oss_base.aos_pool_, "%.*s", + params->next_key_marker.len, + params->next_key_marker.data))) { + ret = OB_OSS_ERROR; + OB_LOG(WARN, "next key marker is NULL", K(ret), KP(next_key_marker)); + } else if (OB_ISNULL(next_upload_id_marker = + apr_psprintf(oss_base.aos_pool_, "%.*s", + params->next_upload_id_marker.len, + params->next_upload_id_marker.data))) { + ret = OB_OSS_ERROR; + OB_LOG(WARN, "next upload id marker is NULL", K(ret), KP(next_upload_id_marker)); + } else { + aos_str_set(¶ms->key_marker, next_key_marker); + aos_str_set(¶ms->upload_id_marker, next_upload_id_marker); + aos_list_init(¶ms->upload_list); + } + } + } while (OB_SUCC(ret) && AOS_TRUE == params->truncated); + } + return ret; +} ObStorageOssAppendWriter::ObStorageOssAppendWriter() : is_opened_(false), file_length_(-1), - allocator_(ObModIds::BACKUP), + allocator_(OB_STORAGE_OSS_ALLOCATOR), bucket_(), object_() { @@ -1834,7 +1853,7 @@ int ObStorageOssAppendWriter::write(const char *buf, if (!is_inited()) { ret = OB_NOT_INIT; OB_LOG(WARN, "oss client not inited", K(ret)); - } else if(NULL == buf || size < 0) { + } else if(NULL == buf || size <= 0) { ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "buf is NULL or size is invalid", KP(buf), K(size), K(ret)); } else if (OB_FAIL(do_write(buf, size, fake_offset, is_pwrite))) { @@ -1852,9 +1871,9 @@ int ObStorageOssAppendWriter::pwrite(const char *buf, const int64_t size, const if (!is_inited()) { ret = OB_NOT_INIT; OB_LOG(WARN, "oss client not inited", K(ret)); - } else if(NULL == buf || size < 0 || offset < 0) { + } else if(NULL == buf || size <= 0 || offset < 0) { ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "buf is NULL or size is invalid", KP(buf), K(size), K(ret)); + OB_LOG(WARN, "invalid arguments", KP(buf), K(size), K(ret), K(offset)); } else if (OB_FAIL(do_write(buf, size, offset, is_pwrite))) { OB_LOG(WARN, "failed to do write", K(ret), KP(buf), K(size), K(offset)); } @@ -1865,20 +1884,9 @@ int ObStorageOssAppendWriter::close() { int ret = OB_SUCCESS; ObExternalIOCounterGuard io_guard; - - if(!is_inited()) { - ret = OB_NOT_INIT; - OB_LOG(WARN, "oss client not inited", K(ret)); - } else if(!is_opened_) { - ret = OB_OSS_ERROR; - OB_LOG(WARN, "oss writer cannot close before it is opened"); - } else { - is_opened_ = false; - // �ͷ��ڴ� - allocator_.clear(); - - reset(); - } + is_opened_ = false; + allocator_.clear(); + reset(); return ret; } @@ -1891,7 +1899,7 @@ int ObStorageOssAppendWriter::do_write(const char *buf, const int64_t size, cons if (!is_inited()) { ret = OB_NOT_INIT; OB_LOG(WARN, "oss client not inited", K(ret)); - } else if(NULL == buf || size < 0) { + } else if(NULL == buf || size <= 0) { ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "buf is NULL or size is invalid", KP(buf), K(size), K(ret)); } else { @@ -1932,8 +1940,9 @@ int ObStorageOssAppendWriter::do_write(const char *buf, const int64_t size, cons if (OB_ISNULL(next_append_position)) { ret = OB_OSS_ERROR; OB_LOG(WARN, "next_append_position is not found", K(ret), K_(bucket), K_(object)); + } else if (OB_FAIL(c_str_to_int(next_append_position, position))) { + OB_LOG(WARN, "fail to get position", K(ret), K(next_append_position), K_(object)); } else { - position = aos_atoi64(next_append_position); if (0 > position) { ObString tmp_position_string(next_append_position); ret = OB_OSS_ERROR; @@ -1968,14 +1977,18 @@ int ObStorageOssAppendWriter::do_write(const char *buf, const int64_t size, cons // If append failed, print the current object meta, to help debugging. aos_table_t *headers3 = NULL; + int tmp_ret = OB_SUCCESS; if(OB_NOT_NULL(headers3 = aos_table_make(aos_pool_, 0))) { if(OB_NOT_NULL(aos_ret = oss_head_object(oss_option_, &bucket, &object, headers3, &resp_headers))) { if ((0 != aos_status_is_ok(aos_ret))) { + int64_t cur_pos = -1; char *append_pos_str = (char*)(apr_table_get(resp_headers, OSS_NEXT_APPEND_POSITION)); if (OB_ISNULL(append_pos_str)) { OB_LOG(WARN, "after append fail, current append pos is not found"); + } else if (OB_TMP_FAIL(c_str_to_int(append_pos_str, cur_pos))) { + OB_LOG(WARN, "after append fail, fail to get append pos", + K(tmp_ret), K(append_pos_str), K_(object)); } else { - int64_t cur_pos = aos_atoi64(append_pos_str); OB_LOG(WARN, "after append fail, we got the object meta", K(cur_pos)); } } @@ -2007,7 +2020,7 @@ int ObStorageOssAppendWriter::do_write(const char *buf, const int64_t size, cons ObStorageOssWriter::ObStorageOssWriter() : is_opened_(false), file_length_(-1), - allocator_(ObModIds::BACKUP), + allocator_(OB_STORAGE_OSS_ALLOCATOR), bucket_(), object_() { @@ -2045,17 +2058,11 @@ int ObStorageOssWriter::open( int ObStorageOssWriter::close() { int ret = OB_SUCCESS; - - if(!is_opened_) { - ret = OB_OSS_ERROR; - OB_LOG(WARN, "oss writer cannot close before it is opened"); - } else { - is_opened_ = false; - // Release memory - allocator_.clear(); - file_length_ = -1; - reset(); - } + is_opened_ = false; + // Release memory + allocator_.clear(); + file_length_ = -1; + reset(); return ret; } diff --git a/deps/oblib/src/lib/restore/ob_storage_oss_base.h b/deps/oblib/src/lib/restore/ob_storage_oss_base.h index eb3efc3a2..53c8a74bc 100644 --- a/deps/oblib/src/lib/restore/ob_storage_oss_base.h +++ b/deps/oblib/src/lib/restore/ob_storage_oss_base.h @@ -51,6 +51,7 @@ const static int64_t MAX_OSS_KEY_LENGTH = 128; const static int64_t OSS_BASE_BUFFER_SIZE = 8 * 1024 * 1024L;//the buf size of upload data const static int64_t MAX_ELEMENT_COUNT = 10000;//oss limit element count const static int64_t MULTI_BASE_BUFFER_SIZE = 16 * 1024 * 1024L;//the buf size of upload data +static constexpr char OB_STORAGE_OSS_ALLOCATOR[] = "StorageOSS"; // Before using oss, you need to initialize oss enviroment. // Thread safe guaranteed by user. @@ -60,7 +61,6 @@ int init_oss_env(); // Thread safe guaranteed by user. void fin_oss_env(); - class ObStorageOssStaticVar { public: @@ -184,7 +184,6 @@ private: common::ObString object_; aos_string_t upload_id_; int partnum_; - MD5_CTX whole_file_md5_; bool is_opened_; int64_t file_length_; @@ -197,8 +196,10 @@ class ObStorageOssReader: public ObStorageOssBase, public ObIStorageReader public: ObStorageOssReader(); virtual ~ObStorageOssReader(); - int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info); - int pread(char *buf,const int64_t buf_size, int64_t offset, int64_t &read_size); + virtual int open(const common::ObString &uri, + common::ObObjectStorageInfo *storage_info, const bool head_meta = true) override; + virtual int pread(char *buf, + const int64_t buf_size, const int64_t offset, int64_t &read_size) override; int close(); int64_t get_length() const { return file_length_; } virtual bool is_opened() const { return is_opened_; } @@ -208,6 +209,7 @@ private: common::ObString object_; int64_t file_length_; bool is_opened_; + bool has_meta_; common::ObArenaAllocator allocator_; DISALLOW_COPY_AND_ASSIGN(ObStorageOssReader); @@ -222,16 +224,19 @@ public: virtual void close(); virtual int is_exist(const common::ObString &uri, bool &exist); virtual int get_file_length(const common::ObString &uri, int64_t &file_length); + virtual int head_object_meta(const common::ObString &uri, ObStorageObjectMetaBase &obj_meta); virtual int write_single_file(const common::ObString &uri, const char *buf, const int64_t size); //oss no dir virtual int mkdir(const common::ObString &uri); virtual int del_file(const common::ObString &uri); - virtual int list_files(const common::ObString &dir_path, common::ObBaseDirEntryOperator &op); + virtual int list_files(const common::ObString &uri, common::ObBaseDirEntryOperator &op); + virtual int list_files(const common::ObString &uri, ObStorageListCtxBase &list_ctx); virtual int del_dir(const common::ObString &uri); virtual int list_directories(const common::ObString &uri, common::ObBaseDirEntryOperator &op); virtual int is_tagging(const common::ObString &uri, bool &is_tagging); + virtual int del_unmerged_parts(const ObString &uri) override; private: int strtotime(const char *date_time, int64_t &time); int tagging_object_( @@ -244,6 +249,10 @@ private: ObStorageOssBase &oss_base, const common::ObString &bucket_str, const common::ObString &object_str); + int do_list_(ObStorageOssBase &oss_base, + const ObString &bucket, const char *full_dir_path, + const int64_t max_ret, const char *delimiter, + const char *next_marker, oss_list_object_params_t *¶ms); bool is_opened_; common::ObObjectStorageInfo *storage_info_; }; diff --git a/deps/oblib/src/lib/restore/ob_storage_s3_base.cpp b/deps/oblib/src/lib/restore/ob_storage_s3_base.cpp new file mode 100644 index 000000000..6a617d85d --- /dev/null +++ b/deps/oblib/src/lib/restore/ob_storage_s3_base.cpp @@ -0,0 +1,2096 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "lib/restore/ob_storage.h" +#include "ob_storage_s3_base.h" + +namespace oceanbase +{ +namespace common +{ +using namespace Aws::S3; +using namespace Aws::Client; +using namespace Aws::Utils; + +/*--------------------------------ObS3Logger--------------------------------*/ +Logging::LogLevel ObS3Logger::GetLogLevel(void) const +{ + Logging::LogLevel log_level = Logging::LogLevel::Info; + int32_t ob_log_level = OB_LOGGER.get_log_level(); + switch (ob_log_level) { + case OB_LOG_LEVEL_INFO: + break; + case OB_LOG_LEVEL_ERROR: + log_level = Logging::LogLevel::Error; + break; + case OB_LOG_LEVEL_WARN: + log_level = Logging::LogLevel::Warn; + break; + case OB_LOG_LEVEL_TRACE: + log_level = Logging::LogLevel::Debug; + break; + case OB_LOG_LEVEL_DEBUG: + log_level = Logging::LogLevel::Trace; + break; + default: + break; + } + return log_level; +} + +void ObS3Logger::Log(Logging::LogLevel logLevel, const char* tag, const char* formatStr, ...) +{ + int ret = OB_SUCCESS; + + const int64_t buf_len = 4096; + char arg_buf[buf_len] = {0}; + va_list args; + va_start(args, formatStr); + int psize = vsnprintf(arg_buf, buf_len - 1, formatStr, args); + va_end(args); + + if (psize > 0) { + const char *new_format = "[S3] module=%s, %s"; + switch (logLevel) { + case Logging::LogLevel::Fatal: + case Logging::LogLevel::Error: + ret = OB_S3_ERROR; + _OB_LOG(ERROR, new_format, tag, arg_buf); + break; + case Logging::LogLevel::Warn: + ret = OB_S3_ERROR; + _OB_LOG(WARN, new_format, tag, arg_buf); + break; + case Logging::LogLevel::Info: + _OB_LOG(INFO, new_format, tag, arg_buf); + break; + // NOTICE: the s3 Debug and Trace level keeps the reverse order with them in ob. + case Logging::LogLevel::Trace: + _OB_LOG(DEBUG, new_format, tag, arg_buf); + break; + case Logging::LogLevel::Debug: + _OB_LOG(TRACE, new_format, tag, arg_buf); + break; + default: + _OB_LOG(WARN, new_format, tag, arg_buf); + break; + } + } +} + +void ObS3Logger::LogStream(Logging::LogLevel logLevel, const char* tag, const Aws::OStringStream &messageStream) +{ + Log(logLevel, tag, "msg=%s", messageStream.str().c_str()); +} + +/*--------------------------------ObS3Client--------------------------------*/ +ObS3Client::ObS3Client() + : lock_(ObLatchIds::OBJECT_DEVICE_LOCK), + is_inited_(false), + stopped_(false), + ref_cnt_(0), + last_modified_ts_(0), + client_(nullptr) +{ +} + +ObS3Client::~ObS3Client() +{ + destroy(); +} + +int ObS3Client::init_s3_client_configuration_(const ObS3Account &account, + S3ClientConfiguration &config) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!account.is_valid())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "S3 account not valid", K(ret)); + } else { + config.region = account.region_; + config.scheme = Aws::Http::Scheme::HTTP; // if change to HTTPS, be careful about checksum logic. + config.verifySSL = false; + config.connectTimeoutMs = S3_CONNECT_TIMEOUT_MS; + config.requestTimeoutMs = S3_REQUEST_TIMEOUT_MS; + config.maxConnections = MAX_S3_CONNECTIONS_PER_CLIENT; + config.payloadSigningPolicy = Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never; + config.endpointOverride = account.endpoint_; + config.executor = nullptr; + } + return ret; +} + +int ObS3Client::init(const ObS3Account &account) +{ + int ret = OB_SUCCESS; + void *client_buf = NULL; + S3ClientConfiguration config; + Aws::Auth::AWSCredentials credentials(account.access_id_, account.secret_key_); + SpinWLockGuard guard(lock_); + if (OB_UNLIKELY(is_inited_)) { + ret = OB_INIT_TWICE; + OB_LOG(WARN, "ObS3Client init twice", K(ret)); + } else if (OB_UNLIKELY(!account.is_valid())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "S3 account not valid", K(ret)); + } else if (OB_FAIL(init_s3_client_configuration_(account, config))) { + OB_LOG(WARN, "failed to init s3 client config", K(ret), K(account)); + } else if (OB_ISNULL(client_buf = ob_malloc(sizeof(Aws::S3::S3Client), OB_STORAGE_S3_ALLOCATOR))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "failed to alloc buf for aws s3 client", K(ret)); + } else { + try { + client_ = new(client_buf) Aws::S3::S3Client(credentials, + Aws::MakeShared(S3_SDK), config); + } catch (const std::exception &e) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "caught exception when initing ObS3Client", + K(ret), K(e.what()), KP(this), K(*this)); + } catch (...) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "caught unknown exception when initing ObS3Client", K(ret), KP(this), K(*this)); + } + + if (OB_SUCC(ret)) { + last_modified_ts_ = ObTimeUtility::current_time(); + is_inited_ = true; + } else { + ob_free(client_buf); + } + } + return ret; +} + +void ObS3Client::destroy() +{ + SpinWLockGuard guard(lock_); + is_inited_ = false; + stopped_ = false; + ref_cnt_ = 0; + last_modified_ts_ = 0; + if (OB_NOT_NULL(client_)) { + client_->~S3Client(); + ob_free(client_); + client_ = NULL; + } +} + +bool ObS3Client::is_stopped() const +{ + SpinRLockGuard guard(lock_); + return stopped_; +} + +bool ObS3Client::try_stop(const int64_t timeout) +{ + bool is_stopped = true; + if (OB_SUCCESS == lock_.wrlock(timeout)) { + if (is_inited_) { + const int64_t cur_time = ObTimeUtility::current_time(); + if (ref_cnt_ <= 0 && cur_time - last_modified_ts_ >= MAX_S3_CLIENT_IDLE_DURATION) { + stopped_ = true; + } else { + is_stopped = false; + } + } + lock_.unlock(); + } else { + is_stopped = false; + } + return is_stopped; +} + +void ObS3Client::stop() +{ + SpinWLockGuard guard(lock_); + stopped_ = true; +} + +void ObS3Client::increase() +{ + SpinWLockGuard guard(lock_); + ref_cnt_++; + last_modified_ts_ = ObTimeUtility::current_time(); +} + +void ObS3Client::release() +{ + SpinWLockGuard guard(lock_); + ref_cnt_--; + last_modified_ts_ = ObTimeUtility::current_time(); +} + +template +int ObS3Client::do_s3_operation_(S3OperationFunc s3_op_func, + const RequestType &request, OutcomeType &outcome) +{ + int ret = OB_SUCCESS; + { + SpinRLockGuard guard(lock_); + if (!is_inited_) { + ret = OB_NOT_INIT; + OB_LOG(WARN, "ObS3Client not init", K(ret)); + } else if (stopped_) { + ret = OB_IN_STOP_STATE; + OB_LOG(WARN, "ObS3Client has been stopped", K(ret)); + } else if (OB_ISNULL(client_)) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "client is NULL in ObS3Client", K(ret), KP(client_)); + } + } + + if (OB_SUCC(ret)) { + outcome = (client_->*s3_op_func)(request); + last_modified_ts_ = ObTimeUtility::current_time(); + } + return ret; +} + +int ObS3Client::head_object(const Aws::S3::Model::HeadObjectRequest &request, + Aws::S3::Model::HeadObjectOutcome &outcome) +{ + S3OperationFunc + s3_op_func = &S3Client::HeadObject; + return do_s3_operation_(s3_op_func, request, outcome); +} + +int ObS3Client::put_object(const Model::PutObjectRequest &request, + Model::PutObjectOutcome &outcome) +{ + S3OperationFunc + s3_op_func = &S3Client::PutObject; + return do_s3_operation_(s3_op_func, request, outcome); +} + +int ObS3Client::get_object(const Model::GetObjectRequest &request, + Model::GetObjectOutcome &outcome) +{ + S3OperationFunc + s3_op_func = &S3Client::GetObject; + return do_s3_operation_(s3_op_func, request, outcome); +} + +int ObS3Client::delete_object(const Model::DeleteObjectRequest &request, + Model::DeleteObjectOutcome &outcome) +{ + S3OperationFunc + s3_op_func = &S3Client::DeleteObject; + return do_s3_operation_(s3_op_func, request, outcome); +} + +int ObS3Client::put_object_tagging(const Model::PutObjectTaggingRequest &request, + Model::PutObjectTaggingOutcome &outcome) +{ + S3OperationFunc + s3_op_func = &S3Client::PutObjectTagging; + return do_s3_operation_(s3_op_func, request, outcome); +} + +int ObS3Client::list_objects_v2(const Model::ListObjectsV2Request &request, + Model::ListObjectsV2Outcome &outcome) +{ + S3OperationFunc + s3_op_func = &S3Client::ListObjectsV2; + return do_s3_operation_(s3_op_func, request, outcome); +} + +int ObS3Client::list_objects(const Aws::S3::Model::ListObjectsRequest &request, + Model::ListObjectsOutcome &outcome) +{ + S3OperationFunc + s3_op_func = &S3Client::ListObjects; + return do_s3_operation_(s3_op_func, request, outcome); +} + +int ObS3Client::get_object_tagging(const Model::GetObjectTaggingRequest &request, + Model::GetObjectTaggingOutcome &outcome) +{ + S3OperationFunc + s3_op_func = &S3Client::GetObjectTagging; + return do_s3_operation_(s3_op_func, request, outcome); +} + +int ObS3Client::create_multipart_upload(const Model::CreateMultipartUploadRequest &request, + Model::CreateMultipartUploadOutcome &outcome) +{ + S3OperationFunc + s3_op_func = &S3Client::CreateMultipartUpload; + return do_s3_operation_(s3_op_func, request, outcome); +} + +int ObS3Client::list_parts(const Model::ListPartsRequest &request, + Model::ListPartsOutcome &outcome) +{ + S3OperationFunc + s3_op_func = &S3Client::ListParts; + return do_s3_operation_(s3_op_func, request, outcome); +} + +int ObS3Client::complete_multipart_upload(const Model::CompleteMultipartUploadRequest &request, + Model::CompleteMultipartUploadOutcome &outcome) +{ + S3OperationFunc + s3_op_func = &S3Client::CompleteMultipartUpload; + return do_s3_operation_(s3_op_func, request, outcome); +} + +int ObS3Client::abort_multipart_upload(const Model::AbortMultipartUploadRequest &request, + Model::AbortMultipartUploadOutcome &outcome) +{ + S3OperationFunc + s3_op_func = &S3Client::AbortMultipartUpload; + return do_s3_operation_(s3_op_func, request, outcome); +} + +int ObS3Client::upload_part(const Model::UploadPartRequest &request, + Model::UploadPartOutcome &outcome) +{ + S3OperationFunc + s3_op_func = &S3Client::UploadPart; + return do_s3_operation_(s3_op_func, request, outcome); +} + +int ObS3Client::list_multipart_uploads(const Model::ListMultipartUploadsRequest &request, + Model::ListMultipartUploadsOutcome &outcome) +{ + S3OperationFunc + s3_op_func = &S3Client::ListMultipartUploads; + return do_s3_operation_(s3_op_func, request, outcome); +} + +/*--------------------------------GLOBAL--------------------------------*/ +int init_s3_env() +{ + return ObS3Env::get_instance().init(); +} + +void fin_s3_env() +{ + ObS3Env::get_instance().stop(); + + // wait doing io finish before destroy s3 env. + const int64_t start_time = ObTimeUtility::current_time(); + const int64_t timeout = ObExternalIOCounter::FLYING_IO_WAIT_TIMEOUT; + int64_t flying_io_cnt = ObExternalIOCounter::get_flying_io_cnt(); + while(0 < flying_io_cnt) { + const int64_t end_time = ObTimeUtility::current_time(); + if (end_time - start_time > timeout) { + int ret = OB_TIMEOUT; + OB_LOG(WARN, "force fin_s3_env", K(ret), K(flying_io_cnt)); + break; + } + usleep(100 * 1000L); // 100ms + flying_io_cnt = ObExternalIOCounter::get_flying_io_cnt(); + } + + ObS3Env::get_instance().destroy(); +} + +// Currently, we only support CRC32 algorithm +void set_s3_checksum_algorithm(const ObStorageCRCAlgorithm crc_algo) +{ + default_s3_crc_algo = crc_algo; +} + +ObStorageCRCAlgorithm get_s3_checksum_algorithm() +{ + return default_s3_crc_algo; +} + +std::shared_ptr s3_clientBootstrap_create_fn() +{ + return nullptr; +}; + +std::shared_ptr s3_logger_create_fn() +{ + return std::make_shared(); +} + +ObS3Env::ObS3Env() + : lock_(ObLatchIds::OBJECT_DEVICE_LOCK), + is_inited_(false), s3_mem_manger_(), + aws_options_(), s3_client_map_() +{ + aws_options_.memoryManagementOptions.memoryManager = &s3_mem_manger_; + aws_options_.ioOptions.clientBootstrap_create_fn = s3_clientBootstrap_create_fn; + ObS3Logger s3_logger; + aws_options_.loggingOptions.logLevel = s3_logger.GetLogLevel(); + aws_options_.loggingOptions.logger_create_fn = s3_logger_create_fn; +} + +ObS3Env &ObS3Env::get_instance() +{ + static ObS3Env s3_env_instance; + return s3_env_instance; +} + +int ObS3Env::init() +{ + int ret = OB_SUCCESS; + SpinWLockGuard guard(lock_); + if (OB_UNLIKELY(is_inited_)) { + ret = OB_INIT_TWICE; + OB_LOG(WARN, "S3 env init twice", K(ret)); + } else if (OB_FAIL(s3_client_map_.create(MAX_S3_CLIENT_NUM, OB_STORAGE_S3_ALLOCATOR))) { + OB_LOG(WARN, "failed to create s3 client map", K(ret)); + } else { + Aws::InitAPI(aws_options_); + is_inited_ = true; + // TO make sure Aws::ShutdownAPI is called before OPENSSL_cleanup + // fin_s3_env is called when the singleton ObDeviceManager is destructed. + // Its destructor is called after the cleanup of the openssl environment. + // So, when executing fin_s3_env, openssl has already been cleaned up. + // However, fin_s3_env invokes Aws::ShutDownAPI, which requires calling functions from openssl. + // This can lead to deadlocks or segmentation faults. + // To avoid this, we register fin_s3_env in advance, + // ensuring that its invocation occurs before the cleanup of openssl + atexit(fin_s3_env); + } + return ret; +} + +void ObS3Env::destroy() +{ + SpinWLockGuard guard(lock_); + if (OB_LIKELY(is_inited_)) { + hash::ObHashMap::iterator iter = s3_client_map_.begin(); + while (iter != s3_client_map_.end()) { + if (OB_NOT_NULL(iter->second)) { + // force destroy s3 client + ObS3Client *s3_client = iter->second; + s3_client->~ObS3Client(); + ob_free(iter->second); + } + iter++; + } + s3_client_map_.destroy(); + Aws::ShutdownAPI(aws_options_); + is_inited_ = false; + } +} + +int ObS3Env::get_or_create_s3_client(const ObS3Account &account, ObS3Client *&client) +{ + int ret = OB_SUCCESS; + const int64_t key = account.hash(); + SpinWLockGuard guard(lock_); + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + OB_LOG(WARN, "ob s3 env not init", K(ret)); + } else if (OB_UNLIKELY(!account.is_valid())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "S3 account not valid", K(ret)); + } else if (OB_FAIL(s3_client_map_.get_refactored(key, client))) { + if (ret == OB_HASH_NOT_EXIST) { + ret = OB_SUCCESS; + void *client_buf = NULL; + if (s3_client_map_.size() > MAX_S3_CLIENT_MAP_THRESHOLD && OB_FAIL(clean_s3_client_map_())) { + OB_LOG(WARN, "failed to clean s3 client map", K(ret), K(s3_client_map_.size())); + } else if (OB_ISNULL(client_buf = ob_malloc(sizeof(ObS3Client), OB_STORAGE_S3_ALLOCATOR))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "failed to alloc buf for ob s3 client", K(ret)); + } else { + client = new(client_buf) ObS3Client(); + if (OB_FAIL(client->init(account))) { + client->~ObS3Client(); + ob_free(client_buf); + client = nullptr; + OB_LOG(WARN, "failed to init ObS3Client", K(ret), K(account)); + } else if (OB_FAIL(s3_client_map_.set_refactored(key, client))) { + client->~ObS3Client(); + ob_free(client_buf); + client = nullptr; + OB_LOG(WARN, "failed to insert into s3 client map", K(ret), K(account)); + } else { + OB_LOG(DEBUG, "succeed create new s3 client", K(account), K(s3_client_map_.size())); + } + } + } else { + OB_LOG(WARN, "failed to get s3 client from map", K(ret), K(account)); + } + } else if (OB_UNLIKELY(client->is_stopped())) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "an stopped client remained in s3 client map", K(ret), KP(client)); + } + + if (OB_SUCC(ret)) { + client->increase(); + } + return ret; +} + +int ObS3Env::clean_s3_client_map_() +{ + int ret = OB_SUCCESS; + hash::ObHashMap::iterator iter = s3_client_map_.begin(); + ObArray s3_clients_to_clean; + while (OB_SUCC(ret) && iter != s3_client_map_.end()) { + if (OB_NOT_NULL(iter->second)) { + ObS3Client *s3_client = iter->second; + if (s3_client->try_stop()) { + if (OB_FAIL(s3_clients_to_clean.push_back(iter->first))) { + OB_LOG(WARN, "failed to push back into s3_clients_to_clean", + K(ret), K(iter->first), KP(s3_client)); + } else { + s3_client->~ObS3Client(); + ob_free(iter->second); + iter->second = NULL; + } + } + } else if (OB_FAIL(s3_clients_to_clean.push_back(iter->first))) { + OB_LOG(WARN, "failed to push back into s3_clients_to_clean", + K(ret), K(iter->first), KP(iter->second)); + } + iter++; + } + + for (int64_t i = 0; OB_SUCC(ret) && i < s3_clients_to_clean.count(); i++) { + if (OB_FAIL(s3_client_map_.erase_refactored(s3_clients_to_clean[i]))) { + OB_LOG(WARN, "failed to clean s3 client map", K(ret)); + } + } + + return ret; +} + +void ObS3Env::stop() +{ + SpinRLockGuard guard(lock_); + hash::ObHashMap::iterator iter = s3_client_map_.begin(); + while (iter != s3_client_map_.end()) { + if (OB_NOT_NULL(iter->second)) { + ObS3Client *s3_client = iter->second; + s3_client->stop(); + } + iter++; + } +} + +static void convert_io_error(const Aws::S3::S3Error &s3_err, int &ob_errcode) +{ + switch (s3_err.GetErrorType()) { + case Aws::S3::S3Errors::NO_SUCH_KEY: { + ob_errcode = OB_BACKUP_FILE_NOT_EXIST; + break; + } + case Aws::S3::S3Errors::RESOURCE_NOT_FOUND: { + ob_errcode = OB_BACKUP_FILE_NOT_EXIST; + break; + } + case Aws::S3::S3Errors::SLOW_DOWN: { + ob_errcode = OB_IO_LIMIT; + break; + } + default: { + ob_errcode = OB_S3_ERROR; + } + } +} + +template +static void log_s3_status(OutcomeType &outcome, const int ob_errcode) +{ + const char *request_id = outcome.GetResult().GetRequestId().c_str(); + const int code = static_cast(outcome.GetError().GetResponseCode()); + const char *exception = outcome.GetError().GetExceptionName().c_str(); + const char *err_msg = outcome.GetError().GetMessage().c_str(); + OB_LOG_RET(WARN, ob_errcode, "S3 info", K(request_id), K(code), K(exception), K(err_msg)); +} + +template +static void handle_s3_outcome(OutcomeType &outcome, int &ob_errcode) +{ + const Aws::S3::S3Error &s3_err = outcome.GetError(); + convert_io_error(s3_err, ob_errcode); + log_s3_status(outcome, ob_errcode); +} + +/*--------------------------------ObS3Account--------------------------------*/ +ObS3Account::ObS3Account() +{ + reset(); +} + +ObS3Account::~ObS3Account() +{ + if (is_valid_) { + reset(); + } +} + +void ObS3Account::reset() +{ + is_valid_ = false; + delete_mode_ = ObIStorageUtil::DELETE; + MEMSET(region_, 0, sizeof(region_)); + MEMSET(endpoint_, 0, sizeof(endpoint_)); + MEMSET(access_id_, 0, sizeof(access_id_)); + MEMSET(secret_key_, 0, sizeof(secret_key_)); +} + +int64_t ObS3Account::hash() const +{ + int64_t hash_value = 0; + hash_value = murmurhash(region_, static_cast(strlen(region_)), hash_value); + hash_value = murmurhash(endpoint_, static_cast(strlen(endpoint_)), hash_value); + hash_value = murmurhash(access_id_, static_cast(strlen(access_id_)), hash_value); + hash_value = murmurhash(secret_key_, static_cast(strlen(secret_key_)), hash_value); + return hash_value; +} + +int ObS3Account::parse_from(const char *storage_info_str, const int64_t size) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(storage_info_str) || size >= OB_MAX_BACKUP_STORAGE_INFO_LENGTH) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "failed to init s3 account, invalid argument", K(ret), KP(storage_info_str), K(size)); + } else { + char tmp[OB_MAX_BACKUP_STORAGE_INFO_LENGTH]; + char *token = NULL; + char *saved_ptr = NULL; + + uint8_t bitmap = 0; + MEMCPY(tmp, storage_info_str, size); + tmp[size] = '\0'; + token = tmp; + for (char *str = token; OB_SUCC(ret); str = NULL) { + token = ::strtok_r(str, "&", &saved_ptr); + if (OB_ISNULL(token)) { + break; + } else if (0 == strncmp(REGION, token, strlen(REGION))) { + if (OB_FAIL(set_field(token + strlen(REGION), region_, sizeof(region_)))) { + OB_LOG(WARN, "failed to set s3 region", K(ret), KCSTRING(token)); + } else { + bitmap |= 1; + } + } else if (0 == strncmp(HOST, token, strlen(HOST))) { + if (OB_FAIL(set_field(token + strlen(HOST), endpoint_, sizeof(endpoint_)))) { + OB_LOG(WARN, "failed to set s3 endpoint", K(ret), KCSTRING(token)); + } else { + bitmap |= (1 << 1); + } + } else if (0 == strncmp(ACCESS_ID, token, strlen(ACCESS_ID))) { + if (OB_FAIL(set_field(token + strlen(ACCESS_ID), access_id_, sizeof(access_id_)))) { + OB_LOG(WARN, "failed to set s3 access id", K(ret), KCSTRING(token)); + } else { + bitmap |= (1 << 2); + } + } else if (0 == strncmp(ACCESS_KEY, token, strlen(ACCESS_KEY))) { + if (OB_FAIL(set_field(token + strlen(ACCESS_KEY), secret_key_, sizeof(secret_key_)))) { + OB_LOG(WARN, "failed to set s3 secret key", K(ret), KP(token)); + } else { + bitmap |= (1 << 3); + } + } else if (0 == strncmp(DELETE_MODE, token, strlen(DELETE_MODE))) { + if (0 == strcmp(token + strlen(DELETE_MODE), "delete")) { + delete_mode_ = ObIStorageUtil::DELETE; + } else if (0 == strcmp(token + strlen(DELETE_MODE), "tagging")) { + delete_mode_ = ObIStorageUtil::TAGGING; + } else { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "delete mode is invalid", K(ret), KCSTRING(token)); + } + } else { + OB_LOG(DEBUG, "unknown s3 info", K(*token)); + } + } + + if (OB_SUCC(ret) && bitmap != 0x0F) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "failed to init s3 account", K(ret), KCSTRING(region_), + KCSTRING(endpoint_), KCSTRING(access_id_), K(bitmap)); + } + } + + if (OB_SUCC(ret)) { + is_valid_ = true; + OB_LOG(DEBUG, "succeed to init s3 account", + KCSTRING(region_), KCSTRING(endpoint_), KCSTRING(access_id_)); + } else { + reset(); + } + return ret; +} + +int ObS3Account::set_field(const char *value, char *field, const uint32_t field_length) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(value) || OB_ISNULL(field)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invliad arguments", K(ret), KP(value), KP(field)); + } else { + const int64_t value_len = strlen(value); + if (value_len >= field_length) { + ret = OB_SIZE_OVERFLOW; + OB_LOG(WARN, "value is too long", K(ret), KP(value), K(value_len), K(field_length)); + } else { + MEMCPY(field, value, value_len); + field[value_len] = '\0'; + } + } + return ret; +} + +/*--------------------------------ObStorageS3Base--------------------------------*/ +ObStorageS3Base::ObStorageS3Base() + : allocator_(OB_STORAGE_S3_ALLOCATOR), + s3_client_(NULL), + bucket_(), + object_(), + is_inited_(false), + s3_account_() +{ +} + +ObStorageS3Base::~ObStorageS3Base() +{ + reset(); +} + +void ObStorageS3Base::reset() +{ + is_inited_ = false; + s3_account_.reset(); + allocator_.clear(); + if (OB_NOT_NULL(s3_client_)) { + s3_client_->release(); + s3_client_ = NULL; + } +} + +int ObStorageS3Base::open(const ObString &uri, ObObjectStorageInfo *storage_info) +{ + int ret = OB_SUCCESS; + char info_str[common::OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 }; + if (OB_UNLIKELY(is_inited_)) { + ret = OB_INIT_TWICE; + OB_LOG(WARN, "s3 base alreagy inited", K(ret)); + } else if (OB_UNLIKELY(uri.empty()) || OB_ISNULL(storage_info)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "failed to init s3 base, invalid arguments", K(ret), K(uri), KP(storage_info)); + } else if (OB_FAIL(build_bucket_and_object_name(uri))) { + OB_LOG(WARN, "failed to parse uri", K(ret), K(uri)); + } else if (OB_FAIL(storage_info->get_storage_info_str(info_str, sizeof(info_str)))) { + OB_LOG(WARN, "failed to get storage info str", K(ret), KPC(storage_info)); + } else if (s3_account_.parse_from(info_str, strlen(info_str))) { + OB_LOG(WARN, "failed to build s3 account", K(ret)); + } else if (OB_FAIL(ObS3Env::get_instance().get_or_create_s3_client(s3_account_, s3_client_))) { + OB_LOG(WARN, "faied to get s3 client", K(ret)); + } else { + is_inited_ = true; + } + + if (OB_FAIL(ret)) { + reset(); + } + return ret; +} + +int ObStorageS3Base::build_bucket_and_object_name(const ObString &uri) +{ + int ret = OB_SUCCESS; + ObString::obstr_size_t bucket_start = 0; + ObString::obstr_size_t bucket_end = 0; + ObString::obstr_size_t object_start = 0; + char *bucket_name_buff = NULL; + char *object_name_buff = NULL; + + if (!uri.prefix_match(OB_S3_PREFIX)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "uri is invalid ", KCSTRING(OB_S3_PREFIX), K(uri), K(ret)); + } else { + bucket_start = static_cast(strlen(OB_S3_PREFIX)); + for (int64_t i = bucket_start; OB_SUCC(ret) && i < uri.length() - 1; i++) { + if ('/' == *(uri.ptr() + i) && '/' == *(uri.ptr() + i + 1)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "uri has two // ", K(uri), K(ret), K(i)); + break; + } + } + } + + if (OB_SUCC(ret)) { + if (OB_ISNULL(bucket_name_buff = static_cast(allocator_.alloc(OB_MAX_URI_LENGTH)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "failed to alloc bucket name buff", K(ret), K(uri)); + } else if (OB_ISNULL(object_name_buff = static_cast(allocator_.alloc(OB_MAX_URI_LENGTH)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "failed to alloc object name buff", K(ret), K(uri)); + } else { + MEMSET(bucket_name_buff, 0, OB_MAX_URI_LENGTH); + MEMSET(object_name_buff, 0, OB_MAX_URI_LENGTH); + } + } + + // parse bucket name + if (OB_SUCC(ret)) { + if (bucket_start >= uri.length()) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "bucket name is NULL", K(uri), K(ret)); + } else { + for (bucket_end = bucket_start; OB_SUCC(ret) && bucket_end < uri.length(); ++bucket_end) { + if ('/' == *(uri.ptr() + bucket_end)) { + ObString::obstr_size_t bucket_length = bucket_end - bucket_start; + //must end with '\0' + if (OB_UNLIKELY(bucket_length <= 0)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "bucket name is empty", K(ret), K(uri), K(bucket_start), K(bucket_length)); + } else if (OB_FAIL(databuff_printf(bucket_name_buff, OB_MAX_URI_LENGTH, + "%.*s", bucket_length, uri.ptr() + bucket_start))) { + OB_LOG(WARN, "fail to deep copy bucket", K(uri), K(bucket_start), K(bucket_length), K(ret)); + } else { + bucket_.assign_ptr(bucket_name_buff, strlen(bucket_name_buff) + 1);// must include '\0' + } + break; + } + } + + if (OB_SUCC(ret) && bucket_end == uri.length()) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "bucket name is invalid", K(uri), K(ret)); + } + } + } + + // parse the object name + if (OB_SUCC(ret)) { + if (bucket_end + 1 >= uri.length()) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "object name is NULL", K(uri), K(ret)); + } else { + object_start = bucket_end + 1; + ObString::obstr_size_t object_length = uri.length() - object_start; + //must end with '\0' + if (OB_UNLIKELY(object_length <= 0)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "bucket name is empty", K(ret), K(uri), K(object_start), K(object_length)); + } else if (OB_FAIL(databuff_printf(object_name_buff, OB_MAX_URI_LENGTH, + "%.*s", object_length, uri.ptr() + object_start))) { + OB_LOG(WARN, "fail to deep copy object", K(uri), K(object_start), K(object_length), K(ret)); + } else { + object_.assign_ptr(object_name_buff, strlen(object_name_buff) + 1);//must include '\0' + } + } + } + + if (OB_SUCC(ret)) { + OB_LOG(DEBUG, "get bucket object name", K(uri), K_(bucket), K_(object)); + } + return ret; +} + +// can only be used to get the metadata of normal objects +int ObStorageS3Base::get_s3_file_meta_(S3ObjectMeta &meta) +{ + int ret = OB_SUCCESS; + meta.reset(); + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + OB_LOG(WARN, "s3 base not inited", K(ret)); + } else { + Aws::S3::Model::HeadObjectRequest request; + request.WithBucket(bucket_.ptr()).WithKey(object_.ptr()); + Aws::S3::Model::HeadObjectOutcome outcome; + if (OB_FAIL(s3_client_->head_object(request, outcome))) { + OB_LOG(WARN, "failed to head s3 object", K(ret)); + } else if (!outcome.IsSuccess()) { + convert_io_error(outcome.GetError(), ret); + if (OB_BACKUP_FILE_NOT_EXIST == ret) { + ret = OB_SUCCESS; + } else { + log_s3_status(outcome, ret); + OB_LOG(WARN, "failed to head s3 object", K(ret), K_(bucket), K_(object)); + } + } else { + meta.is_exist_ = true; + meta.length_ = outcome.GetResult().GetContentLength(); + } + } + return ret; +} + +int ObStorageS3Base::do_list_(const int64_t max_list_num, const char *delimiter, + const Aws::String &next_marker, Aws::S3::Model::ListObjectsOutcome &outcome) +{ + int ret = OB_SUCCESS; + char dir_path_buf[OB_MAX_URI_LENGTH] = {0}; + + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + OB_LOG(WARN, "s3 base not inited", K(ret)); + } else if (OB_UNLIKELY(max_list_num <= 0 || max_list_num > OB_STORAGE_LIST_MAX_NUM)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid arguments", K(ret), K_(bucket), K_(object), K(max_list_num)); + } else if (OB_UNLIKELY(!is_end_with_slash(object_.ptr()))) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "uri is not a valid dir path", K(ret), K_(object), K_(bucket)); + } else { + Aws::S3::Model::ListObjectsRequest request; + request.WithBucket(bucket_.ptr()).WithPrefix(object_.ptr()).WithMaxKeys(max_list_num); + if (NULL != delimiter && strlen(delimiter) > 0) { + request.SetDelimiter(delimiter); + } + if (!next_marker.empty()) { + request.SetMarker(next_marker); + } + + if (OB_FAIL(s3_client_->list_objects(request, outcome))) { + OB_LOG(WARN, "failed to list s3 objects", K(ret)); + } else if (!outcome.IsSuccess()) { + const Aws::S3::S3Error &s3_err = outcome.GetError(); + convert_io_error(s3_err, ret); + const char *err_msg = s3_err.GetMessage().c_str(); + OB_LOG(WARN, "failed to list s3 objects", K(ret), + K_(bucket), K_(object), K(err_msg), K(max_list_num), K(delimiter)); + } + } + return ret; +} + +/*--------------------------------ObStorageS3Writer--------------------------------*/ +ObStorageS3Writer::ObStorageS3Writer() + : ObStorageS3Base(), + is_opened_(false), + file_length_(-1) +{ +} + +ObStorageS3Writer::~ObStorageS3Writer() +{ + if (is_opened_) { + close(); + } +} + +int ObStorageS3Writer::open_(const ObString &uri, ObObjectStorageInfo *storage_info) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(is_opened_)) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "s3 writer already open, cannot open again", K(ret), K(uri)); + } else if (OB_FAIL(ObStorageS3Base::open(uri, storage_info))) { + OB_LOG(WARN, "failed to open in s3 base", K(ret), K(uri)); + } else { + is_opened_ = true; + file_length_ = 0; + } + return ret; +} + +static Aws::String calculate_crc32(const char *buf, const int64_t size) +{ + Aws::String crc; + if (NULL != buf && size > 0) { + Aws::String buf_str(buf, size); + Aws::Utils::ByteBuffer crc_buf = Aws::Utils::HashingUtils::CalculateCRC32(buf_str); + crc = Aws::Utils::HashingUtils::Base64Encode(crc_buf); + } + return crc; +} + +static int init_put_object_request(const char *bucket, const char *object, + const char *buf, const int64_t size, const bool check_crc, + Aws::S3::Model::PutObjectRequest &request) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(bucket) || OB_ISNULL(object) || OB_ISNULL(buf) || OB_UNLIKELY(size < 0)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid arguments", K(ret), KP(bucket), KP(object), KP(buf), K(size)); + } else { + request.WithBucket(bucket).WithKey(object); + if (check_crc) { + request.SetChecksumAlgorithm(Aws::S3::Model::ChecksumAlgorithm::CRC32); + } + std::shared_ptr data_stream = + Aws::MakeShared(S3_SDK); + data_stream->write(buf, size); + data_stream->flush(); + request.SetBody(data_stream); + request.SetAdditionalCustomHeaderValue("Connection", "keep-alive"); + request.SetContentLength(static_cast(request.GetBody()->tellp())); + } + return ret; +} + +int ObStorageS3Writer::write_(const char *buf, const int64_t size) +{ + int ret = OB_SUCCESS; + Aws::S3::Model::PutObjectRequest request; + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_NOT_INIT; + OB_LOG(WARN, "s3 writer not opened", K(ret)); + } else if (OB_ISNULL(buf) || OB_UNLIKELY(size < 0)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "buf is NULL or size is invalid", K(ret), KP(buf), K(size)); + } else if (OB_FAIL(write_obj_(object_.ptr(), buf, size))) { + OB_LOG(WARN, "fail to put s3 object", K(ret), K_(bucket), K_(object)); + } else { + file_length_ = size; + } + return ret; +} + +int ObStorageS3Writer::write_obj_(const char *obj_name, const char *buf, const int64_t size) +{ + int ret = OB_SUCCESS; + const bool check_crc = (default_s3_crc_algo == ObStorageCRCAlgorithm::OB_CRC32_ALGO); + Aws::S3::Model::PutObjectRequest request; + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_NOT_INIT; + OB_LOG(WARN, "s3 writer not opened", K(ret)); + } else if (OB_ISNULL(obj_name) || OB_ISNULL(buf) || OB_UNLIKELY(size < 0)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid arguments", K(ret), KP(obj_name), KP(buf), K(size)); + } else if (OB_FAIL(init_put_object_request(bucket_.ptr(), obj_name, buf, + size, check_crc, request))) { + OB_LOG(WARN, "failed to init put obejct request", K(ret), + K_(bucket), K(obj_name), KP(buf), K(size), K(check_crc)); + } else { + Aws::S3::Model::PutObjectOutcome outcome; + if (OB_FAIL(s3_client_->put_object(request, outcome))) { + OB_LOG(WARN, "failed to put s3 object", K(ret)); + } else if (!outcome.IsSuccess()) { + handle_s3_outcome(outcome, ret); + OB_LOG(WARN, "failed to write object into s3", + K(ret), K_(bucket), K(obj_name), KP(buf), K(size)); + } + } + return ret; +} + +int ObStorageS3Writer::pwrite_(const char *buf, const int64_t size, const int64_t offset) +{ + int ret = OB_NOT_SUPPORTED; + UNUSED(buf); + UNUSED(size); + UNUSED(offset); + return ret; +} + +int ObStorageS3Writer::close_() +{ + int ret = OB_SUCCESS; + is_opened_ = false; + file_length_ = -1; + reset(); + return ret; +} + +/*--------------------------------ObStorageS3Reader--------------------------------*/ +ObStorageS3Reader::ObStorageS3Reader() + : ObStorageS3Base(), + is_opened_(false), + has_meta_(false), + file_length_(-1) +{ +} + +ObStorageS3Reader::~ObStorageS3Reader() +{ +} + +void ObStorageS3Reader::reset() +{ + ObStorageS3Base::reset(); + is_opened_ = false; + has_meta_ = false; + file_length_ = -1; +} + +int ObStorageS3Reader::open_(const ObString &uri, + ObObjectStorageInfo *storage_info, const bool head_meta) +{ + int ret = OB_SUCCESS; + S3ObjectMeta meta; + ObExternalIOCounterGuard io_guard; + if (OB_UNLIKELY(is_opened_)) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "s3 reader already open, cannot open again", K(ret), K(uri)); + } else if (OB_FAIL(ObStorageS3Base::open(uri, storage_info))) { + OB_LOG(WARN, "failed to open in s3 base", K(ret), K(uri)); + } else { + if (head_meta) { + if (OB_FAIL(get_s3_file_meta(meta))) { + OB_LOG(WARN, "failed to get s3 object meta", K(ret), K(uri)); + } else if (!meta.is_exist_) { + ret = OB_BACKUP_FILE_NOT_EXIST; + OB_LOG(WARN, "backup file is not exist", K(ret), K(uri), K_(bucket), K_(object)); + } else { + file_length_ = meta.length_; + has_meta_ = true; + } + } + } + + if (OB_FAIL(ret)) { + reset(); + } else { + is_opened_ = true; + } + return ret; +} + +int ObStorageS3Reader::pread_(char *buf, + const int64_t buf_size, const int64_t offset, int64_t &read_size) +{ + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "s3 reader not opened", K(ret)); + } else if (OB_ISNULL(buf) || OB_UNLIKELY(buf_size <= 0 || offset < 0)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid arguments", K(ret), KP(buf), K(buf_size), K(offset)); + } else { + if (has_meta_ && offset == file_length_) { + read_size = 0; + } else if (has_meta_ && offset > file_length_) { + ret = OB_FILE_LENGTH_INVALID; + OB_LOG(WARN, "offset is larger than file length", + K(ret), K(offset), K_(file_length), K_(bucket), K_(object)); + } else { + Aws::S3::Model::GetObjectRequest request; + Aws::S3::Model::GetObjectOutcome outcome; + char range_read[64] = { 0 }; + request.WithBucket(bucket_.ptr()).WithKey(object_.ptr()); + request.SetAdditionalCustomHeaderValue("Connection", "keep-alive"); + + if (OB_FAIL(databuff_printf(range_read, sizeof(range_read), + "bytes=%ld-%ld", offset, offset + buf_size - 1))) { + OB_LOG(WARN, "fail to set range to read", + K(ret), K_(bucket), K_(object), K(offset), K(buf_size)); + } else if (FALSE_IT(request.SetRange(range_read))) { + } else if (OB_FAIL(s3_client_->get_object(request, outcome))) { + OB_LOG(WARN, "failed to get s3 object", K(ret), K(range_read)); + } else if (!outcome.IsSuccess()) { + handle_s3_outcome(outcome, ret); + OB_LOG(WARN, "failed to read object from s3", + K(ret), K_(bucket), K_(object), K(range_read)); + } else { + read_size = outcome.GetResult().GetContentLength(); + outcome.GetResult().GetBody().read(buf, read_size); + } + } + } + return ret; +} + +int ObStorageS3Reader::close_() +{ + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + reset(); + return ret; +} + +/*--------------------------------ObStorageS3Util--------------------------------*/ +ObStorageS3Util::ObStorageS3Util() : is_opened_(false), storage_info_(NULL) +{ +} + +ObStorageS3Util::~ObStorageS3Util() +{ +} + +int ObStorageS3Util::open(ObObjectStorageInfo *storage_info) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(is_opened_)) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "s3 util already open, cannot open again", K(ret)); + } else if (OB_ISNULL(storage_info)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "storage info is NULL", K(ret), KP(storage_info)); + } else { + is_opened_ = true; + storage_info_ = storage_info; + } + return ret; +} + +void ObStorageS3Util::close() +{ + is_opened_ = false; + storage_info_ = NULL; +} + +int ObStorageS3Util::is_exist_(const ObString &uri, bool &exist) +{ + int ret = OB_SUCCESS; + exist = false; + ObStorageObjectMetaBase obj_meta; + if (OB_FAIL(head_object_meta(uri, obj_meta))) { + OB_LOG(WARN, "fail to head object meta", K(ret), K(uri)); + } else { + exist = obj_meta.is_exist_; + } + return ret; +} + +int ObStorageS3Util::get_file_length_(const ObString &uri, int64_t &file_length) +{ + int ret = OB_SUCCESS; + file_length = 0; + ObStorageObjectMetaBase obj_meta; + if (OB_FAIL(head_object_meta(uri, obj_meta))) { + OB_LOG(WARN, "fail to head object meta", K(ret), K(uri)); + } else if (!obj_meta.is_exist_) { + ret = OB_BACKUP_FILE_NOT_EXIST; + OB_LOG(WARN, "backup file is not exist", K(ret), K(uri)); + } else { + file_length = obj_meta.length_; + } + return ret; +} + +int ObStorageS3Util::head_object_meta(const ObString &uri, ObStorageObjectMetaBase &obj_meta) +{ + int ret = OB_SUCCESS; + S3ObjectMeta meta; + ObStorageS3Base s3_base; + obj_meta.reset(); + ObExternalIOCounterGuard io_guard; + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "s3 util not opened", K(ret)); + } else if (OB_UNLIKELY(uri.empty())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "uri is empty", K(ret), K(uri)); + } else if (OB_FAIL(s3_base.open(uri, storage_info_))) { + OB_LOG(WARN, "failed to open s3 base", K(ret), K(uri)); + } else if (OB_FAIL(s3_base.get_s3_file_meta(meta))) { + OB_LOG(WARN, "failed to get s3 file meta", K(ret), K(uri)); + } else { + obj_meta.is_exist_ = meta.is_exist_; + if (obj_meta.is_exist_) { + obj_meta.length_ = meta.length_; + } + } + return ret; +} + +int ObStorageS3Util::delete_object_(ObStorageS3Base &s3_base) +{ + int ret = OB_SUCCESS; + Aws::S3::Model::DeleteObjectRequest request; + request.WithBucket(s3_base.bucket_.ptr()).WithKey(s3_base.object_.ptr()); + Aws::S3::Model::DeleteObjectOutcome outcome; + if (OB_FAIL(s3_base.s3_client_->delete_object(request, outcome))) { + OB_LOG(WARN, "failed to delete s3 object", K(ret)); + } else if (!outcome.IsSuccess()) { + handle_s3_outcome(outcome, ret); + OB_LOG(WARN, "failed to delete s3 object", + K(ret), K(s3_base.bucket_), K(s3_base.object_)); + } else { + OB_LOG(DEBUG, "delete s3 object succeed", K(s3_base.bucket_), K(s3_base.object_)); + } + return ret; +} + +int ObStorageS3Util::tagging_object_(ObStorageS3Base &s3_base) +{ + int ret = OB_SUCCESS; + Aws::S3::Model::PutObjectTaggingRequest request; + request.WithBucket(s3_base.bucket_.ptr()).WithKey(s3_base.object_.ptr()); + Aws::S3::Model::Tag tag; + tag.WithKey("delete_mode").WithValue("tagging"); + Aws::S3::Model::Tagging tagging; + tagging.AddTagSet(tag); + request.SetTagging(tagging); + Aws::S3::Model::PutObjectTaggingOutcome outcome; + if (OB_FAIL(s3_base.s3_client_->put_object_tagging(request, outcome))) { + OB_LOG(WARN, "failed to put s3 object tagging", K(ret)); + } else if (!outcome.IsSuccess()) { + handle_s3_outcome(outcome, ret); + OB_LOG(WARN, "failed to tagging s3 object", + K(ret), K(s3_base.bucket_), K(s3_base.object_)); + } else { + OB_LOG(DEBUG, "tagging s3 object succeed", K(s3_base.bucket_), K(s3_base.object_)); + } + return ret; +} + +int ObStorageS3Util::del_file_(const ObString &uri) +{ + int ret = OB_SUCCESS; + ObStorageS3Base s3_base; + ObExternalIOCounterGuard io_guard; + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "s3 util not opened", K(ret)); + } else if (OB_UNLIKELY(uri.empty())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid arguments", K(ret), K(uri)); + } else if (OB_FAIL(s3_base.open(uri, storage_info_))) { + OB_LOG(WARN, "failed to open s3 base", K(ret), K(uri)); + } else { + const int64_t delete_mode = s3_base.s3_account_.delete_mode_; + if (ObIStorageUtil::DELETE == delete_mode) { + if (OB_FAIL(delete_object_(s3_base))) { + OB_LOG(WARN, "failed to delete s3 object", K(ret), K(uri)); + } + } else if (ObIStorageUtil::TAGGING == delete_mode) { + if (OB_FAIL(tagging_object_(s3_base))) { + OB_LOG(WARN, "failed to tag s3 object", K(ret), K(uri)); + } + } else { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "s3 delete mode invalid", K(ret), K(uri), K(delete_mode)); + } + } + return ret; +} + +int ObStorageS3Util::write_single_file_(const ObString &uri, const char *buf, const int64_t size) +{ + int ret = OB_SUCCESS; + ObStorageS3Writer s3_writer; + ObExternalIOCounterGuard io_guard; + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "s3 util not opened", K(ret)); + } else if (OB_UNLIKELY(uri.empty() || size < 0) || OB_ISNULL(buf)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid arguments", K(ret), K(uri), KP(buf), K(size)); + } else if (OB_FAIL(s3_writer.open(uri, storage_info_))) { + OB_LOG(WARN, "failed to open s3 writer", K(ret), K(uri)); + } else if (OB_FAIL(s3_writer.write(buf, size))) { + OB_LOG(WARN, "failed to write into s3", K(ret), K(uri), KP(buf), K(size)); + } else if (OB_FAIL(s3_writer.close())) { + OB_LOG(WARN, "failed to close s3 writer", K(ret), K(uri), KP(buf), K(size)); + } + return ret; +} + +int ObStorageS3Util::mkdir_(const ObString &uri) +{ + int ret = OB_SUCCESS; + OB_LOG(DEBUG, "no need to create dir in s3", K(uri)); + UNUSED(uri); + return ret; +} + + +int ObStorageS3Util::list_files_(const ObString &uri, ObBaseDirEntryOperator &op) +{ + int ret = OB_SUCCESS; + ObStorageS3Base s3_base; + ObExternalIOCounterGuard io_guard; + const char *full_dir_path = NULL; + + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "s3 util not opened", K(ret)); + } else if (OB_UNLIKELY(uri.empty())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid arguments", K(ret), K(uri)); + } else if (OB_FAIL(s3_base.open(uri, storage_info_))) { + OB_LOG(WARN, "fail to open s3 base", K(ret), K(uri)); + } else if (FALSE_IT(full_dir_path = s3_base.object_.ptr())) { + } else if (OB_UNLIKELY(!is_end_with_slash(full_dir_path))) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "uri is not terminated with '/'", K(ret), K(uri), K(full_dir_path)); + } else { + const int64_t full_dir_path_len = strlen(full_dir_path); // actual dir path len + Aws::S3::Model::ListObjectsOutcome outcome; + Aws::String next_marker; + do { + if (OB_FAIL(s3_base.do_list_(OB_STORAGE_LIST_MAX_NUM, NULL/*delimiter*/, + next_marker, outcome))) { + OB_LOG(WARN, "fail to list s3 objects", K(ret), K(uri)); + } else { + const char *request_id = outcome.GetResult().GetRequestId().c_str(); + const Aws::Vector &contents = outcome.GetResult().GetContents(); + for (int64_t i = 0; OB_SUCC(ret) && i < contents.size(); i++) { + const Aws::S3::Model::Object &obj = contents[i]; + const char *obj_path = obj.GetKey().c_str(); + const int64_t obj_path_len = obj.GetKey().size(); + + // For example, we can use oss console to create a 'read dir', like aaa/bbb/ccc/. + // When list 'aaa/bbb/ccc/' this dir, we will get it self, that means we will get + // a object whose name length is same with its parent dir length. + if (OB_UNLIKELY(false == ObString(obj_path).prefix_match(full_dir_path))) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "returned object prefix not match", + K(ret), K(request_id), K(obj_path), K(full_dir_path), K(uri)); + } else if (OB_UNLIKELY(obj_path_len == full_dir_path_len)) { + // skip + OB_LOG(INFO, "exist object path length is same with dir path length", + K(request_id), K(obj_path), K(full_dir_path), K(full_dir_path_len)); + } else if (OB_FAIL(handle_listed_object(op, obj_path + full_dir_path_len, + obj_path_len - full_dir_path_len, + obj.GetSize()))) { + OB_LOG(WARN, "fail to handle listed s3 object", K(ret), K(request_id), + K(obj_path), K(obj_path_len), K(full_dir_path), K(full_dir_path_len), K(uri)); + } + } // end for + if (OB_SUCC(ret) && outcome.GetResult().GetIsTruncated()) { + // We cannot set next_marker to outcome.GetResult().GetNextMarker() directly + // because GetNextMarker() might return empty data. + // Below is the description of next marker from the S3 official documentation: + // This element is returned only if you have the delimiter request parameter specified. + // If the response does not include the NextMarker element and it is truncated, + // you can use the value of the last Key element in the response + // as the marker parameter in the subsequent request to get the next set of object keys. + if (contents.size() > 0) { + next_marker = contents[contents.size() - 1].GetKey(); + } else { + // if result is truncated, contents should not be empty + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "listed s3 objects are empty", K(ret), K(request_id), K(contents.size())); + } + } + } + } while (OB_SUCC(ret) && outcome.GetResult().GetIsTruncated()); + } + return ret; +} + +int ObStorageS3Util::list_files2_( + const ObString &uri, + ObStorageListCtxBase &ctx_base) +{ + int ret = OB_SUCCESS; + ObStorageS3Base s3_base; + ObExternalIOCounterGuard io_guard; + const char *full_dir_path = NULL; + ObStorageListObjectsCtx &list_ctx = static_cast(ctx_base); + + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "s3 util not opened", K(ret)); + } else if (OB_UNLIKELY(uri.empty() || !list_ctx.is_valid())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid arguments", K(ret), K(uri), K(list_ctx)); + } else if (OB_FAIL(s3_base.open(uri, storage_info_))) { + OB_LOG(WARN, "fail to open s3 base", K(ret), K(uri)); + } else if (FALSE_IT(full_dir_path = s3_base.object_.ptr())) { + } else if (OB_UNLIKELY(!is_end_with_slash(full_dir_path))) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "uri is not terminated with '/'", K(ret), K(uri), K(full_dir_path)); + } else { + const int64_t full_dir_path_len = strlen(full_dir_path); // actual dir path len + Aws::S3::Model::ListObjectsOutcome outcome; + Aws::String next_marker; + if (list_ctx.next_token_ != NULL && list_ctx.next_token_[0] != '\0') { + next_marker.assign(list_ctx.next_token_); + } + const int64_t max_list_num = MIN(OB_STORAGE_LIST_MAX_NUM, list_ctx.max_list_num_); + + if (OB_FAIL(s3_base.do_list_(max_list_num, NULL/*delimiter*/, + next_marker, outcome))) { + OB_LOG(WARN, "fail to list s3 objects", K(ret), K(uri), K(max_list_num)); + } else { + const char *request_id = outcome.GetResult().GetRequestId().c_str(); + const Aws::Vector &contents = outcome.GetResult().GetContents(); + if (outcome.GetResult().GetIsTruncated()) { + if (contents.size() > 0) { + next_marker = contents[contents.size() - 1].GetKey(); + } else { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "listed s3 objects are empty", K(ret), K(request_id), K(contents.size())); + } + } else { + next_marker = ""; + } + + if (OB_FAIL(ret)) { + } else if (contents.size() > list_ctx.max_list_num_) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "can't hold all contents", K(ret), K(request_id), K(list_ctx), K(contents.size())); + } else if (OB_FAIL(list_ctx.set_next_token(outcome.GetResult().GetIsTruncated(), + next_marker.c_str(), + next_marker.length()))) { + OB_LOG(WARN, "fail to set next token when listing s3 objects", K(ret), K(request_id)); + } + + for (int64_t i = 0; OB_SUCC(ret) && (i < contents.size()); i++) { + const Aws::S3::Model::Object &obj = contents[i]; + const int64_t obj_size = obj.GetSize(); + const char *cur_obj_path = obj.GetKey().c_str(); // object full path + const int64_t cur_obj_path_len = obj.GetKey().size(); + OB_LOG(DEBUG, "s3 list files content", K(cur_obj_path), K(cur_obj_path_len)); + + if (OB_UNLIKELY(false == ObString(cur_obj_path).prefix_match(full_dir_path))) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "returned object prefix not match", + K(ret), K(request_id), K(cur_obj_path), K(full_dir_path), K(uri)); + } else if (OB_UNLIKELY(cur_obj_path_len == full_dir_path_len)) { + // skip + OB_LOG(INFO, "exist object path length is same with dir path length", + K(request_id), K(cur_obj_path), K(full_dir_path), K(full_dir_path_len)); + } else if (OB_FAIL(list_ctx.handle_object(cur_obj_path, cur_obj_path_len, obj_size))) { + OB_LOG(WARN, "fail to add listed s3 obejct meta into ctx", + K(ret), K(request_id), K(cur_obj_path), K(cur_obj_path_len), K(obj_size)); + } + } // end for + } + } + return ret; +} + +int ObStorageS3Util::del_dir_(const ObString &uri) +{ + int ret = OB_SUCCESS; + OB_LOG(DEBUG, "no need to del dir in s3", K(uri)); + UNUSED(uri); + return ret; +} + +int ObStorageS3Util::list_directories_(const ObString &uri, ObBaseDirEntryOperator &op) +{ + int ret = OB_SUCCESS; + ObStorageS3Base s3_base; + const char *delimiter = "/"; + const char *full_dir_path = NULL; + ObExternalIOCounterGuard io_guard; + + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "s3 util not opened", K(ret)); + } else if (OB_UNLIKELY(uri.empty())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid arguments", K(ret), K(uri)); + } else if (OB_FAIL(s3_base.open(uri, storage_info_))) { + OB_LOG(WARN, "fail to open s3 base", K(ret), K(uri)); + } else if (FALSE_IT(full_dir_path = s3_base.object_.ptr())) { + } else if (OB_UNLIKELY(!is_end_with_slash(full_dir_path))) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "uri is not terminated with '/'", K(ret), K(uri), K(full_dir_path)); + } else { + const int64_t full_dir_path_len = strlen(full_dir_path); // actual dir path len + Aws::S3::Model::ListObjectsOutcome outcome; + Aws::String next_marker; + do { + if (OB_FAIL(s3_base.do_list_(OB_STORAGE_LIST_MAX_NUM, delimiter, + next_marker, outcome))) { + OB_LOG(WARN, "fail to list s3 objects", K(ret), K(uri), K(delimiter)); + } else { + const char *request_id = outcome.GetResult().GetRequestId().c_str(); + const Aws::Vector &common_prefixes = outcome.GetResult().GetCommonPrefixes(); + for (int64_t i = 0; OB_SUCC(ret) && i < common_prefixes.size(); i++) { + const Aws::S3::Model::CommonPrefix &tmp_common_prefix = common_prefixes[i]; + // For example, + // dir1 + // --file1 + // --dir11 + // --file11 + // if we list directories in 'dir1', then full_dir_path == 'dir1/' + // and listed_dir_full_path == 'dir1/dir11/', which represents the full directory path of 'dir11' + const char *listed_dir_full_path = tmp_common_prefix.GetPrefix().c_str(); + const int64_t listed_dir_full_path_len = tmp_common_prefix.GetPrefix().size(); + OB_LOG(DEBUG, "s3 list directories", K(i), K(listed_dir_full_path)); + + if (OB_UNLIKELY(false == ObString(listed_dir_full_path).prefix_match(full_dir_path))) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "returned object prefix not match", + K(ret), K(request_id), K(listed_dir_full_path), K(full_dir_path), K(uri)); + } else if (OB_UNLIKELY(!is_end_with_slash(listed_dir_full_path))) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "the data has no directory", + K(ret), K(request_id), K(full_dir_path), K(listed_dir_full_path), K(uri)); + } else if (OB_FAIL(handle_listed_directory(op, + listed_dir_full_path + full_dir_path_len, + listed_dir_full_path_len - 1 - full_dir_path_len))) { + OB_LOG(WARN, "fail to handle s3 directory name", K(ret), + K(request_id), K(listed_dir_full_path), K(full_dir_path), K(full_dir_path_len)); + } + } // end for + if (OB_SUCC(ret) && outcome.GetResult().GetIsTruncated()) { + next_marker = outcome.GetResult().GetNextMarker(); + if (next_marker.empty()) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "when listing s3 directories, next marker should not be empty if result is truncated", + K(ret), K(request_id), K(outcome.GetResult().GetIsTruncated()), K(next_marker.c_str())); + } + } + } + } while (OB_SUCC(ret) && outcome.GetResult().GetIsTruncated()); + } + return ret; +} + +int ObStorageS3Util::is_tagging_(const ObString &uri, bool &is_tagging) +{ + int ret = OB_SUCCESS; + ObStorageS3Base s3_base; + ObExternalIOCounterGuard io_guard; + is_tagging = false; + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "s3 util not opened", K(ret)); + } else if (OB_UNLIKELY(uri.empty())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid arguments", K(ret), K(uri)); + } else if (OB_FAIL(s3_base.open(uri, storage_info_))) { + OB_LOG(WARN, "failed to open s3 base", K(ret), K(uri)); + } else { + Aws::S3::Model::GetObjectTaggingRequest request; + request.WithBucket(s3_base.bucket_.ptr()).WithKey(s3_base.object_.ptr()); + Aws::S3::Model::GetObjectTaggingOutcome outcome; + if (OB_FAIL(s3_base.s3_client_->get_object_tagging(request, outcome))) { + OB_LOG(WARN, "failed to get s3 object tagging", K(ret)); + } else if (!outcome.IsSuccess()) { + handle_s3_outcome(outcome, ret); + OB_LOG(WARN, "failed to get s3 object tagging", + K(ret), K(uri), K(s3_base.bucket_), K(s3_base.object_)); + } else { + for (const Aws::S3::Model::Tag &tag : outcome.GetResult().GetTagSet()) { + if (tag.GetKey() == "delete_mode" && tag.GetValue() == "tagging") { + is_tagging = true; + break; + } + } + } + } + return ret; +} + +int ObStorageS3Util::del_unmerged_parts_(const ObString &uri) +{ + int ret = OB_SUCCESS; + ObStorageS3Base s3_base; + ObExternalIOCounterGuard io_guard; + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "s3 util not opened", K(ret)); + } else if (OB_UNLIKELY(uri.empty())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid arguments", K(ret), K(uri)); + } else if (OB_FAIL(s3_base.open(uri, storage_info_))) { + OB_LOG(WARN, "failed to open s3 base", K(ret), K(uri), KPC_(storage_info)); + } else { + Aws::S3::Model::ListMultipartUploadsRequest request; + Aws::S3::Model::ListMultipartUploadsOutcome outcome; + request.WithBucket(s3_base.bucket_.ptr()).WithPrefix(s3_base.object_.ptr()); + request.SetMaxUploads(OB_STORAGE_LIST_MAX_NUM); + Aws::S3::Model::AbortMultipartUploadRequest abort_request; + Aws::S3::Model::AbortMultipartUploadOutcome abort_outcome; + abort_request.WithBucket(s3_base.bucket_.ptr()); + do { + if (OB_FAIL(s3_base.s3_client_->list_multipart_uploads(request, outcome))) { + OB_LOG(WARN, "failed to list s3 multipart uploads", K(ret)); + } else if (!outcome.IsSuccess()) { + handle_s3_outcome(outcome, ret); + OB_LOG(WARN, "failed to list s3 multipart uploads", + K(ret), K(uri), K(s3_base.bucket_), K(s3_base.object_)); + } else { + const char *list_request_id = outcome.GetResult().GetRequestId().c_str(); + const Aws::Vector &uploads = outcome.GetResult().GetUploads(); + for (int64_t i = 0; OB_SUCC(ret) && i < uploads.size(); i++) { + const Aws::String &obj = uploads[i].GetKey(); + const Aws::String &upload_id = uploads[i].GetUploadId(); + abort_request.WithKey(obj).WithUploadId(upload_id); + OB_LOG(DEBUG, "list s3 multipat upload", + K(ret), K(s3_base.bucket_), K(obj.c_str()), K(upload_id.c_str())); + + if (OB_FAIL(s3_base.s3_client_->abort_multipart_upload(abort_request, abort_outcome))) { + OB_LOG(WARN, "failed to abort s3 multipart upload", K(ret), + K(list_request_id), K(s3_base.bucket_), K(obj.c_str()), K(upload_id.c_str())); + } else if (!abort_outcome.IsSuccess()) { + handle_s3_outcome(abort_outcome, ret); + OB_LOG(WARN, "failed to abort s3 multipart upload", K(ret), + K(list_request_id), K(s3_base.bucket_), K(obj.c_str()), K(upload_id.c_str())); + } else { + OB_LOG(INFO, "succeed to abort s3 multipart upload", + K(s3_base.bucket_), K(obj.c_str()), K(upload_id.c_str())); + } + } + request.SetKeyMarker(outcome.GetResult().GetNextKeyMarker()); + request.SetUploadIdMarker(outcome.GetResult().GetNextUploadIdMarker()); + } + } while (OB_SUCC(ret) && outcome.GetResult().GetIsTruncated()); + } + return ret; +} + +/*--------------------------------ObStorageS3AppendWriter--------------------------------*/ +ObStorageS3AppendWriter::ObStorageS3AppendWriter() + : ObStorageS3Writer(), + storage_info_(NULL) +{ +} + +ObStorageS3AppendWriter::~ObStorageS3AppendWriter() +{ +} + +int ObStorageS3AppendWriter::open_(const ObString &uri, ObObjectStorageInfo *storage_info) +{ + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + if (OB_UNLIKELY(is_opened_)) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "s3 append writer already open, cannot open again", K(ret), K(uri)); + } else if (OB_FAIL(ObStorageS3Writer::open(uri, storage_info))) { + OB_LOG(WARN, "failed to open in s3 base", K(ret), K(uri)); + } else { + is_opened_ = true; + storage_info_ = storage_info; + } + return ret; +} + +int ObStorageS3AppendWriter::write_(const char *buf, const int64_t size) +{ + return pwrite(buf, size, 0); +} + +int ObStorageS3AppendWriter::pwrite_(const char *buf, const int64_t size, const int64_t offset) +{ + int ret = OB_SUCCESS; + char fragment_name[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 }; + Aws::S3::Model::PutObjectRequest request; + ObExternalIOCounterGuard io_guard; + if(OB_UNLIKELY(!is_opened_)) { + ret = OB_NOT_INIT; + OB_LOG(WARN, "s3 append writer cannot write before it is not opened", K(ret)); + } else if(OB_ISNULL(buf) || OB_UNLIKELY(size <= 0 || offset < 0)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid arguments", K(ret), KP(buf), K(size), K(offset)); + } else { + // write the format file when writing the first fragment because the appender may open multiple times + if (offset == 0) { + if (OB_FAIL(construct_fragment_full_name(object_, OB_S3_APPENDABLE_FORMAT_META, + fragment_name, sizeof(fragment_name)))) { + OB_LOG(WARN, "failed to construct s3 mock append object foramt name", + K(ret), K_(bucket), K_(object)); + } else if (OB_FAIL(write_obj_(fragment_name, OB_S3_APPENDABLE_FORMAT_CONTENT_V1, + strlen(OB_S3_APPENDABLE_FORMAT_CONTENT_V1)))) { + OB_LOG(WARN, "fail to write s3 mock append object format file", K(ret), K(fragment_name)); + } + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(construct_fragment_full_name(object_, offset, offset + size, + fragment_name, sizeof(fragment_name)))) { + // fragment_name: /xxx/xxx/appendable_obj_name/start-end, + // the data range covered by this file is from start to end == [start, end), include start not include end + // start == offset, end == offset + size + // fragment length == size + OB_LOG(WARN, "failed to set fragment name for s3 append writer", + K(ret), K_(bucket), K_(object), K(buf), K(size), K(offset)); + } else if (OB_FAIL(write_obj_(fragment_name, buf, size))) { + OB_LOG(WARN, "fail to append a fragment into s3", + K(ret), K_(bucket), K_(object), K(fragment_name), K(size)); + } + } + + if (OB_SUCC(ret)) { + OB_LOG(DEBUG, "succeed to append a fragment into s3", + K_(bucket), K_(object), K(fragment_name), K(size), K(offset)); + } + return ret; +} + +int ObStorageS3AppendWriter::close_() +{ + int ret = OB_SUCCESS; + + is_opened_ = false; + storage_info_ = NULL; + reset(); + return ret; +} + +int64_t ObStorageS3AppendWriter::get_length() const +{ + int ret = OB_NOT_SUPPORTED; + OB_LOG(WARN, "s3 appender do not support get length now", K(ret), K_(bucket), K_(object)); + return -1; +} + +/*--------------------------------ObStorageS3MultiPartWriter--------------------------------*/ +ObStorageS3MultiPartWriter::ObStorageS3MultiPartWriter() + : ObStorageS3Base(), + is_opened_(false), + base_buf_(NULL), base_buf_pos_(-1), + upload_id_(NULL), + partnum_(0), + file_length_(-1), + sum_hash_(NULL) +{} + +ObStorageS3MultiPartWriter::~ObStorageS3MultiPartWriter() +{} + +void ObStorageS3MultiPartWriter::reset() +{ + is_opened_ = false; + base_buf_ = NULL; + base_buf_pos_ = -1; + upload_id_ = NULL; + partnum_ = 0; + file_length_ = -1; + sum_hash_ = NULL; + ObStorageS3Base::reset(); +} + +int ObStorageS3MultiPartWriter::open_(const ObString &uri, ObObjectStorageInfo *storage_info) +{ + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + if (OB_UNLIKELY(is_opened_)) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "s3 multipart writer already opened, cannot open again", K(ret), K(uri)); + } else if (OB_FAIL(ObStorageS3Base::open(uri, storage_info))) { + OB_LOG(WARN, "failed to open in s3 base", K(ret), K(uri)); + } else { + Aws::S3::Model::CreateMultipartUploadRequest request; + request.WithBucket(bucket_.ptr()).WithKey(object_.ptr()); + if (default_s3_crc_algo == ObStorageCRCAlgorithm::OB_CRC32_ALGO) { + request.SetChecksumAlgorithm(Aws::S3::Model::ChecksumAlgorithm::CRC32); + } + Aws::S3::Model::CreateMultipartUploadOutcome outcome; + if (OB_FAIL(s3_client_->create_multipart_upload(request, outcome))) { + OB_LOG(WARN, "failed to create s3 multipart upload", K(ret)); + } else if (!outcome.IsSuccess()) { + handle_s3_outcome(outcome, ret); + OB_LOG(WARN, "failed to create multipart upload for s3", + K(ret), K_(bucket), K_(object)); + } else { + const bool check_crc = (default_s3_crc_algo == ObStorageCRCAlgorithm::OB_CRC32_ALGO); + void *sum_hash_buf = NULL; + const Aws::String &upload_id = outcome.GetResult().GetUploadId(); + if (OB_ISNULL(upload_id_ = static_cast(allocator_.alloc(upload_id.size() + 1)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "failed to alloc buf for s3 multipartupload upload id", + K(ret), K(upload_id.size())); + } else if (OB_ISNULL(base_buf_ = + static_cast(allocator_.alloc(S3_MULTIPART_UPLOAD_BUFFER_SIZE)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "failed to alloc buf for s3 multipartupload buffer", + K(ret), K(S3_MULTIPART_UPLOAD_BUFFER_SIZE)); + } else if (check_crc && OB_ISNULL(sum_hash_buf = allocator_.alloc(sizeof(Aws::Utils::Crypto::CRC32)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "failed to alloc buf for s3 checksum hash", + K(ret), "size", sizeof(Aws::Utils::Crypto::CRC32)); + } else if (check_crc && FALSE_IT(sum_hash_ = new(sum_hash_buf)Aws::Utils::Crypto::CRC32())) { + } else { + STRCPY(upload_id_, upload_id.c_str()); + base_buf_pos_ = 0; + file_length_ = 0; + is_opened_ = true; + } + + if (OB_FAIL(ret)) { + cleanup(); + } + } + } + + if (OB_FAIL(ret)) { + reset(); + } + return ret; +} + +int ObStorageS3MultiPartWriter::write_(const char *buf, const int64_t size) +{ + int ret = OB_SUCCESS; + int64_t fill_size = 0; + int64_t buf_pos = 0; + ObExternalIOCounterGuard io_guard; + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_NOT_INIT; + OB_LOG(WARN, "s3 multipart writer not opened", K(ret)); + } else if (OB_ISNULL(buf) || OB_UNLIKELY(size < 0)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "buf is NULL or size is invalid", K(ret), KP(buf), K(size)); + } + + while (OB_SUCC(ret) && buf_pos != size) { + fill_size = MIN(S3_MULTIPART_UPLOAD_BUFFER_SIZE - base_buf_pos_, size - buf_pos); + memcpy(base_buf_ + base_buf_pos_, buf + buf_pos, fill_size); + base_buf_pos_ += fill_size; + buf_pos += fill_size; + if (base_buf_pos_ == S3_MULTIPART_UPLOAD_BUFFER_SIZE) { + if (OB_FAIL(write_single_part_())) { + OB_LOG(WARN, "failed to write single s3 part", K(ret), K_(bucket), K_(object)); + } else { + base_buf_pos_ = 0; + } + } + } + + if (OB_SUCCESS == ret) { + file_length_ += size; + } + return ret; +} + +int ObStorageS3MultiPartWriter::pwrite_(const char *buf, const int64_t size, const int64_t offset) +{ + UNUSED(offset); + return write(buf, size); +} + +int ObStorageS3MultiPartWriter::close_() +{ + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_NOT_INIT; + OB_LOG(WARN, "s3 multipart writer cannot close before it is opened", K(ret)); + } else if (base_buf_pos_ > 0) { + if (OB_FAIL(write_single_part_())) { + OB_LOG(WARN, "failed to upload last part into s3", K(ret), K_(base_buf_pos)); + cleanup(); + } else { + base_buf_pos_ = 0; + } + } + + // list parts + Aws::S3::Model::CompletedMultipartUpload completedMultipartUpload; + int64_t part_num = 0; + if (OB_SUCC(ret)) { + Aws::S3::Model::ListPartsRequest request; + request.WithBucket(bucket_.ptr()).WithKey(object_.ptr()).WithUploadId(upload_id_); + request.SetMaxParts(OB_STORAGE_LIST_MAX_NUM); + Aws::S3::Model::ListPartsOutcome outcome; + do { + if (OB_FAIL(s3_client_->list_parts(request, outcome))) { + OB_LOG(WARN, "failed to list s3 multipart upload parts", K(ret)); + } else if (!outcome.IsSuccess()) { + handle_s3_outcome(outcome, ret); + OB_LOG(WARN, "failed to list s3 uploaded parts", + K(ret), K_(bucket), K_(object)); + } else { + const Aws::Vector &parts = outcome.GetResult().GetParts(); + part_num += parts.size(); + for (int64_t i = 0; OB_SUCC(ret) && i < parts.size(); i++) { + Aws::S3::Model::CompletedPart tmp_part; + tmp_part.WithPartNumber(parts[i].GetPartNumber()).WithETag(parts[i].GetETag()); + completedMultipartUpload.AddParts(std::move(tmp_part)); + } + } + request.SetPartNumberMarker(outcome.GetResult().GetNextPartNumberMarker()); + } while (OB_SUCC(ret) && outcome.GetResult().GetIsTruncated()); + } + + // complete upload + if (OB_SUCC(ret)) { + Aws::S3::Model::CompleteMultipartUploadRequest request; + request.WithBucket(bucket_.ptr()).WithKey(object_.ptr()); + request.WithUploadId(upload_id_).WithMultipartUpload(completedMultipartUpload); + const bool check_crc = (default_s3_crc_algo == ObStorageCRCAlgorithm::OB_CRC32_ALGO); + Aws::String complete_crc; + if (check_crc) { + if (OB_ISNULL(sum_hash_)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "sum_hash should not be null", K(ret)); + } else { + complete_crc = Aws::Utils::HashingUtils::Base64Encode(sum_hash_->GetHash().GetResult()); + request.SetChecksumCRC32(complete_crc); + } + } + + Aws::S3::Model::CompleteMultipartUploadOutcome outcome; + if (FAILEDx(s3_client_->complete_multipart_upload(request, outcome))) { + OB_LOG(WARN, "failed to complete s3 multipart upload", K(ret)); + } else if (!outcome.IsSuccess()) { + handle_s3_outcome(outcome, ret); + OB_LOG(WARN, "failed to complete multipart upload for s3", + K(ret), K_(bucket), K_(object)); + } else if (check_crc) { + const char *request_id = outcome.GetResult().GetRequestId().c_str(); + const int64_t crc_len = complete_crc.length(); + const Aws::String& expect_crc = outcome.GetResult().GetChecksumCRC32(); + + if ((0 != strcmp(expect_crc.substr(0, crc_len).c_str(), complete_crc.c_str())) + || (part_num != std::stoi(expect_crc.substr(crc_len + 1).c_str()))) { + ret = OB_CHECKSUM_ERROR; + OB_LOG(WARN, "occur checksum error when complete multipartupload", + K(ret), K(request_id), K(part_num), K(complete_crc.c_str()), K(expect_crc.c_str())); + } + } + } + + reset(); + return ret; +} + +int ObStorageS3MultiPartWriter::cleanup() +{ + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_NOT_INIT; + OB_LOG(WARN, "s3 multipart writer not opened", K(ret)); + } else { + Aws::S3::Model::AbortMultipartUploadRequest request; + request.WithBucket(bucket_.ptr()).WithKey(object_.ptr()); + request.WithUploadId(upload_id_); + Aws::S3::Model::AbortMultipartUploadOutcome outcome; + if (OB_FAIL(s3_client_->abort_multipart_upload(request, outcome))) { + OB_LOG(WARN, "failed to abort s3 multipart upload", K(ret)); + } else if (!outcome.IsSuccess()) { + handle_s3_outcome(outcome, ret); + OB_LOG(WARN, "failed to abort s3 multipart upload", + K(ret), K_(bucket), K_(object), K_(partnum), K_(upload_id)); + } + } + return ret; +} + +int ObStorageS3MultiPartWriter::write_single_part_() +{ + // TODO @fangdan: compress data + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + ++partnum_; // partnum is between 1 and 10000 + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_NOT_INIT; + OB_LOG(WARN, "s3 multipart writer not opened", K(ret)); + } else if (partnum_ > MAX_S3_PART_NUM) { + ret = OB_OUT_OF_ELEMENT; + OB_LOG(WARN, "out of s3 part num effective range", K(ret), K_(partnum), K(MAX_S3_PART_NUM)); + } else { + Aws::S3::Model::UploadPartRequest request; + request.WithBucket(bucket_.ptr()).WithKey(object_.ptr()); + request.WithPartNumber(partnum_).WithUploadId(upload_id_); + std::shared_ptr data_stream = + Aws::MakeShared(S3_SDK); + data_stream->write(base_buf_, base_buf_pos_); + data_stream->flush(); + request.SetBody(data_stream); + if (default_s3_crc_algo == ObStorageCRCAlgorithm::OB_CRC32_ALGO) { + if (OB_ISNULL(sum_hash_)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "sum_hash should not be null", K(ret)); + } else { + Aws::Utils::Crypto::CRC32 part_hash; + request.SetChecksumAlgorithm(Aws::S3::Model::ChecksumAlgorithm::CRC32); + Aws::String part_str(base_buf_, base_buf_pos_); + Aws::Utils::ByteBuffer part_crc_buf = part_hash.Calculate(part_str).GetResult(); + sum_hash_->Update(part_crc_buf.GetUnderlyingData(), part_crc_buf.GetLength()); + } + } + + Aws::S3::Model::UploadPartOutcome outcome; + if (FAILEDx(s3_client_->upload_part(request, outcome))) { + OB_LOG(WARN, "failed to upload s3 multipart", K(ret)); + } else if (!outcome.IsSuccess()) { + handle_s3_outcome(outcome, ret); + OB_LOG(WARN, "failed to upload part into s3", + K(ret), K_(bucket), K_(object), K_(partnum)); + } else { + OB_LOG(DEBUG, "succed upload a part into s3", K_(partnum), K_(bucket), K_(object)); + } + } + return ret; +} + +} // common +} // oceanbase \ No newline at end of file diff --git a/deps/oblib/src/lib/restore/ob_storage_s3_base.h b/deps/oblib/src/lib/restore/ob_storage_s3_base.h new file mode 100644 index 000000000..c79c86db2 --- /dev/null +++ b/deps/oblib/src/lib/restore/ob_storage_s3_base.h @@ -0,0 +1,547 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef SRC_LIBRARY_SRC_LIB_RESTORE_OB_STORAGE_S3_BASE_H_ +#define SRC_LIBRARY_SRC_LIB_RESTORE_OB_STORAGE_S3_BASE_H_ + +#include +#include "lib/restore/ob_i_storage.h" +#include "lib/container/ob_array.h" +#include "lib/container/ob_se_array.h" +#include "lib/container/ob_array_iterator.h" +#include "lib/container/ob_se_array_iterator.h" +#include +#include + +#pragma push_macro("private") +#undef private +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#pragma pop_macro("private") + +namespace oceanbase +{ +namespace common +{ + +// Before using s3, you need to initialize s3 enviroment. +// Thread safe guaranteed by user. +int init_s3_env(); + +// You need to clean s3 resource when not use cos any more. +// Thread safe guaranteed by user. +void fin_s3_env(); + +// default s3 checksum algorithm +static ObStorageCRCAlgorithm default_s3_crc_algo = ObStorageCRCAlgorithm::OB_INVALID_CRC_ALGO; +// set checksum algorithm for writing object into s3 +void set_s3_checksum_algorithm(const ObStorageCRCAlgorithm crc_algo); +// get current checksum algorithm +ObStorageCRCAlgorithm get_s3_checksum_algorithm(); + +static constexpr int64_t S3_CONNECT_TIMEOUT_MS = 10 * 1000; +static constexpr int64_t S3_REQUEST_TIMEOUT_MS = 10 * 1000; +static constexpr int64_t MAX_S3_CONNECTIONS_PER_CLIENT = 128; +static constexpr int64_t STOP_S3_TIMEOUT_US = 10 * 1000L; // 10ms +// max allowed idle duration for a s3 client: 12h +static constexpr int64_t MAX_S3_CLIENT_IDLE_DURATION = 12 * 3600 * 1000 * 1000L; +static constexpr int64_t MAX_S3_CLIENT_MAP_THRESHOLD = 500; + +// TODO: check length +static constexpr int MAX_S3_REGION_LENGTH = 128; +static constexpr int MAX_S3_ENDPOINT_LENGTH = 128; +static constexpr int MAX_S3_ACCESS_ID_LENGTH = 128; // ak, access key id +static constexpr int MAX_S3_SECRET_KEY_LENGTH = 128; // sk, secret key +static constexpr int MAX_S3_CLIENT_NUM = 97; +static constexpr int MAX_S3_PART_NUM = 10000; +static constexpr int64_t S3_MULTIPART_UPLOAD_BUFFER_SIZE = 8 * 1024 * 1024L; + +static constexpr char OB_S3_APPENDABLE_FORMAT_CONTENT_V1[] = "version=1"; +static constexpr char OB_STORAGE_S3_ALLOCATOR[] = "StorageS3"; +static constexpr char S3_SDK[] = "S3SDK"; + +struct ObS3Account +{ + ObS3Account(); + ~ObS3Account(); + void reset(); + bool is_valid() const { return is_valid_; } + int64_t hash() const; + TO_STRING_KV(K_(is_valid), K_(delete_mode), K_(region), K_(endpoint), K_(access_id)); + + int parse_from(const char *storage_info_str, const int64_t size); + int set_field(const char *value, char *field, const uint32_t field_length); + + bool is_valid_; + int64_t delete_mode_; + char region_[MAX_S3_REGION_LENGTH]; // region of endpoint + char endpoint_[MAX_S3_ENDPOINT_LENGTH]; + char access_id_[MAX_S3_ACCESS_ID_LENGTH]; // ak + char secret_key_[MAX_S3_SECRET_KEY_LENGTH]; // sk +}; + +class ObS3MemoryManager : public Aws::Utils::Memory::MemorySystemInterface +{ +public: + ObS3MemoryManager() : attr_() + { + attr_.label_ = S3_SDK; + } + virtual ~ObS3MemoryManager() {} + // when aws init/shutdown, it will execute like this: init/shutdown_memory_system->Begin()/End() + virtual void Begin() override {} + virtual void End() override {} + + virtual void *AllocateMemory(std::size_t blockSize, + std::size_t alignment, const char *allocationTag = NULL) override + { + UNUSED(allocationTag); + std::size_t real_alignment = MAX(alignment, 16); // should not be smaller than 16 + return ob_malloc_align(real_alignment, blockSize, attr_); + } + virtual void FreeMemory(void *memoryPtr) override + { + ob_free_align(memoryPtr); + memoryPtr = NULL; + } + +private: + ObMemAttr attr_; +}; + +class ObS3Logger : public Aws::Utils::Logging::LogSystemInterface +{ +public: + ObS3Logger() {} + virtual ~ObS3Logger() {} + // Gets the currently configured log level for this logger. + virtual Aws::Utils::Logging::LogLevel GetLogLevel(void) const override; + // Does a printf style output to the output stream. Don't use this, it's unsafe. See LogStream + virtual void Log(Aws::Utils::Logging::LogLevel logLevel, const char* tag, const char* formatStr, ...) override; + // Writes the stream to the output stream. + virtual void LogStream(Aws::Utils::Logging::LogLevel logLevel, const char* tag, const Aws::OStringStream &messageStream) override; + // Writes any buffered messages to the underlying device if the logger supports buffering. + virtual void Flush() override {} +}; + +class ObS3Client +{ +public: + ObS3Client(); + virtual ~ObS3Client(); + int init(const ObS3Account &account); + void destroy(); + bool is_stopped() const; + bool try_stop(const int64_t timeout = STOP_S3_TIMEOUT_US); + void stop(); + void increase(); + void release(); + TO_STRING_KV(KP(&lock_), K_(is_inited), K_(ref_cnt), K_(last_modified_ts), KP(client_)); + + int head_object(const Aws::S3::Model::HeadObjectRequest &request, + Aws::S3::Model::HeadObjectOutcome &outcome); + int put_object(const Aws::S3::Model::PutObjectRequest &request, + Aws::S3::Model::PutObjectOutcome &outcome); + int get_object(const Aws::S3::Model::GetObjectRequest &request, + Aws::S3::Model::GetObjectOutcome &outcome); + int delete_object(const Aws::S3::Model::DeleteObjectRequest &request, + Aws::S3::Model::DeleteObjectOutcome &outcome); + int put_object_tagging(const Aws::S3::Model::PutObjectTaggingRequest &request, + Aws::S3::Model::PutObjectTaggingOutcome &outcome); + int list_objects_v2(const Aws::S3::Model::ListObjectsV2Request &request, + Aws::S3::Model::ListObjectsV2Outcome &outcome); + int list_objects(const Aws::S3::Model::ListObjectsRequest &request, + Aws::S3::Model::ListObjectsOutcome &outcome); + int get_object_tagging(const Aws::S3::Model::GetObjectTaggingRequest &request, + Aws::S3::Model::GetObjectTaggingOutcome &outcome); + int create_multipart_upload(const Aws::S3::Model::CreateMultipartUploadRequest &request, + Aws::S3::Model::CreateMultipartUploadOutcome &outcome); + int list_parts(const Aws::S3::Model::ListPartsRequest &request, + Aws::S3::Model::ListPartsOutcome &outcome); + int complete_multipart_upload(const Aws::S3::Model::CompleteMultipartUploadRequest &request, + Aws::S3::Model::CompleteMultipartUploadOutcome &outcome); + int abort_multipart_upload(const Aws::S3::Model::AbortMultipartUploadRequest &request, + Aws::S3::Model::AbortMultipartUploadOutcome &outcome); + int upload_part(const Aws::S3::Model::UploadPartRequest &request, + Aws::S3::Model::UploadPartOutcome &outcome); + int list_multipart_uploads(const Aws::S3::Model::ListMultipartUploadsRequest &request, + Aws::S3::Model::ListMultipartUploadsOutcome &outcome); + +private: + int init_s3_client_configuration_(const ObS3Account &account, + Aws::S3::S3ClientConfiguration &config); + + template + using S3OperationFunc = OutcomeType (Aws::S3::S3Client::*)(const RequestType &) const; + + template + int do_s3_operation_(S3OperationFunc s3_op_func, + const RequestType &request, OutcomeType &outcome); + +private: + SpinRWLock lock_; + bool is_inited_; + bool stopped_; + int64_t ref_cnt_; + int64_t last_modified_ts_; + Aws::S3::S3Client *client_; +}; + +class ObS3Env +{ +public: + static ObS3Env &get_instance(); + + // global init s3 env resource, must and only can be called once + int init(); + // global clean s3 resource when don't use s3 any more + void destroy(); + + int get_or_create_s3_client(const ObS3Account &account, ObS3Client *&client); + void stop(); + +private: + ObS3Env(); + int clean_s3_client_map_(); + +private: + SpinRWLock lock_; + bool is_inited_; + ObS3MemoryManager s3_mem_manger_; + Aws::SDKOptions aws_options_; + hash::ObHashMap s3_client_map_; +}; + +struct S3ObjectMeta : public ObStorageObjectMetaBase +{ +}; + +class SafeExecutor +{ +public: + template + int do_safely(Function f, Obj obj, Args && ... args) + { + int ret = OB_SUCCESS; + try { + ret = std::mem_fn(f)(obj, std::forward(args)...); + } catch (const std::exception &e) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "caught exception when doing s3 operation", K(ret), K(e.what()), KP(this)); + } catch (...) { + ret = OB_S3_ERROR; + OB_LOG(WARN, "caught unknown exception when doing s3 operation", K(ret), KP(this)); + } + return ret; + } +}; + +class ObStorageS3Util; + +class ObStorageS3Base : public SafeExecutor +{ +public: + ObStorageS3Base(); + virtual ~ObStorageS3Base(); + virtual void reset(); + + virtual int open(const ObString &uri, ObObjectStorageInfo *storage_info); + virtual bool is_inited() const { return is_inited_; } + int build_bucket_and_object_name(const ObString &uri); + + int get_s3_file_meta(S3ObjectMeta &meta) + { + return do_safely(&ObStorageS3Base::get_s3_file_meta_, this, meta); + } + +protected: + int get_s3_file_meta_(S3ObjectMeta &meta); + int do_list_(const int64_t max_list_num, const char *delimiter, + const Aws::String &next_marker, Aws::S3::Model::ListObjectsOutcome &outcome); + +protected: + ObArenaAllocator allocator_; + ObS3Client *s3_client_; + ObString bucket_; + ObString object_; + +private: + bool is_inited_; + ObS3Account s3_account_; + + friend class ObStorageS3Util; + DISALLOW_COPY_AND_ASSIGN(ObStorageS3Base); +}; + +class ObStorageS3Writer : public ObStorageS3Base, public ObIStorageWriter +{ +public: + ObStorageS3Writer(); + virtual ~ObStorageS3Writer(); + + virtual int open(const ObString &uri, ObObjectStorageInfo *storage_info) override + { + return do_safely(&ObStorageS3Writer::open_, this, uri, storage_info); + } + virtual int write(const char *buf, const int64_t size) override + { + return do_safely(&ObStorageS3Writer::write_, this, buf, size); + } + virtual int pwrite(const char *buf, const int64_t size, const int64_t offset) override + { + return do_safely(&ObStorageS3Writer::pwrite_, this, buf, size, offset); + } + virtual int close() override + { + return do_safely(&ObStorageS3Writer::close_, this); + } + virtual int64_t get_length() const override { return file_length_; } + virtual bool is_opened() const override { return is_opened_; } + +protected: + int open_(const ObString &uri, ObObjectStorageInfo *storage_info); + int write_(const char *buf, const int64_t size); + int write_obj_(const char *obj_name, const char *buf, const int64_t size); + int pwrite_(const char *buf, const int64_t size, const int64_t offset); + int close_(); + +protected: + bool is_opened_; + int64_t file_length_; + +private: + DISALLOW_COPY_AND_ASSIGN(ObStorageS3Writer); +}; + +class ObStorageS3Reader : public ObStorageS3Base, public ObIStorageReader +{ +public: + ObStorageS3Reader(); + virtual ~ObStorageS3Reader(); + virtual void reset() override; + + virtual int open(const ObString &uri, + ObObjectStorageInfo *storage_info, const bool head_meta = true) override + { + return do_safely(&ObStorageS3Reader::open_, this, uri, storage_info, head_meta); + } + virtual int pread(char *buf, const int64_t buf_size, const int64_t offset, int64_t &read_size) override + { + return do_safely(&ObStorageS3Reader::pread_, this, buf, buf_size, offset, read_size); + } + virtual int close() override + { + return do_safely(&ObStorageS3Reader::close_, this); + } + virtual int64_t get_length() const override { return file_length_; } + virtual bool is_opened() const override { return is_opened_; } + +protected: + int open_(const ObString &uri, ObObjectStorageInfo *storage_info, const bool head_meta = true); + int pread_(char *buf, const int64_t buf_size, const int64_t offset, int64_t &read_size); + int close_(); + +protected: + bool is_opened_; + bool has_meta_; + int64_t file_length_; + +private: + DISALLOW_COPY_AND_ASSIGN(ObStorageS3Reader); +}; + +class ObStorageS3Util : public SafeExecutor, public ObIStorageUtil +{ +public: + ObStorageS3Util(); + virtual ~ObStorageS3Util(); + + virtual int open(ObObjectStorageInfo *storage_info) override; + virtual void close() override; + virtual int head_object_meta(const ObString &uri, ObStorageObjectMetaBase &obj_meta) override; + + virtual int is_exist(const ObString &uri, bool &exist) override + { + return do_safely(&ObStorageS3Util::is_exist_, this, uri, exist); + } + virtual int get_file_length(const ObString &uri, int64_t &file_length) override + { + return do_safely(&ObStorageS3Util::get_file_length_, this, uri, file_length); + } + virtual int del_file(const ObString &uri) override + { + return do_safely(&ObStorageS3Util::del_file_, this, uri); + } + virtual int write_single_file(const ObString &uri, const char *buf, const int64_t size) override + { + return do_safely(&ObStorageS3Util::write_single_file_, this, uri, buf, size); + } + virtual int mkdir(const ObString &uri) override + { + return do_safely(&ObStorageS3Util::mkdir_, this, uri); + } + virtual int list_files(const ObString &uri, ObBaseDirEntryOperator &op) override + { + return do_safely(&ObStorageS3Util::list_files_, this, uri, op); + } + virtual int list_files(const ObString &uri, ObStorageListCtxBase &list_ctx) override + { + return do_safely(&ObStorageS3Util::list_files2_, this, uri, list_ctx); + } + virtual int del_dir(const ObString &uri) override + { + return do_safely(&ObStorageS3Util::del_dir_, this, uri); + } + virtual int list_directories(const ObString &uri, ObBaseDirEntryOperator &op) override + { + return do_safely(&ObStorageS3Util::list_directories_, this, uri, op); + } + virtual int is_tagging(const ObString &uri, bool &is_tagging) override + { + return do_safely(&ObStorageS3Util::is_tagging_, this, uri, is_tagging); + } + virtual int del_unmerged_parts(const ObString &uri) override + { + return do_safely(&ObStorageS3Util::del_unmerged_parts_, this, uri); + } + +private: + int is_exist_(const ObString &uri, bool &exist); + int get_file_length_(const ObString &uri, int64_t &file_length); + int del_file_(const ObString &uri); + int write_single_file_(const ObString &uri, const char *buf, const int64_t size); + int mkdir_(const ObString &uri); + int list_files_(const ObString &uri, ObBaseDirEntryOperator &op); + int list_files2_(const ObString &uri, ObStorageListCtxBase &list_ctx); + int del_dir_(const ObString &uri); + int list_directories_(const ObString &uri, ObBaseDirEntryOperator &op); + int is_tagging_(const ObString &uri, bool &is_tagging); + int del_unmerged_parts_(const ObString &uri); + + int delete_object_(ObStorageS3Base &s3_base); + int tagging_object_(ObStorageS3Base &s3_base); + +private: + bool is_opened_; + ObObjectStorageInfo *storage_info_; +}; + +class ObStorageS3AppendWriter : public ObStorageS3Writer +{ +public: + ObStorageS3AppendWriter(); + virtual ~ObStorageS3AppendWriter(); + + virtual int open(const ObString &uri, ObObjectStorageInfo *storage_info) override + { + return do_safely(&ObStorageS3AppendWriter::open_, this, uri, storage_info); + } + virtual int write(const char *buf, const int64_t size) override + { + return do_safely(&ObStorageS3AppendWriter::write_, this, buf, size); + } + virtual int pwrite(const char *buf, const int64_t size, const int64_t offset) override + { + return do_safely(&ObStorageS3AppendWriter::pwrite_, this, buf, size, offset); + } + virtual int close() override + { + return do_safely(&ObStorageS3AppendWriter::close_, this); + } + virtual int64_t get_length() const override; + virtual bool is_opened() const override { return is_opened_; } + +protected: + int open_(const ObString &uri, ObObjectStorageInfo *storage_info); + int write_(const char *buf, const int64_t size); + int pwrite_(const char *buf, const int64_t size, const int64_t offset); + int close_(); + +private: + ObObjectStorageInfo *storage_info_; + + DISALLOW_COPY_AND_ASSIGN(ObStorageS3AppendWriter); +}; + +class ObStorageS3MultiPartWriter : public ObStorageS3Base, public ObIStorageWriter +{ +public: + ObStorageS3MultiPartWriter(); + virtual ~ObStorageS3MultiPartWriter(); + virtual void reset() override; + + virtual int open(const ObString &uri, ObObjectStorageInfo *storage_info) override + { + return do_safely(&ObStorageS3MultiPartWriter::open_, this, uri, storage_info); + } + virtual int write(const char *buf, const int64_t size) override + { + return do_safely(&ObStorageS3MultiPartWriter::write_, this, buf, size); + } + virtual int pwrite(const char *buf, const int64_t size, const int64_t offset) override + { + return do_safely(&ObStorageS3MultiPartWriter::pwrite_, this, buf, size, offset); + } + virtual int close() override + { + return do_safely(&ObStorageS3MultiPartWriter::close_, this); + } + virtual int64_t get_length() const override { return file_length_; } + virtual bool is_opened() const override { return is_opened_; } + + int cleanup(); + +private: + int open_(const ObString &uri, ObObjectStorageInfo *storage_info); + int write_(const char *buf, const int64_t size); + int pwrite_(const char *buf, const int64_t size, const int64_t offset); + int close_(); + int write_single_part_(); + +protected: + bool is_opened_; + char *base_buf_; + int64_t base_buf_pos_; + char *upload_id_; + int partnum_; + int64_t file_length_; + Aws::Utils::Crypto::CRC32 *sum_hash_; // for calc the complete crc based on each part's crc. + +private: + DISALLOW_COPY_AND_ASSIGN(ObStorageS3MultiPartWriter); +}; + +} // common +} // oceanbase + +#endif \ No newline at end of file diff --git a/deps/oblib/unittest/lib/CMakeLists.txt b/deps/oblib/unittest/lib/CMakeLists.txt index 1367ae7f0..1011f704e 100644 --- a/deps/oblib/unittest/lib/CMakeLists.txt +++ b/deps/oblib/unittest/lib/CMakeLists.txt @@ -88,6 +88,9 @@ oblib_addtest(resource/test_resource_mgr.cpp) #oblib_addtest(restore/test_storage_file.cpp) #oblib_addtest(restore/test_storage_oss.cpp) oblib_addtest(restore/test_storage_cos.cpp) +oblib_addtest(restore/test_storage_s3.cpp) +oblib_addtest(restore/test_object_storage.cpp) +oblib_addtest(restore/test_common_storage.cpp) oblib_addtest(restore/test_storage_info.cpp) #oblib_addtest(restore/test_storage.cpp) oblib_addtest(stat/test_di_cache.cpp) diff --git a/deps/oblib/unittest/lib/alloc/test_malloc_allocator.cpp b/deps/oblib/unittest/lib/alloc/test_malloc_allocator.cpp index da7bca6bd..289da8fd4 100644 --- a/deps/oblib/unittest/lib/alloc/test_malloc_allocator.cpp +++ b/deps/oblib/unittest/lib/alloc/test_malloc_allocator.cpp @@ -12,6 +12,7 @@ #include #include "lib/alloc/ob_malloc_allocator.h" +#include "lib/allocator/ob_malloc.h" using namespace oceanbase::lib; using namespace oceanbase::common; @@ -60,6 +61,17 @@ TEST(TestMallocAllocator, idle) OB_MALLOC_BIG_BLOCK_SIZE)); } +TEST(TestMallocAllocator, ob_malloc_align) +{ + void *ptr = ob_malloc_align(1, 4, "test"); + ASSERT_TRUE(ptr != NULL); + ASSERT_EQ(0, (int64_t)ptr % 16); + + ptr = ob_malloc_align(4096, 4, "test"); + ASSERT_TRUE(ptr != NULL); + ASSERT_EQ(0, (int64_t)ptr % 4096); +} + int main(int argc, char *argv[]) { signal(49, SIG_IGN); diff --git a/deps/oblib/unittest/lib/restore/test_common_storage.cpp b/deps/oblib/unittest/lib/restore/test_common_storage.cpp new file mode 100644 index 000000000..c75ffaacb --- /dev/null +++ b/deps/oblib/unittest/lib/restore/test_common_storage.cpp @@ -0,0 +1,1439 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#include "test_common_storage.h" +#include "test_common_storage_util.h" + +namespace oceanbase { + +using namespace oceanbase::common; + +/*-------------------------------------ObTestStorageMeta--------------------------------------*/ +int ObTestStorageMeta::build_config(const ObTestStorageType type) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_valid_type(type))) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid argument", K(ret), K(type)); + } else { + switch (type) { + case ObTestStorageType::TEST_STORAGE_S3: + build_s3_cfg(); + if (config_.is_valid() && OB_FAIL(init_s3_env())) { + OB_LOG(WARN, "fail to init s3 env", K(ret)); + } + break; + case ObTestStorageType::TEST_STORAGE_OSS: + build_oss_cfg(); + if (config_.is_valid() && OB_FAIL(init_oss_env())) { + OB_LOG(WARN, "fail to init oss env", K(ret)); + } + break; + case ObTestStorageType::TEST_STORAGE_COS: + build_cos_cfg(); + if (config_.is_valid() && OB_FAIL(init_cos_env())) { + OB_LOG(WARN, "fail to init cos env", K(ret)); + } + break; + case ObTestStorageType::TEST_STORAGE_FS: + build_fs_cfg(); + break; + default: + ret = OB_ERR_UNEXPECTED; + break; + } + + if (config_.is_valid()) { + type_ = type; + } + } + return ret; +} + +bool ObTestStorageMeta::is_valid() const +{ + return is_valid_type(type_) && config_.is_valid(); +} + +bool ObTestStorageMeta::is_valid_type(const ObTestStorageType type) const +{ + return !(type == ObTestStorageType::TEST_STORAGE_INVALID || + type == ObTestStorageType::TEST_STORAGE_MAX); +} + +void ObTestStorageMeta::build_s3_cfg() +{ + if (!config_.is_valid()) { + if (0 != STRCMP(S3_BUCKET, INVALID_STR) && + 0 != STRCMP(S3_REGION, INVALID_STR) && + 0 != STRCMP(S3_ENDPOINT, INVALID_STR) && + 0 != STRCMP(S3_AK, INVALID_STR) && + 0 != STRCMP(S3_SK, INVALID_STR)) { + config_.set_bucket(S3_BUCKET); + config_.set_region(S3_REGION); + config_.set_endpoint(S3_ENDPOINT); + config_.set_ak(S3_AK); + config_.set_sk(S3_SK); + config_.is_valid_ = true; + } else { + config_.is_valid_ = false; + } + } +} + +void ObTestStorageMeta::build_oss_cfg() +{ + if (!config_.is_valid()) { + if (0 != STRCMP(OSS_BUCKET, INVALID_STR) && + 0 != STRCMP(OSS_ENDPOINT, INVALID_STR) && + 0 != STRCMP(OSS_AK, INVALID_STR) && + 0 != STRCMP(OSS_SK, INVALID_STR)) { + config_.set_bucket(OSS_BUCKET); + config_.set_endpoint(OSS_ENDPOINT); + config_.set_ak(OSS_AK); + config_.set_sk(OSS_SK); + config_.is_valid_ = true; + } else { + config_.is_valid_ = false; + } + } +} + +void ObTestStorageMeta::build_cos_cfg() +{ + if (!config_.is_valid()) { + if (0 != STRCMP(COS_BUCKET, INVALID_STR) && + 0 != STRCMP(COS_ENDPOINT, INVALID_STR) && + 0 != STRCMP(COS_AK, INVALID_STR) && + 0 != STRCMP(COS_SK, INVALID_STR) && + 0 != STRCMP(COS_APPID, INVALID_STR)) { + config_.set_bucket(COS_BUCKET); + config_.set_endpoint(COS_ENDPOINT); + config_.set_ak(COS_AK); + config_.set_sk(COS_SK); + config_.set_appid(COS_APPID); + config_.is_valid_ = true; + } else { + config_.is_valid_ = false; + } + } +} + +void ObTestStorageMeta::build_fs_cfg() +{ + if (0 == STRCMP(FS_PATH, INVALID_STR)) { + config_.is_valid_ = false; + } else { + if (getcwd(config_.fs_path_, sizeof(config_.fs_path_)) != nullptr) { + config_.is_valid_ = true; + } else { + config_.is_valid_ = false; + } + } +} + +/*-------------------------------------TestStorageListOp--------------------------------------*/ +class TestStorageListOp : public ObBaseDirEntryOperator +{ +public: + TestStorageListOp() {} + virtual ~TestStorageListOp() { reset(); } + int func(const dirent *entry) override; + + void reset() { item_names_.reset(); } + + ObArray item_names_; +private : + ObArenaAllocator allocator_; +}; + +int TestStorageListOp::func(const dirent *entry) +{ + int ret = OB_SUCCESS; + char *name_buf = (char *)allocator_.alloc(strlen(entry->d_name) + 1); + if (OB_ISNULL(name_buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN,"fail to alloc buf for item name", K(ret), "name", entry->d_name); + } else { + STRCPY(name_buf, entry->d_name); + item_names_.push_back(name_buf); + } + return ret; +} + +/*-------------------------------------TestCommonStorage--------------------------------------*/ +class TestCommonStorage : public testing::Test +{ +public: + typedef hash::ObHashMap MetaMap; + // for object_storage, it is the common prefix + // for file system, it is the parent dir + const char *default_dir = "TestCStorageDir"; + const char *default_put_obj = "cs_put"; + const char *default_append_obj = "cs_append.bak"; + const static int64_t default_round = 5; + const char *default_multi_obj = "cs_multipart"; + const static int64_t default_size = 512 * 1024; + const static int64_t part_size = 2 * 1024 * 1024; + +public: + TestCommonStorage(); + virtual ~TestCommonStorage(); + + virtual void SetUp() {} + virtual void TearDown() {} + virtual void TestBody() {} + + void reset(); + int init(); + + int mainly_check(const bool is_basic_situation, const bool clear_prev_data, const bool clear_cur_data); // Entrance of all + +private: + + int check_basic_situation(); + int check_abnormal_situation(); + + int common_prepare(); + + /* + * basic situation check + */ + int basic_prepare(); + int check_basic_writer(); + int check_basic_appender(); + int check_basic_multipartwriter(); + int check_basic_reader(); // include reader and adaptive_reader + int check_basic_util(); + + /* + * abnormal situation check + */ + int abnormal_prepare(); + int check_abnormal_writer(); + int check_abnormal_appender(); + int check_abnormal_multipartwriter(); + int check_abnormal_reader(); // include reader and adaptive_reader + int check_abnormal_util(); + + // + // some common func + // + int init_storage_info(); + int clear_all_data(); + void fin_obj_env(const ObTestStorageType type); + int build_uri(char *uri_buf, const int64_t buf_len, const char *file_name); + int delete_listed_objects(const common::ObString &uri); + int create_fs_dir(const common::ObString &uri, const bool drop_if_exist = true); + int is_exist(const common::ObString &uri, const bool is_adaptive, bool &is_exist); + int remove_empty_dir(const common::ObString &uri); + int force_delete_dir(const common::ObString &uri); + int write_single_file(const common::ObString &uri, const int64_t length, const int64_t buf_len = -1); + int write_single_file(const common::ObString &uri, const char *buf, const int64_t length); + int write_multi_files(const common::ObString &uri, const int64_t length, const int64_t file_num, + const char *file_prefix, const int64_t start_idx = 0); + int append_write(const common::ObString &uri, const int64_t offset, const int64_t length); + int read_single_file(const common::ObString &uri, const int64_t offset, const int64_t length); + int delete_single_file(const common::ObString &uri); + int adaptive_read_file(const common::ObString &uri, const int64_t offset, const int64_t length); + int get_file_length(const common::ObString &uri, const bool is_adaptive, int64_t &file_length); + int list_directories(const common::ObString &uri, const bool is_adaptive, common::ObBaseDirEntryOperator &op); + int list_files(const common::ObString &uri, const bool is_adaptive, common::ObBaseDirEntryOperator &op); + int detect_storage_obj_meta(const common::ObString &uri, const bool is_adaptive, const bool need_fragment_meta, + ObStorageObjectMeta &obj_meta); + +private: + bool is_inited_; + MetaMap meta_map_; + ObTestStorageMeta *cur_meta_; + ObTestStorageType cur_type_; + ObObjectStorageInfo cur_storage_info_; + + bool need_clear_prev_data_; + bool need_clear_cur_data_; +}; + +TestCommonStorage::TestCommonStorage() + : meta_map_(), cur_meta_(nullptr), cur_type_(ObTestStorageType::TEST_STORAGE_INVALID), + cur_storage_info_(), need_clear_prev_data_(false), need_clear_cur_data_(false) +{} + +TestCommonStorage::~TestCommonStorage() +{} + +void TestCommonStorage::reset() +{ + meta_map_.clear(); + cur_type_ = ObTestStorageType::TEST_STORAGE_INVALID; + need_clear_prev_data_ = false; + need_clear_cur_data_ = false; +} + +int TestCommonStorage::init() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(init_storage_info())) { + OB_LOG(WARN, "fail to init storage info", K(ret)); + } else { + is_inited_ = true; + } + return ret; +} + +int TestCommonStorage::init_storage_info() +{ + int ret = OB_SUCCESS; + uint8_t type_start = static_cast(ObTestStorageType::TEST_STORAGE_INVALID) + 1; + uint8_t type_end = static_cast(ObTestStorageType::TEST_STORAGE_MAX); + if (OB_FAIL(meta_map_.create(type_end - 1, "TestObjMetaMap"))) { + OB_LOG(WARN, "fail to create hashmap", K(ret), K(type_end)); + } + + for (uint8_t i = type_start; OB_SUCC(ret) && i < type_end; ++i) { + ObTestStorageType type = static_cast(i); + ObTestStorageMeta meta; + if (OB_FAIL(meta.build_config(type))) { + OB_LOG(WARN, "fail to build config", K(ret), K(type)); + } else if (meta.is_valid()) { + if (OB_FAIL(meta_map_.set_refactored(i, meta))) { + OB_LOG(WARN, "fail to set refactored", K(ret), K(type)); + } + } + } + return ret; +} + +void TestCommonStorage::fin_obj_env(const ObTestStorageType type) +{ + switch (type) { + case ObTestStorageType::TEST_STORAGE_S3: + fin_s3_env(); + break; + case ObTestStorageType::TEST_STORAGE_OSS: + fin_oss_env(); + break; + case ObTestStorageType::TEST_STORAGE_COS: + fin_cos_env(); + break; + default: + break; + } +} + +int TestCommonStorage::mainly_check(const bool is_basic_situation, const bool clear_prev_data, const bool clear_cur_data) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + OB_LOG(WARN, "not init", K(ret), K(is_basic_situation)); + } else { + need_clear_prev_data_ = clear_prev_data; + need_clear_cur_data_ = clear_cur_data; + } + + for (MetaMap::iterator it = meta_map_.begin(); OB_SUCC(ret) && it != meta_map_.end(); ++it) { + cur_storage_info_.reset(); + ObTestStorageMeta &meta = it->second; + if (OB_UNLIKELY(!meta.is_valid())) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "meta should not be invalid", K(ret), K(meta)); + } else if (meta.is_obj_type() && OB_FAIL(TestCommonStorageUtil::build_object_storage_info(meta.config_.bucket_, + meta.config_.endpoint_, meta.config_.ak_, meta.config_.sk_, meta.config_.region_, meta.config_.appid_, + cur_storage_info_))) { + OB_LOG(WARN, "fail to build object storage info", K(ret), K(meta)); + } else if (meta.is_file_type() && OB_FAIL(TestCommonStorageUtil::build_fs_storage_info(cur_storage_info_))) { + OB_LOG(WARN, "fail to build fs storage info", K(ret), K(meta)); + } else if (FALSE_IT(cur_meta_ = &meta)) { + } else if (FALSE_IT(cur_type_ = meta.type_)) { + } else { + if (need_clear_prev_data_ && clear_all_data()) { + OB_LOG(WARN, "fail to clear all data", K(ret), K_(cur_type)); + } + + if (OB_FAIL(ret)) { + } else if (is_basic_situation) { + if (OB_FAIL(check_basic_situation())) { + OB_LOG(WARN, "fail to check basic situation", K(ret), K(meta)); + } + } else { + if (OB_FAIL(check_abnormal_situation())) { + OB_LOG(WARN, "fail to check abnormal situation", K(ret), K(meta)); + } + } + + if (OB_FAIL(ret)) { + } else if (need_clear_cur_data_ && clear_all_data()) { + OB_LOG(WARN, "fail to clear all data", K(ret), K_(cur_type)); + } + + fin_obj_env(meta.type_); + } + + uint8_t type = MIN(static_cast(cur_type_), static_cast(ObTestStorageType::TEST_STORAGE_MAX)); + const char *ret_info = (OB_SUCCESS == ret) ? "PASSED!" : "FAILED!"; + printf("[%s] finish %s test, %s\n", test_storage_type_str_arr[type], is_basic_situation ? "basic" : "abnormal", ret_info); + } + return ret; +} + +int TestCommonStorage::check_basic_situation() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(basic_prepare())) { + OB_LOG(WARN, "fail to basic_prepare", K(ret)); + } else if (OB_FAIL(check_basic_writer())) { + OB_LOG(WARN, "fail to check basic writer", K(ret)); + } else if (OB_FAIL(check_basic_appender())) { + OB_LOG(WARN, "fail to check basic appender", K(ret)); + } else if (OB_FAIL(check_basic_multipartwriter())) { + OB_LOG(WARN, "fail to check basic multipartwriter", K(ret)); + } else if (OB_FAIL(check_basic_reader())) { + OB_LOG(WARN, "fail to check basic reader", K(ret)); + } else if (OB_FAIL(check_basic_util())) { + OB_LOG(WARN, "fail to check basic util", K(ret)); + } + return ret; +} + +int TestCommonStorage::check_abnormal_situation() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(abnormal_prepare())) { + OB_LOG(WARN, "fail to abnormal_prepare", K(ret)); + } else if (OB_FAIL(check_abnormal_writer())) { + OB_LOG(WARN, "fail to check abnormal writer", K(ret)); + } else if (OB_FAIL(check_abnormal_appender())) { + OB_LOG(WARN, "fail to check abnormal appender", K(ret)); + } else if (OB_FAIL(check_abnormal_multipartwriter())) { + OB_LOG(WARN, "fail to check abnormal multipartwriter", K(ret)); + } else if (OB_FAIL(check_abnormal_reader())) { + OB_LOG(WARN, "fail to check abnormal reader", K(ret)); + } else if (OB_FAIL(check_abnormal_util())) { + OB_LOG(WARN, "fail to check abnormal util", K(ret)); + } + return ret; +} + +int TestCommonStorage::common_prepare() +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(cur_meta_)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "cur meta should not be null", K(ret)); + } else if (cur_meta_->is_file_type()) { + char uri_buf[OB_MAX_URI_LENGTH] = { 0 }; + if (OB_FAIL(build_uri(uri_buf, OB_MAX_URI_LENGTH, nullptr))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(create_fs_dir(uri_buf, false/*drop_if_exist*/))) { + OB_LOG(WARN, "fail to create fs dir", K(ret), K(uri_buf)); + } + } + return ret; +} + +/*----------------------------------------- START --- BUILD --- DIFFERENT --- CASE -----------------------------------------*/ +int TestCommonStorage::basic_prepare() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(common_prepare())) { + OB_LOG(WARN, "fail to common prepare", K(ret)); + } + return ret; +} + +int TestCommonStorage::check_basic_writer() +{ + int ret = OB_SUCCESS; + char uri_buf[OB_MAX_URI_LENGTH] = { 0 }; + // write "/parent_path/cs_put" + if (OB_FAIL(build_uri(uri_buf, OB_MAX_URI_LENGTH, default_put_obj))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(write_single_file(uri_buf, default_size))) { + OB_LOG(WARN, "fail to write single file", K(ret), K(uri_buf)); + } + + // mkdir "/parent_path/append_file1.bak/" and write "/parent_path/append_file1.bak/@APD_PART@FORMAT_META" + if (FAILEDx(build_uri(uri_buf, OB_MAX_URI_LENGTH, "append_file1.bak/"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(create_fs_dir(uri_buf, true))) { + OB_LOG(WARN, "fail to create fs dir", K(ret), K(uri_buf)); + } else if (OB_FAIL(build_uri(uri_buf, OB_MAX_URI_LENGTH, "append_file1.bak/@APD_PART@FORMAT_META"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(write_single_file(uri_buf, default_size))) { + OB_LOG(WARN, "fail to write single file", K(ret), K(uri_buf)); + } + // write "/parent_path/append_file1.bak/@APD_PART@0-100" + if (FAILEDx(build_uri(uri_buf, OB_MAX_URI_LENGTH, "append_file1.bak/@APD_PART@0-100.bak"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(write_single_file(uri_buf, 100))) { + OB_LOG(WARN, "fail to write single file", K(ret), K(uri_buf)); + } + // write "/parent_path/append_file1.bak/@APD_PART@100-300" + if (FAILEDx(build_uri(uri_buf, OB_MAX_URI_LENGTH, "append_file1.bak/@APD_PART@100-300.bak"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(write_single_file(uri_buf, 200))) { + OB_LOG(WARN, "fail to write single file", K(ret), K(uri_buf)); + } + // write "/parent_path/append_file1.bak/@APD_PART@200-500" + if (FAILEDx(build_uri(uri_buf, OB_MAX_URI_LENGTH, "append_file1.bak/@APD_PART@200-500.bak"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(write_single_file(uri_buf, 300))) { + OB_LOG(WARN, "fail to write single file", K(ret), K(uri_buf)); + } + return ret; +} + +int TestCommonStorage::check_basic_appender() +{ + int ret = OB_SUCCESS; + char uri_buf[OB_MAX_URI_LENGTH] = { 0 }; + if (OB_FAIL(build_uri(uri_buf, OB_MAX_URI_LENGTH, default_append_obj))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else { + const int64_t round = default_round; + const int64_t each_append_size = 100; + for (int64_t i = 0; OB_SUCC(ret) && i < round; ++i) { + if (OB_FAIL(append_write(uri_buf, each_append_size * i, each_append_size))) { + OB_LOG(WARN, "fail to append_write", K(ret), K(i)); + } + } + } + + return ret; +} + +int TestCommonStorage::check_basic_multipartwriter() +{ + int ret = OB_SUCCESS; + char uri_buf[OB_MAX_URI_LENGTH] = { 0 }; + if (OB_FAIL(build_uri(uri_buf, OB_MAX_URI_LENGTH, default_multi_obj))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else { + ObStorageMultiPartWriter multiwriter; + ObArenaAllocator allocator; + char *buf = NULL; + const int64_t buf_len = part_size; + if (OB_ISNULL(buf = static_cast(allocator.alloc(buf_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "fail to alloc", K(ret), K(buf_len)); + } else { + MEMSET(buf, 'a', buf_len); + if (OB_FAIL(multiwriter.open(uri_buf, &cur_storage_info_))) { + OB_LOG(WARN, "fail to open multipartwriter", K(ret), K(uri_buf)); + } else { + const int64_t round = default_round; + for (int64_t i = 0; OB_SUCC(ret) && i < round; ++i) { + if (OB_FAIL(multiwriter.pwrite(buf, buf_len, buf_len * i))) { + OB_LOG(WARN, "fail to pwrite", K(ret), K(i)); + } + } + + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(multiwriter.close())) { + ret = (OB_SUCCESS != ret) ? ret : tmp_ret; + OB_LOG(WARN, "fail to close multipartwriter", K(ret), K(tmp_ret), K(uri_buf)); + } + } + } + } + return ret; +} + +int TestCommonStorage::check_basic_reader() +{ + int ret = OB_SUCCESS; + char uri_buf[OB_MAX_URI_LENGTH] = { 0 }; + // read "/parent_path/cs_put" + if (OB_FAIL(build_uri(uri_buf, OB_MAX_URI_LENGTH, default_put_obj))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(read_single_file(uri_buf, 0, default_size))) { + OB_LOG(WARN, "fail to read single file", K(ret), K(uri_buf)); + } + + // read "/parent_path/append_file1.bak/@APD_PART@FORMAT_META" + if (FAILEDx(build_uri(uri_buf, OB_MAX_URI_LENGTH, "append_file1.bak/@APD_PART@FORMAT_META"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(read_single_file(uri_buf, 0, default_size))) { + OB_LOG(WARN, "fail to read single file", K(ret), K(uri_buf)); + } + + // read "/parent_path/cs_append.bak" + if (FAILEDx(build_uri(uri_buf, OB_MAX_URI_LENGTH, default_append_obj))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(adaptive_read_file(uri_buf, 200/*offset*/, 100/*length*/))) { + OB_LOG(WARN, "fail to adaptive read file", K(ret), K(uri_buf)); + } + + // read "/parent_path/cs_multipart" + if (FAILEDx(build_uri(uri_buf, OB_MAX_URI_LENGTH, default_multi_obj))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(read_single_file(uri_buf, part_size * 2, part_size))) { + OB_LOG(WARN, "fail to read single file", K(ret), K(uri_buf)); + } + + // adaptive read "/parent_path/append_file1.bak/" + if (FAILEDx(build_uri(uri_buf, OB_MAX_URI_LENGTH, "append_file1.bak"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(adaptive_read_file(uri_buf, 100/*offset*/, 400))) { + OB_LOG(WARN, "fail to adaptive read file", K(ret), K(uri_buf)); + } + + return ret; +} + +int TestCommonStorage::check_basic_util() +{ + int ret = OB_SUCCESS; + /* + * Mainly to check these interfaces: + * + * mkdir、write_single_file、del_dir、is_tagging、is_exist、get_file_length、 + * list_appendable_file_fragments、del_file、detect_storage_obj_meta、 + * list_files、list_directories + */ + char uri_buf[OB_MAX_URI_LENGTH] = { 0 }; + const int64_t buf_len = OB_MAX_URI_LENGTH; + + // create dir && delete dir && is_exist + if (OB_FAIL(build_uri(uri_buf, buf_len, "tmp_dir/"))) { + OB_LOG(WARN, "fail to build uri", K(ret), K(default_dir)); + } else if (OB_FAIL(create_fs_dir(uri_buf, true/*drop_if_exist*/))) { + OB_LOG(WARN, "fail to create fs dir", K(ret), K(uri_buf)); + } else if (OB_FAIL(remove_empty_dir(uri_buf))) { // it will delete dir and check its exist + OB_LOG(WARN, "fail to remove empty dir", K(ret), K(uri_buf)); + } + + // write_single_file && get_file_length && del_file + int64_t file_length = -1; + if (FAILEDx(build_uri(uri_buf, OB_MAX_URI_LENGTH, "tmp_write"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(write_single_file(uri_buf, default_size))) { + OB_LOG(WARN, "fail to write single file", K(ret), K(uri_buf)); + } else if (OB_FAIL(get_file_length(uri_buf, false/*is_adaptive*/, file_length))) { + OB_LOG(WARN, "fail to get file length", K(ret), K(uri_buf)); + } else if (OB_UNLIKELY(file_length != default_size)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "get file length is not same with original length", K(ret), K(file_length)); + } else if (OB_FAIL(delete_single_file(uri_buf))) { + OB_LOG(WARN, "fail to delete single file", K(ret), K(uri_buf)); + } + + // list_directories: ---tmp_dir_1 + // ---sub_dir1 + // ---sub_dir2 + TestStorageListOp list_op; + if (FAILEDx(build_uri(uri_buf, buf_len, "tmp_dir_1/"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(create_fs_dir(uri_buf, true/*drop_if_exist*/))) { + OB_LOG(WARN, "fail to create fs dir", K(ret), K(uri_buf)); + } else if (OB_FAIL(build_uri(uri_buf, buf_len, "tmp_dir_1/sub_dir1"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(create_fs_dir(uri_buf, true/*drop_if_exist*/))) { + OB_LOG(WARN, "fail to create fs dir", K(ret), K(uri_buf)); + } else if (OB_FAIL(build_uri(uri_buf, buf_len, "tmp_dir_1/sub_dir2"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(create_fs_dir(uri_buf, true/*drop_if_exist*/))) { + OB_LOG(WARN, "fail to create fs dir", K(ret), K(uri_buf)); + } else if (OB_FAIL(build_uri(uri_buf, buf_len, "tmp_dir_1/"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(list_directories(uri_buf, false/*is_adaptive*/, list_op))) { + OB_LOG(WARN, "fail to list_directories", K(ret), K(uri_buf)); + } else if (OB_UNLIKELY(list_op.item_names_.size() != 2)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "list directories result is unexpected", "actual_item_size", list_op.item_names_.size()); + } + + // list_files: ---tmp_dir_2 + // ---file1 + // ---file2 + // ---file3 + list_op.reset(); + if (FAILEDx(build_uri(uri_buf, buf_len, "tmp_dir_2/"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(create_fs_dir(uri_buf, true/*drop_if_exist*/))) { + OB_LOG(WARN, "fail to create fs dir", K(ret), K(uri_buf)); + } else if (OB_FAIL(build_uri(uri_buf, buf_len, "tmp_dir_2/file1"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(write_single_file(uri_buf, default_size))) { + OB_LOG(WARN, "fail to write single file", K(ret), K(uri_buf)); + } else if (OB_FAIL(build_uri(uri_buf, buf_len, "tmp_dir_2/file2"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(write_single_file(uri_buf, default_size))) { + OB_LOG(WARN, "fail to write single file", K(ret), K(uri_buf)); + } else if (OB_FAIL(build_uri(uri_buf, buf_len, "tmp_dir_2/file3"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(write_single_file(uri_buf, default_size))) { + OB_LOG(WARN, "fail to write single file", K(ret), K(uri_buf)); + } else if (OB_FAIL(build_uri(uri_buf, buf_len, "tmp_dir_2/"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(list_files(uri_buf, false/*is_adaptive*/, list_op))) { + OB_LOG(WARN, "fail to list_files", K(ret), K(uri_buf)); + } else if (OB_UNLIKELY(list_op.item_names_.size() != 3)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "list files result is unexpected", "actual_item_size", list_op.item_names_.size()); + } + + // list 999 common items in one dir + list_op.reset(); + if (OB_FAIL(build_uri(uri_buf, buf_len, "basic_tmp_list/"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(create_fs_dir(uri_buf, true/*drop_if_exist*/))) { + OB_LOG(WARN, "fail to create fs dir", K(ret), K(uri_buf)); + } else if (OB_FAIL(write_multi_files(uri_buf, 100/*file_length*/, 999/*file_num*/, "basic_tmp_file", 0/*start_idx*/))) { + OB_LOG(WARN, "fail to write multi files", K(ret), K(uri_buf)); + } else if (OB_FAIL(list_files(uri_buf, false/*is_adaptive*/, list_op))) { + OB_LOG(WARN, "fail to list files", K(ret), K(uri_buf)); + } else if (OB_UNLIKELY(list_op.item_names_.size() != 999)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "listed result not match with original", K(ret), "listed_size", list_op.item_names_.size()); + } + + // list 1001 common items in one dir + list_op.reset(); + if (FAILEDx(write_multi_files(uri_buf, 100/*file_length*/, 2/*file_num*/, "basic_tmp_file", 999/*start_idx*/))) { + OB_LOG(WARN, "fail to write multi files", K(ret), K(uri_buf)); + } else if (OB_FAIL(list_files(uri_buf, false/*is_adaptive*/, list_op))) { + OB_LOG(WARN, "fail to list files", K(ret), K(uri_buf)); + } else if (OB_UNLIKELY(list_op.item_names_.size() != 1001)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "listed result not match with original", K(ret), "listed_size", list_op.item_names_.size()); + } + + // detect_storage_obj_meta + // list_files: ---tmp_dir_3 + // ---file1 + // ---append_file2.bak + // ---@APD_PART@FORMAT_META + // ---@APD_PART@0-100 + // ---@APD_PART@100-400 + list_op.reset(); + ObStorageObjectMeta storage_meta; + if (FAILEDx(build_uri(uri_buf, buf_len, "tmp_dir_3/"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(create_fs_dir(uri_buf, true/*drop_if_exist*/))) { + OB_LOG(WARN, "fail to create fs dir", K(ret), K(uri_buf)); + } else if (OB_FAIL(build_uri(uri_buf, buf_len, "tmp_dir_3/file1"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(write_single_file(uri_buf, default_size))) { + OB_LOG(WARN, "fail to write single file", K(ret), K(uri_buf)); + } else if (OB_FAIL(build_uri(uri_buf, buf_len, "tmp_dir_3/append_file2.bak"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(create_fs_dir(uri_buf, true/*drop_if_exist*/))) { + OB_LOG(WARN, "fail to create fs dir", K(ret), K(uri_buf)); + } else if (OB_FAIL(build_uri(uri_buf, buf_len, "tmp_dir_3/append_file2.bak/@APD_PART@FORMAT_META"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(write_single_file(uri_buf, default_size))) { + OB_LOG(WARN, "fail to write single file", K(ret), K(uri_buf)); + } else if (OB_FAIL(build_uri(uri_buf, buf_len, "tmp_dir_3/append_file2.bak/@APD_PART@0-100.bak"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(write_single_file(uri_buf, 100/*file_length*/))) { + OB_LOG(WARN, "fail to write single file", K(ret), K(uri_buf)); + } else if (OB_FAIL(build_uri(uri_buf, buf_len, "tmp_dir_3/append_file2.bak/@APD_PART@100-400.bak"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(write_single_file(uri_buf, 300/*file_length*/))) { + OB_LOG(WARN, "fail to write single file", K(ret), K(uri_buf)); + } else if (OB_FAIL(build_uri(uri_buf, buf_len, "tmp_dir_3/"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(list_files(uri_buf, true/*is_adaptive*/, list_op))) { + OB_LOG(WARN, "fail to list_files", K(ret), K(uri_buf)); + } else if (OB_UNLIKELY(list_op.item_names_.size() != 2)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "adaptive list files result is unexpected", "actual_item_size", list_op.item_names_.size()); + } else if (OB_FAIL(build_uri(uri_buf, buf_len, "tmp_dir_3/append_file2.bak"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(detect_storage_obj_meta(uri_buf, true/*is_adaptive*/, true/*need_fragment*/, storage_meta))) { + OB_LOG(WARN, "fail to detect_storage_obj_meta", K(ret), K(uri_buf)); + } else if (OB_UNLIKELY(!storage_meta.is_valid())) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "appendable storage meta should be valid", K(ret), K(uri_buf), K(storage_meta)); + } else if (OB_UNLIKELY(storage_meta.fragment_metas_.count() != 2 || storage_meta.length_ != 400)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "appendable storage meta should be valid", K(ret), K(uri_buf), K(storage_meta)); + } + + return ret; +} + +int TestCommonStorage::abnormal_prepare() +{ + int ret = OB_SUCCESS; + char uri_buf[OB_MAX_URI_LENGTH] = { 0 }; + const int64_t buf_len = OB_MAX_URI_LENGTH; + if (OB_FAIL(common_prepare())) { + OB_LOG(WARN, "fail to common prepare", K(ret)); + } + + // 1. build a appendable file which not have FORMAT_META + if (FAILEDx(build_uri(uri_buf, buf_len, "ab_tmp_append_file.bak"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(create_fs_dir(uri_buf, true/*drop_if_exist*/))) { + OB_LOG(WARN, "fail to create fs dir", K(ret), K(uri_buf)); + } else if (OB_FAIL(build_uri(uri_buf, buf_len, "ab_tmp_append_file.bak/@APD_PART@0-100.bak"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(write_single_file(uri_buf, 100))) { + OB_LOG(WARN, "fail to write single file", K(ret), K(uri_buf)); + } else if (OB_FAIL(build_uri(uri_buf, buf_len, "ab_tmp_append_file.bak/@APD_PART@100-200.bak"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(write_single_file(uri_buf, 100))) { + OB_LOG(WARN, "fail to write single file", K(ret), K(uri_buf)); + } + + // 2. build a appendable file which content not continuous + if (FAILEDx(build_uri(uri_buf, buf_len, "ab_tmp_append_file_nc.bak"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(create_fs_dir(uri_buf, true/*drop_if_exist*/))) { + OB_LOG(WARN, "fail to create fs dir", K(ret), K(uri_buf)); + } else if (OB_FAIL(build_uri(uri_buf, buf_len, "ab_tmp_append_file_nc.bak/@APD_PART@FORMAT_META"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(write_single_file(uri_buf, 100))) { + OB_LOG(WARN, "fail to write single file", K(ret), K(uri_buf)); + } else if (OB_FAIL(build_uri(uri_buf, buf_len, "ab_tmp_append_file_nc.bak/@APD_PART@0-100.bak"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(write_single_file(uri_buf, 100))) { + OB_LOG(WARN, "fail to write single file", K(ret), K(uri_buf)); + } else if (OB_FAIL(build_uri(uri_buf, buf_len, "ab_tmp_append_file_nc.bak/@APD_PART@200-300.bak"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(write_single_file(uri_buf, 100))) { + OB_LOG(WARN, "fail to write single file", K(ret), K(uri_buf)); + } + return ret; +} + +int TestCommonStorage::check_abnormal_writer() +{ + int ret = OB_SUCCESS; + char uri_buf[OB_MAX_URI_LENGTH] = { 0 }; + // 1. write_content buf length < param_length, the final length is param_length + // [Expect]: succ to write, and the file length will be param_length + // for the file content, there will exist some unknown content(param_length-buf_length). + int64_t actual_len = 0; + if (OB_FAIL(build_uri(uri_buf, OB_MAX_URI_LENGTH, "ab_tmp_w_file1"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(write_single_file(uri_buf, 120/*param_length*/, 100/*buf_length*/))) { + OB_LOG(WARN, "fail to write single file", K(ret), K(uri_buf)); + } else if (OB_FAIL(get_file_length(uri_buf, false/*is_adaptive*/, actual_len))) { + OB_LOG(WARN, "fail to get file length", K(ret), K(uri_buf)); + } else if (OB_UNLIKELY(actual_len != 120)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "actual write length is unexpected", K(ret), K(actual_len), K(uri_buf)); + } + + // 2. write_content buf length > param_length + // [Expect]: succ to write, and file length will be param_length + actual_len = 0; + if (FAILEDx(build_uri(uri_buf, OB_MAX_URI_LENGTH, "ab_tmp_w_file2"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(write_single_file(uri_buf, 40/*param_length*/, 60/*buf_length*/))) { + OB_LOG(WARN, "fail to write single file", K(ret), K(uri_buf)); + } else if (OB_FAIL(get_file_length(uri_buf, false/*is_adaptive*/, actual_len))) { + OB_LOG(WARN, "fail to get file length", K(ret), K(uri_buf)); + } else if (OB_UNLIKELY(actual_len != 40)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "actual write length is unexpected", K(ret), K(actual_len), K(uri_buf)); + } + + return ret; +} + +int TestCommonStorage::check_abnormal_appender() +{ + int ret = OB_SUCCESS; + char uri_buf[OB_MAX_URI_LENGTH] = { 0 }; + // 1. when first append, the offset not start with 0 + // [Expect]: for nfs and s3, succ; for oss and cos, fail + if (OB_FAIL(build_uri(uri_buf, OB_MAX_URI_LENGTH, "ab_tmp_append_file_1.bak"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else { + int tmp_ret = append_write(uri_buf, 100/*offset*/, 200/*length*/); + if (cur_type_ == ObTestStorageType::TEST_STORAGE_S3 || + cur_type_ == ObTestStorageType::TEST_STORAGE_FS) { + // nfs can succ, but the file length is 200, cuz nfs open file with O_APPEND + // s3 can succ, but there won't exist FORMAT_META + if (tmp_ret != OB_SUCCESS) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "s3 and fs can append succ if offset not start with 0", K(ret), K_(cur_type)); + } + } else { + // cos and oss must append with start_offset=0 + if (tmp_ret == OB_SUCCESS) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "cos and oss should append fail if offset not start with 0", K(ret), K_(cur_type)); + } + } + } + + // 2. append offset not continuous + if (FAILEDx(build_uri(uri_buf, OB_MAX_URI_LENGTH, "ab_tmp_append_file_2.bak"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(append_write(uri_buf, 0/*offset*/, 100/*length*/))) { + OB_LOG(WARN, "fail to append write", K(ret), K(uri_buf)); + } else { + int tmp_ret = append_write(uri_buf, 200/*offset*/, 100/*length*/); + if (cur_type_ == ObTestStorageType::TEST_STORAGE_S3 || + cur_type_ == ObTestStorageType::TEST_STORAGE_FS) { + // nfs can succ, but the file length is 200, cuz nfs open file with O_APPEND + // s3 can succ, but there won't exist FORMAT_META + if (tmp_ret != OB_SUCCESS) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "s3 and fs can append succ if offset not continuous", K(ret), K_(cur_type)); + } + } else { + // cos and oss must append with continuous offset + if (tmp_ret == OB_SUCCESS) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "cos and oss should append fail if offset not continuous", K(ret), K_(cur_type)); + } + } + } + return ret; +} + +int TestCommonStorage::check_abnormal_multipartwriter() +{ + int ret = OB_SUCCESS; + return ret; +} + +int TestCommonStorage::check_abnormal_reader() +{ + int ret = OB_SUCCESS; + char uri_buf[OB_MAX_URI_LENGTH] = { 0 }; + // 1. use adaptive_reader to read a normal file + // [Expect]: succ + if (OB_FAIL(build_uri(uri_buf, OB_MAX_URI_LENGTH, "ab_tmp_w_file1"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(adaptive_read_file(uri_buf, 0/*offset*/, 100/*length*/))) { + OB_LOG(WARN, "fail to adaptive read normal file", K(ret), K(uri_buf)); + } + + // 2. use adaptive_reader to read a appendable file which doesn't have FORMAT_META + // [Expect]: return OB_BACKUP_FILE_NOT_EXIST + if (FAILEDx(build_uri(uri_buf, OB_MAX_URI_LENGTH, "ab_tmp_append_file.bak"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(adaptive_read_file(uri_buf, 0/*offset*/, 100/*length*/))) { + if (OB_BACKUP_FILE_NOT_EXIST == ret) { + ret = OB_SUCCESS; + } else { + OB_LOG(WARN, "fail to adaptive read file", K(ret), K(uri_buf)); + } + } + + // 3. use adaptive_reader to read a appendable file which doesn't have continuous + // [Expect]: return error + if (FAILEDx(build_uri(uri_buf, OB_MAX_URI_LENGTH, "ab_tmp_append_file_nc.bak"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(adaptive_read_file(uri_buf, 100/*offset*/, 200/*length*/))) { + ret = OB_SUCCESS; + } + return ret; +} + +int TestCommonStorage::check_abnormal_util() +{ + int ret = OB_SUCCESS; + + char uri_buf[OB_MAX_URI_LENGTH] = { 0 }; + const int64_t buf_len = OB_MAX_URI_LENGTH; + + // 1. create a file and then create a same name dir + // [Expect]: return OB_FILE_ALREADY_EXIST cuz already exist a same name file + bool exist = false; + if (FAILEDx(build_uri(uri_buf, buf_len, "ab_tmp_f1"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(write_single_file(uri_buf, default_size))) { + OB_LOG(WARN, "fail to write single file", K(ret), K(uri_buf)); + } else if (OB_FAIL(build_uri(uri_buf, buf_len, "ab_tmp_f1/"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(is_exist(uri_buf, false/*is_adaptive*/, exist))) { + OB_LOG(WARN, "fail to check is_exist", K(ret), K(uri_buf)); + } else if (OB_UNLIKELY(exist)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "this dir should not be existed, cuz it exists as a file", K(ret), K(uri_buf)); + } else if (OB_FAIL(create_fs_dir(uri_buf, true/*drop_if_exist*/))) { + if (OB_FILE_ALREADY_EXIST == ret) { + ret = OB_SUCCESS; + } else { + OB_LOG(WARN, "fail to create fs dir", K(ret), K(uri_buf)); + } + } + + // 2. create a dir whose parent dir not exist + // [Expect]: succ to create 'ab_tmp_dir1/' and 'ab_tmp_dir1/sub_dir1/' + if (FAILEDx(build_uri(uri_buf, buf_len, "ab_tmp_dir1/sub_dir1/"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(create_fs_dir(uri_buf, true/*drop_if_exist*/))) { + OB_LOG(WARN, "fail to create fs dir", K(ret), K(uri_buf)); + } + + // 3. get file length of appendable file which doesn't have FORMAT_META + // [Expect]: return OB_BACKUP_FILE_NOT_EXIST + int64_t file_len = 0; + if (FAILEDx(build_uri(uri_buf, buf_len, "ab_tmp_append_file.bak"))) { + OB_LOG(WARN, "fail to build uri", K(ret)); + } else if (OB_FAIL(get_file_length(uri_buf, true/*is_adaptive*/, file_len))) { + if (OB_BACKUP_FILE_NOT_EXIST == ret) { + ret = OB_SUCCESS; + } else { + OB_LOG(WARN, "fail to get file length", K(ret)); + } + } + + return ret; +} + +/*----------------------------------------- COMMON --- FUNC -----------------------------------------*/ +int TestCommonStorage::clear_all_data() +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(cur_meta_)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "cur meta should not be null", K(ret)); + } else { + char uri_buf[OB_MAX_URI_LENGTH] = { 0 }; + if (OB_FAIL(build_uri(uri_buf, OB_MAX_URI_LENGTH, nullptr))) { + OB_LOG(WARN, "fail to build uri", K(ret), K_(cur_storage_info)); + } else if (cur_meta_->is_file_type() && OB_FAIL(force_delete_dir(uri_buf))) { + OB_LOG(WARN, "fail to force delete fs dir", K(ret), K(uri_buf)); + } else if (cur_meta_->is_obj_type() && OB_FAIL(delete_listed_objects(uri_buf))) { + OB_LOG(WARN, "fail to delete listed objects", K(ret), K(uri_buf)); + } + } + return ret; +} + +int TestCommonStorage::build_uri(char *uri_buf, const int64_t buf_len, const char *file_name) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(cur_meta_)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "cur meta should not be null", K(ret)); + } else { + if (cur_meta_->is_file_type()) { + if (OB_FAIL(TestCommonStorageUtil::gen_fs_uri(uri_buf, buf_len, cur_meta_->config_.fs_path_, + default_dir, file_name))) { + OB_LOG(WARN, "fail to gen fs uri", K(ret), K(default_dir)); + } + } else if (cur_meta_->is_obj_type()) { + if (OB_FAIL(TestCommonStorageUtil::gen_object_uri(uri_buf, buf_len, cur_meta_->config_.bucket_, + default_dir, file_name))) { + OB_LOG(WARN, "fail to gen object uri", K(ret), "bucket", cur_meta_->config_.bucket_, K(default_dir)); + } + } + } + return ret; +} + +int TestCommonStorage::delete_listed_objects(const common::ObString &uri) +{ + int ret = OB_SUCCESS; + ObStorageUtil util; + if (OB_FAIL(util.open(&cur_storage_info_))) { + OB_LOG(WARN, "fail to open storage_util", K(ret)); + } else { + DelAppendableObjectFragmentOp del_op(uri, util); + if (OB_FAIL(util.list_files(uri, false/*is_adaptive*/, del_op))) { + OB_LOG(WARN, "fail to list files", K(ret), K(uri)); + } + } + + util.close(); + return ret; +} + +int TestCommonStorage::create_fs_dir( + const common::ObString &uri, + const bool drop_if_exist) +{ + int ret = OB_SUCCESS; + ObStorageUtil util; + bool is_dir_exist = false; + if (OB_FAIL(util.open(&cur_storage_info_))) { + OB_LOG(WARN, "fail to open storage_util", K(ret)); + } else if (OB_FAIL(util.is_exist(uri, false, is_dir_exist))) { + OB_LOG(WARN, "fail to check is_exist", K(ret), K(uri)); + } else if (is_dir_exist) { + if (drop_if_exist) { + if (OB_FAIL(force_delete_dir(uri))) { + OB_LOG(WARN, "fail to force delete dir", K(ret), K(uri)); + } else if (OB_FAIL(util.mkdir(uri))) { + OB_LOG(WARN, "fail to mkdir", K(ret), K(uri)); + } + } else { + OB_LOG(INFO, "dir already exist", K(uri)); + } + } else if (OB_FAIL(util.mkdir(uri))) { + OB_LOG(WARN, "fail to mkdir", K(ret), K(uri)); + } + + util.close(); + return ret; +} + +int TestCommonStorage::is_exist(const common::ObString &uri, const bool is_adaptive, bool &is_exist) +{ + int ret = OB_SUCCESS; + ObStorageUtil util; + if (OB_FAIL(util.open(&cur_storage_info_))) { + OB_LOG(WARN, "fail to open storage_util", K(ret)); + } else if (OB_FAIL(util.is_exist(uri, is_adaptive, is_exist))) { + OB_LOG(WARN, "fail to check is_exist", K(ret), K(uri)); + } + + util.close(); + return ret; +} + +int TestCommonStorage::force_delete_dir(const common::ObString &uri) +{ + int ret = OB_SUCCESS; + char cmd[OB_MAX_URI_LENGTH] = { 0 }; + const int64_t offset = strlen(OB_FILE_PREFIX); + + if (OB_FAIL(databuff_printf(cmd, OB_MAX_URI_LENGTH, "%s%.*s", "rm -rf ", + static_cast(uri.length() - offset), uri.ptr() + offset))) { + STORAGE_LOG(WARN, "fail to fill path", K(ret), K(uri)); + } else if (0 != std::system(cmd)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "fail to delete dir", K(ret), K(uri), K(cmd)); + } + return ret; +} + +int TestCommonStorage::remove_empty_dir(const common::ObString &uri) +{ + int ret = OB_SUCCESS; + ObStorageUtil util; + if (OB_FAIL(util.open(&cur_storage_info_))) { + OB_LOG(WARN, "fail to open storage_util", K(ret)); + } else if (OB_FAIL(util.del_dir(uri))) { + OB_LOG(WARN, "fail to delete dir", K(ret), K(uri)); + } + + bool is_dir_exist = false; + if (OB_FAIL(util.is_exist(uri, false/*is_adaptive*/, is_dir_exist))) { + OB_LOG(WARN, "fail to check is_exist", K(ret), K(uri)); + } else if (is_dir_exist) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "fail to delete dir cuz dir still exist", K(ret), K(uri)); + } + + util.close(); + return ret; +} + +int TestCommonStorage::write_single_file(const common::ObString &uri, const int64_t length, const int64_t param_buf_len) +{ + int ret = OB_SUCCESS; + ObArenaAllocator allocator; + int64_t buf_len = param_buf_len; + if (buf_len == -1) { + buf_len = length; + } + char *buf = NULL; + if (OB_UNLIKELY(length < 1)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid argument", K(ret), K(length)); + } else if (OB_ISNULL(buf = static_cast(allocator.alloc(buf_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "fail to alloc", K(ret), K(buf_len)); + } else { + MEMSET(buf, 'a', buf_len); + if (OB_FAIL(write_single_file(uri, buf, length))) { + OB_LOG(WARN, "fail to write single file", K(ret), K(uri), K(length)); + } + } + return ret; +} + +int TestCommonStorage::write_single_file(const common::ObString &uri, const char *buf, const int64_t length) +{ + int ret = OB_SUCCESS; + ObStorageWriter writer; + if (OB_ISNULL(buf) || OB_UNLIKELY(length < 1)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid argument", K(ret), K(length), KP(buf)); + } else if (OB_FAIL(writer.open(uri, &cur_storage_info_))) { + OB_LOG(WARN, "fail to open writer", K(ret), K(uri)); + } else { + if (OB_FAIL(writer.write(buf, length))) { + OB_LOG(WARN, "fail to write", K(ret), K(length), K(uri)); + } + + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(writer.close())) { + ret = (OB_SUCCESS != ret) ? ret : tmp_ret; + OB_LOG(WARN, "fail to close writer", K(ret), K(tmp_ret), K(length), K(uri)); + } + } + return ret; +} + +int TestCommonStorage::write_multi_files( + const common::ObString &uri, + const int64_t length, + const int64_t file_num, + const char *file_prefix, + const int64_t start_idx) +{ + int ret = OB_SUCCESS; + ObArenaAllocator allocator; + const int64_t buf_len = length; + char *buf = NULL; + if (OB_ISNULL(buf = static_cast(allocator.alloc(buf_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "fail to alloc", K(ret), K(length)); + } else { + MEMSET(buf, 'a', buf_len); + const int64_t end_idx = start_idx + file_num; + const bool use_multi_thread = true; + constexpr int64_t thread_cnt = 5; + if (file_num <= thread_cnt || !use_multi_thread) { + char *full_uri = NULL; + if (OB_ISNULL(full_uri = static_cast(allocator.alloc(OB_MAX_URI_LENGTH)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "fail to alloc", K(ret), K(length)); + } + + for (int64_t i = start_idx; OB_SUCC(ret) && (i < end_idx); ++i) { + if (OB_FAIL(databuff_printf(full_uri, OB_MAX_URI_LENGTH, "%.*s%s_%ld", + static_cast(uri.length()), uri.ptr(), file_prefix, (i + 1)))) { + STORAGE_LOG(WARN, "fail to fill path", K(ret), K(uri)); + } else if (OB_FAIL(write_single_file(full_uri, buf, buf_len))) { + OB_LOG(WARN, "fail to write single file", K(ret), K(uri), K(length)); + } + } + } else { + std::thread insert_threads[thread_cnt]; + char **full_uri_arr = NULL; + if (OB_ISNULL(full_uri_arr = static_cast(allocator.alloc(sizeof(void*) * thread_cnt)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "fail to alloc", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && (i < thread_cnt); ++i) { + if (OB_ISNULL(full_uri_arr[i] = static_cast(allocator.alloc(OB_MAX_URI_LENGTH)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "fail to alloc", K(ret)); + } + } + } + + for (auto i = 0; OB_SUCC(ret) && (i < thread_cnt); ++i) { + insert_threads[i] = std::thread([&, i]() { + for (auto j = start_idx + i; OB_SUCC(ret) && j < end_idx; j += thread_cnt) { + if (OB_FAIL(databuff_printf(full_uri_arr[i], OB_MAX_URI_LENGTH, "%.*s%s_%ld", + static_cast(uri.length()), uri.ptr(), file_prefix, (j + 1)))) { + STORAGE_LOG(WARN, "fail to fill path", K(ret), K(uri)); + } else if (OB_FAIL(write_single_file(full_uri_arr[i], buf, buf_len))) { + OB_LOG(WARN, "fail to write single file", K(ret), K(uri), K(length)); + } + } + }); + } + + for (auto i = 0; i < thread_cnt; ++i) { + insert_threads[i].join(); + } + } + } + return ret; +} + +int TestCommonStorage::append_write(const common::ObString &uri, const int64_t offset, const int64_t length) +{ + int ret = OB_SUCCESS; + ObStorageAppender appender; + ObArenaAllocator allocator; + char *buf = NULL; + if (OB_ISNULL(buf = static_cast(allocator.alloc(length)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "fail to alloc", K(ret), K(length)); + } else { + MEMSET(buf, 'a', length); + if (OB_FAIL(appender.open(uri, &cur_storage_info_))) { + OB_LOG(WARN, "fail to open appender", K(ret), K(uri)); + } else { + if (OB_FAIL(appender.pwrite(buf, length, offset))) { + OB_LOG(WARN, "fail to pwrite", K(ret), K(length), K(offset)); + } + + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(appender.close())) { + ret = (OB_SUCCESS != ret) ? ret : tmp_ret; + OB_LOG(WARN, "fail to close appender", K(ret), K(tmp_ret), K(uri), K(length), K(offset)); + } + } + } + return ret; +} + +int TestCommonStorage::read_single_file(const common::ObString &uri, const int64_t offset, const int64_t length) +{ + int ret = OB_SUCCESS; + ObStorageReader reader; + if (OB_FAIL(reader.open(uri, &cur_storage_info_))) { + OB_LOG(WARN, "fail to open reader", K(ret), K(uri)); + } else { + int64_t read_size = 0; + char *buf = NULL; + ObArenaAllocator allocator; + const int64_t buf_len = length; + if (OB_ISNULL(buf = static_cast(allocator.alloc(buf_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "fail to alloc", K(ret), K(length)); + } else if (OB_FAIL(reader.pread(buf, buf_len, offset, read_size))) { + OB_LOG(WARN, "fail to pread", K(ret), K(buf_len), K(uri)); + } else if (OB_UNLIKELY(read_size != length)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "the read size not equal to the write size", K(read_size), "write_size", length, K(uri)); + } + + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(reader.close())) { + ret = (OB_SUCCESS != ret) ? ret : tmp_ret; + OB_LOG(WARN, "fail to close reader", K(ret), K(tmp_ret)); + } + } + return ret; +} + +int TestCommonStorage::delete_single_file(const common::ObString &uri) +{ + int ret = OB_SUCCESS; + ObStorageUtil util; + if (OB_FAIL(util.open(&cur_storage_info_))) { + OB_LOG(WARN, "fail to open storage_util", K(ret)); + } else if (OB_FAIL(util.del_file(uri))) { + OB_LOG(WARN, "fail to delete file", K(ret), K(uri)); + } + + bool is_file_exist = false; + if (FAILEDx(util.is_exist(uri, false/*is_adaptive*/, is_file_exist))) { + OB_LOG(WARN, "fail to check is_exist", K(ret), K(uri)); + } else if (is_file_exist) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "fail to delete file cuz file still exist", K(ret), K(uri)); + } + + util.close(); + return ret; +} + +int TestCommonStorage::adaptive_read_file(const common::ObString &uri, const int64_t offset, const int64_t length) +{ + int ret = OB_SUCCESS; + ObStorageAdaptiveReader adaptive_reader; + if (OB_FAIL(adaptive_reader.open(uri, &cur_storage_info_))) { + OB_LOG(WARN, "fail to open adaptive reader", K(ret), K(uri)); + } else { + int64_t read_size = 0; + char *buf = NULL; + ObArenaAllocator allocator; + const int64_t buf_len = length; + if (OB_ISNULL(buf = static_cast(allocator.alloc(buf_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "fail to alloc", K(ret), K(length)); + } else if (OB_FAIL(adaptive_reader.pread(buf, buf_len, offset, read_size))) { + OB_LOG(WARN, "fail to pread", K(ret), K(buf_len), K(uri)); + } else if (OB_UNLIKELY(read_size != length)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "the adaptive read size not equal to the write size", K(read_size), "write_size", length, K(uri)); + } + + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(adaptive_reader.close())) { + ret = (OB_SUCCESS != ret) ? ret : tmp_ret; + OB_LOG(WARN, "fail to close adaptive reader", K(ret), K(tmp_ret)); + } + } + return ret; +} + +int TestCommonStorage::get_file_length(const common::ObString &uri, const bool is_adaptive, int64_t &file_length) +{ + int ret = OB_SUCCESS; + ObStorageUtil util; + file_length = 0; + if (OB_FAIL(util.open(&cur_storage_info_))) { + OB_LOG(WARN, "fail to open storage_util", K(ret)); + } else if (OB_FAIL(util.get_file_length(uri, is_adaptive, file_length))) { + OB_LOG(WARN, "fail to get file length", K(ret), K(uri)); + } + + util.close(); + return ret; +} + +int TestCommonStorage::list_directories(const common::ObString &uri, const bool is_adaptive, common::ObBaseDirEntryOperator &op) +{ + int ret = OB_SUCCESS; + ObStorageUtil util; + if (OB_FAIL(util.open(&cur_storage_info_))) { + OB_LOG(WARN, "fail to open storage_util", K(ret)); + } else if (OB_FAIL(util.list_directories(uri, is_adaptive, op))) { + OB_LOG(WARN, "fail to list directories", K(ret), K(uri)); + } + util.close(); + return ret; +} + +int TestCommonStorage::list_files(const common::ObString &uri, const bool is_adaptive, common::ObBaseDirEntryOperator &op) +{ + int ret = OB_SUCCESS; + ObStorageUtil util; + if (OB_FAIL(util.open(&cur_storage_info_))) { + OB_LOG(WARN, "fail to open storage_util", K(ret)); + } else if (OB_FAIL(util.list_files(uri, is_adaptive, op))) { + OB_LOG(WARN, "fail to list files", K(ret), K(uri)); + } + util.close(); + return ret; +} + +int TestCommonStorage::detect_storage_obj_meta( + const common::ObString &uri, const bool is_adaptive, + const bool need_fragment_meta, ObStorageObjectMeta &obj_meta) +{ + int ret = OB_SUCCESS; + ObStorageUtil util; + if (OB_FAIL(util.open(&cur_storage_info_))) { + OB_LOG(WARN, "fail to open storage_util", K(ret)); + } else if (OB_FAIL(util.detect_storage_obj_meta(uri, is_adaptive, need_fragment_meta, obj_meta))) { + OB_LOG(WARN, "fail to list files", K(ret), K(uri)); + } + util.close(); + return ret; +} + +/*-------------------------------------Test Cases--------------------------------------*/ +TEST_F(TestCommonStorage, base_situation_test) +{ + TestCommonStorage storage; + ASSERT_EQ(OB_SUCCESS, storage.init()); + ASSERT_EQ(OB_SUCCESS, storage.mainly_check(true/*is_basic*/, true/*clear_prev_data*/, false/*clear_cur_data*/)); +} + +TEST_F(TestCommonStorage, abnormal_situation_test) +{ + TestCommonStorage storage; + ASSERT_EQ(OB_SUCCESS, storage.init()); + ASSERT_EQ(OB_SUCCESS, storage.mainly_check(false/*is_basic*/, false/*clear_pre_data*/, true/*clear_cur_data*/)); +} + +} // end of oceanbase + +int main(int argc, char **argv) +{ + system("rm -f test_common_storage.log*"); + OB_LOGGER.set_file_name("test_common_storage.log", true, true); + OB_LOGGER.set_log_level("DEBUG"); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/deps/oblib/unittest/lib/restore/test_common_storage.h b/deps/oblib/unittest/lib/restore/test_common_storage.h new file mode 100644 index 000000000..5cd493cba --- /dev/null +++ b/deps/oblib/unittest/lib/restore/test_common_storage.h @@ -0,0 +1,149 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#ifndef TEST_STORAGE_COMMON_STORAGE_H_ +#define TEST_STORAGE_COMMON_STORAGE_H_ + +#include "lib/utility/ob_print_utils.h" +#include "lib/hash/ob_hashmap.h" +#include "gtest/gtest.h" + +/** + * USER GUIDE + * + * This test case is for testing different storage media type, like oss, s3, fs, etc. + * It will check reader/writer/util/appender/multipartupload primary interfaces correctness. + * Besides, it will generate some abnormal situations to check the relative function correctness. + * + * If you want to run this test case, you just need to execute some simple steps. + * + * 1. For object storage, for example, s3, you just need to revise S3_BUCKET、S3_REGION、S3_ENDPOINT、S3_AK、 + * S3_SK as correct info, so as to oss、cos + * + * 2. For NFS, you just need to revise FS_PATH as empty string(means "") or some other value, but not INVALID_STR. It will + * build a directory in the same directory as this test bin file. + * + * NOTICE: consider the running time, you'd better check these media one by one, or it may be timeout. + */ + +namespace oceanbase{ +const char *INVALID_STR = "xxx"; + +// S3 CONFIG +const char *S3_BUCKET = INVALID_STR; +const char *S3_REGION = INVALID_STR; +const char *S3_ENDPOINT = INVALID_STR; +const char *S3_AK = INVALID_STR; +const char *S3_SK = INVALID_STR; + +// OSS CONFIG +const char *OSS_BUCKET = INVALID_STR; +const char *OSS_ENDPOINT = INVALID_STR; +const char *OSS_AK = INVALID_STR; +const char *OSS_SK = INVALID_STR; + +// COS CONFIG +const char *COS_BUCKET = INVALID_STR; +const char *COS_ENDPOINT = INVALID_STR; +const char *COS_AK = INVALID_STR; +const char *COS_SK = INVALID_STR; +const char *COS_APPID = INVALID_STR; + +// NFS CONFIG +const char *FS_PATH = INVALID_STR; // if FS_PATH value not equals to invalid string, we will use current path as FS_PATH + +enum class ObTestStorageType : uint8_t +{ + TEST_STORAGE_INVALID = 0, + TEST_STORAGE_OSS = 1, + TEST_STORAGE_S3 = 2, + TEST_STORAGE_COS = 3, + TEST_STORAGE_FS = 4, + TEST_STORAGE_MAX = 5 +}; + +const char test_storage_type_str_arr[5][8] = {"INVALID", "OSS", "S3", "COS", "NFS"}; + +struct ObTestStorageInfoConfig +{ +public: + const static int64_t CFG_BUF_LEN = 1024; + + char bucket_[CFG_BUF_LEN]; + char region_[CFG_BUF_LEN]; + char endpoint_[CFG_BUF_LEN]; + char ak_[CFG_BUF_LEN]; + char sk_[CFG_BUF_LEN]; + union { + char appid_[CFG_BUF_LEN]; + char fs_path_[CFG_BUF_LEN]; + }; + + bool is_valid_; + + ObTestStorageInfoConfig() : is_valid_(false) {} + ~ObTestStorageInfoConfig() {} + + #define SET_FIELD(field_name) \ + void set_##field_name(const char *value) \ + { \ + if (nullptr != value) { \ + const int64_t val_len = strlen(value); \ + MEMCPY(field_name##_, value, val_len); \ + field_name##_[val_len] = '\0'; \ + } \ + } \ + + SET_FIELD(bucket); + SET_FIELD(region); + SET_FIELD(endpoint); + SET_FIELD(ak); + SET_FIELD(sk); + SET_FIELD(appid); + SET_FIELD(fs_path); + + bool is_valid() const { return is_valid_; } + + TO_STRING_KV(K_(bucket), K_(region), K_(endpoint), K_(ak), K_(sk), K_(appid), K_(fs_path)); +}; + +struct ObTestStorageMeta +{ +public: + ObTestStorageType type_; + ObTestStorageInfoConfig config_; + + ObTestStorageMeta() {} + + ~ObTestStorageMeta() {} + + int build_config(const ObTestStorageType type); + bool is_valid() const; + + bool is_file_type() const { return type_ == ObTestStorageType::TEST_STORAGE_FS; } + bool is_obj_type() const { return type_ == ObTestStorageType::TEST_STORAGE_S3 || + type_ == ObTestStorageType::TEST_STORAGE_COS || + type_ == ObTestStorageType::TEST_STORAGE_OSS;} + + TO_STRING_KV(K_(type), K_(config)); + +private: + bool is_valid_type(const ObTestStorageType type) const; + void build_s3_cfg(); + void build_oss_cfg(); + void build_cos_cfg(); + void build_fs_cfg(); + +}; + +} // end of oceanbase + +#endif \ No newline at end of file diff --git a/deps/oblib/unittest/lib/restore/test_common_storage_util.h b/deps/oblib/unittest/lib/restore/test_common_storage_util.h new file mode 100644 index 000000000..c59ad0dc8 --- /dev/null +++ b/deps/oblib/unittest/lib/restore/test_common_storage_util.h @@ -0,0 +1,145 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef TEST_STORAGE_COMMON_STORAGE_UTIL_H_ +#define TEST_STORAGE_COMMON_STORAGE_UTIL_H_ + +#include "lib/restore/ob_storage.h" + +namespace oceanbase { + +using namespace oceanbase::common; + +class TestCommonStorageUtil +{ +public: + static int build_object_storage_info(const char *bucket, const char *endpoint, + const char *ak, const char *sk, const char *region, const char *appid, + ObObjectStorageInfo &storage_info); + + static int build_fs_storage_info(ObObjectStorageInfo &storage_info); + + + // The format of obj_uri will have two types: + // 1. media_type://bucket/raw_dir_path/obj_name + // 2. media_type://bucket/raw_dir_path/ + static int gen_object_uri(char *obj_uri, const int64_t uri_buf_len, const char *bucket, const char *raw_dir_path, const char *obj_name); + + // The format of fs_uri will have two types: + // 1. media_type://pwd_path/raw_dir_path/file_name + // 2. media_type://pwd_path/raw_dir_path/ + static int gen_fs_uri(char *fs_uri, const int64_t uri_buf_len, const char *pwd_path, const char *raw_dir_path, const char *file_name); +}; + +int TestCommonStorageUtil::build_object_storage_info( + const char *bucket, + const char *endpoint, + const char *ak, + const char *sk, + const char *region, + const char *appid, + ObObjectStorageInfo &storage_info) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(bucket) || OB_ISNULL(endpoint) || + OB_ISNULL(ak) || OB_ISNULL(sk)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid argument", K(ret), KP(bucket), KP(region), KP(endpoint), KP(ak), KP(sk)); + } else { + char account[OB_MAX_URI_LENGTH] = { 0 }; + ObStorageType storage_type = ObStorageType::OB_STORAGE_MAX_TYPE; + if (OB_FAIL(get_storage_type_from_path(bucket, storage_type))) { + OB_LOG(WARN, "fail to get storage type from path", K(ret), K(bucket)); + } else if (ObStorageType::OB_STORAGE_FILE == storage_type) { + if (OB_FAIL(storage_info.set(storage_type, account))) { + OB_LOG(WARN, "fail to set storage info", K(ret)); + } + } else { + int64_t pos = 0; + if (OB_FAIL(databuff_printf(account, sizeof(account), pos, + "host=%s&access_id=%s&access_key=%s", endpoint, ak, sk))) { + OB_LOG(WARN, "fail to databuff printf", K(ret), K(endpoint), K(ak), K(sk)); + } else if (ObStorageType::OB_STORAGE_COS == storage_type && + databuff_printf(account, sizeof(account), pos, "&appid=%s", appid)) { + OB_LOG(WARN, "fail to databuff printf", K(ret), K(appid)); + } else if (ObStorageType::OB_STORAGE_S3 == storage_type && + databuff_printf(account, sizeof(account), pos, "&s3_region=%s", region)) { + OB_LOG(WARN, "fail to databuff printf", K(ret), K(region)); + } + + if (FAILEDx(storage_info.set(storage_type, account))) { + OB_LOG(WARN, "fail to set storage info", K(ret), K(storage_type), K(account)); + } + } + } + return ret; +} + +int TestCommonStorageUtil::build_fs_storage_info( + ObObjectStorageInfo &storage_info) +{ + int ret = OB_SUCCESS; + char account[OB_MAX_URI_LENGTH] = { 0 }; + if (OB_FAIL(storage_info.set(ObStorageType::OB_STORAGE_FILE, account))) { + OB_LOG(WARN, "fail to set storage info", K(ret)); + } + return ret; +} + +int TestCommonStorageUtil::gen_object_uri( + char *obj_uri, + const int64_t uri_buf_len, + const char *bucket, + const char *raw_dir_path, + const char *obj_name) +{ + int ret = OB_SUCCESS; + int64_t pos = 0; + const bool exist_obj = (obj_name != nullptr); + if (OB_ISNULL(obj_uri) || OB_ISNULL(bucket) || OB_ISNULL(raw_dir_path)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid argument", K(ret), KP(obj_uri), KP(bucket), KP(raw_dir_path)); + } else if (exist_obj && OB_FAIL(databuff_printf(obj_uri, uri_buf_len, pos, "%s/%s/%s", + bucket, raw_dir_path, obj_name))) { + OB_LOG(WARN, "fail to databuff printf", K(ret), K(bucket), K(raw_dir_path), K(obj_name), K(uri_buf_len)); + } else if (!exist_obj && OB_FAIL(databuff_printf(obj_uri, uri_buf_len, pos, "%s/%s/", bucket, raw_dir_path))) { + OB_LOG(WARN, "fail to databuff printf", K(ret), K(bucket), K(raw_dir_path), K(uri_buf_len)); + } + return ret; +} + +int TestCommonStorageUtil::gen_fs_uri( + char *fs_uri, + const int64_t uri_buf_len, + const char *pwd_path, + const char *raw_dir_path, + const char *file_name) +{ + int ret = OB_SUCCESS; + int64_t pos = 0; + const bool exist_file = (file_name != nullptr); + if (OB_ISNULL(fs_uri) || OB_ISNULL(pwd_path) || OB_ISNULL(raw_dir_path)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid argument", K(ret), KP(fs_uri), KP(pwd_path), KP(raw_dir_path)); + } else if (exist_file && OB_FAIL(databuff_printf(fs_uri, uri_buf_len, pos, "%s%s/%s/%s", "file://", + pwd_path, raw_dir_path, file_name))) { + OB_LOG(WARN, "fail to databuff printf", K(ret), K(pwd_path), K(raw_dir_path), K(file_name), K(uri_buf_len)); + } else if (!exist_file && OB_FAIL(databuff_printf(fs_uri, uri_buf_len, pos, "%s%s/%s/", "file://", + pwd_path, raw_dir_path))) { + OB_LOG(WARN, "fail to databuff printf", K(ret), K(pwd_path), K(raw_dir_path), K(uri_buf_len)); + } + return ret; +} + +} + +#endif \ No newline at end of file diff --git a/deps/oblib/unittest/lib/restore/test_object_storage.cpp b/deps/oblib/unittest/lib/restore/test_object_storage.cpp new file mode 100644 index 000000000..d32f76817 --- /dev/null +++ b/deps/oblib/unittest/lib/restore/test_object_storage.cpp @@ -0,0 +1,1272 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#include "test_object_storage.h" +#include "lib/utility/ob_test_util.h" +#include "lib/restore/ob_storage.h" +#include "lib/allocator/page_arena.h" +#include "lib/container/ob_array_serialization.h" +#include + +using namespace oceanbase::common; + +class TestObjectStorageListOp : public ObBaseDirEntryOperator +{ +public: + TestObjectStorageListOp(const int64_t expect_file_size = -1) : expect_file_size_(expect_file_size) {} + virtual ~TestObjectStorageListOp() {} + int func(const dirent *entry) override; + virtual bool need_get_file_size() const { return true; } + + std::set object_names_; + const int64_t expect_file_size_; +}; + +int TestObjectStorageListOp::func(const dirent *entry) +{ + int ret = OB_SUCCESS; + OB_LOG(INFO, "TestObjectStorageListOp list file", K(entry->d_name)); + const int64_t size = get_size(); + if (expect_file_size_ != -1 && size != expect_file_size_) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "wrong file size", + K_(expect_file_size), K(size), K(entry->d_name)); + } + std::string tmp(entry->d_name); + object_names_.emplace(std::move(tmp)); + return ret; +} + +class TestObjectStorage : public ::testing::Test +{ +public: + TestObjectStorage() {} + virtual ~TestObjectStorage(){} + virtual void SetUp() {} + virtual void TearDown() {} + + static void SetUpTestCase() + { + ASSERT_EQ(OB_SUCCESS, init_oss_env()); + ASSERT_EQ(OB_SUCCESS, init_cos_env()); + ASSERT_EQ(OB_SUCCESS, init_s3_env()); + if (enable_test) { + set_storage_info(bucket, endpoint, secretid, secretkey, appid, region, info_base); + } + } + static void TearDownTestCase() + { + fin_oss_env(); + fin_cos_env(); + fin_s3_env(); + } + + int del_appendable_file(const ObString &uri) + { + int ret = OB_SUCCESS; + ObStorageUtil util; + TestObjectStorageListOp op; + if (OB_FAIL(util.open(&info_base))) { + OB_LOG(WARN, "fail to open util", K(ret), K(uri)); + } else if (OB_FAIL(util.list_files(uri, op))) { + OB_LOG(WARN, "fail to list files", K(ret), K(uri)); + } else { + for (auto &obj_name : op.object_names_) { + std::string tmp = std::string(uri.ptr()) + "/" + obj_name; + if (OB_FAIL(util.del_file(tmp.c_str()))) { + OB_LOG(WARN, "fail to del file", K(ret), K(tmp.c_str())); + break; + } + } + if (OB_SUCC(ret) && OB_FAIL(util.del_file(uri))) { + OB_LOG(WARN, "fail to del normal file", K(ret), K(uri)); + } + } + util.close(); + return ret; + } + + static void set_storage_info(const char *bucket, const char *endpoint, + const char *secretid, const char *secretkey, const char *appid, const char *region, + ObObjectStorageInfo &info_base) + { + char account[OB_MAX_URI_LENGTH] = { 0 }; + ObStorageType storage_type = ObStorageType::OB_STORAGE_MAX_TYPE; + ASSERT_EQ(OB_SUCCESS, get_storage_type_from_path(bucket, storage_type)); + ASSERT_NE(ObStorageType::OB_STORAGE_MAX_TYPE, storage_type); + if (storage_type == ObStorageType::OB_STORAGE_FILE) { + ASSERT_EQ(OB_SUCCESS, info_base.set(storage_type, account)); + } else { + int64_t pos = 0; + ASSERT_EQ(OB_SUCCESS, databuff_printf(account, sizeof(account), pos, + "host=%s&access_id=%s&access_key=%s", + endpoint, secretid, secretkey)); + if (ObStorageType::OB_STORAGE_COS == storage_type) { + ASSERT_EQ(OB_SUCCESS, + databuff_printf(account, sizeof(account), pos, "&appid=%s", appid)); + } else if (ObStorageType::OB_STORAGE_S3 == storage_type) { + ASSERT_EQ(OB_SUCCESS, + databuff_printf(account, sizeof(account), pos, "&s3_region=%s", region)); + } + + ASSERT_EQ(OB_SUCCESS, info_base.set(storage_type, account)); + } + } + +protected: + static char account[OB_MAX_URI_LENGTH]; + static const char *dir_name; + static char uri[OB_MAX_URI_LENGTH]; + static char dir_uri[OB_MAX_URI_LENGTH]; + static ObObjectStorageInfo info_base; + + int object_prefix_len = 5; +}; + +char TestObjectStorage::account[OB_MAX_URI_LENGTH] = { 0 }; +const char *TestObjectStorage::dir_name = "object_storage_unittest_dir"; +char TestObjectStorage::uri[OB_MAX_URI_LENGTH] = { 0 }; +char TestObjectStorage::dir_uri[OB_MAX_URI_LENGTH] = { 0 }; +ObObjectStorageInfo TestObjectStorage::info_base; + +int test_gen_object_meta( + const char **fragments, + const int64_t n_fragments, + const int64_t n_remained_fragments, + const int64_t *expected_start, + const int64_t *expected_end, + const int64_t expected_file_length, + ObStorageObjectMeta &appendable_obj_meta) +{ + int ret = OB_SUCCESS; + dirent entry; + ListAppendableObjectFragmentOp op(false); + appendable_obj_meta.reset(); + + for (int64_t i = 0; OB_SUCC(ret) && i < n_fragments; i++) { + if (OB_FAIL(databuff_printf(entry.d_name, sizeof(entry.d_name), "%s%s", + OB_S3_APPENDABLE_FRAGMENT_PREFIX, fragments[i]))) { + OB_LOG(WARN, "fail to databuff printf", K(ret), K(i), K(fragments[i])); + } else if (OB_FAIL(op.func(&entry))) { + OB_LOG(WARN, "fail to execute op", K(ret), K(i)); + } + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(op.gen_object_meta(appendable_obj_meta))) { + OB_LOG(WARN, "fail to gen object meta", K(ret), K(appendable_obj_meta)); + } else if (ObStorageObjectType::OB_OBJ_SIMULATE_APPEND != appendable_obj_meta.type_) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "not expected value", K(ret), K(appendable_obj_meta.type_)); + } else if (expected_file_length != appendable_obj_meta.length_) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "not expected value", K(ret), K(expected_file_length), K(appendable_obj_meta.length_)); + } else if (n_remained_fragments != appendable_obj_meta.fragment_metas_.count()) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "not expected value", K(ret), K(n_remained_fragments), K(appendable_obj_meta.fragment_metas_.count())); + } + + for (int64_t i = 0; OB_SUCC(ret) && i < n_remained_fragments; i++) { + if (!appendable_obj_meta.fragment_metas_[i].is_data() || !appendable_obj_meta.fragment_metas_[i].is_valid() || + expected_start[i] != appendable_obj_meta.fragment_metas_[i].start_ || + expected_end[i] != appendable_obj_meta.fragment_metas_[i].end_) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "not expected value", K(ret), K(i), K(appendable_obj_meta.fragment_metas_[i]), K(expected_start[i]), K(expected_end[i])); + } + } + return ret; +} + +int test_get_needed_fragments( + const int64_t start, + const int64_t end, + const int64_t n_expected_fragments, + const int64_t *expected_start, + const int64_t *expected_end, + ObStorageObjectMeta &appendable_obj_meta) +{ + int ret = OB_SUCCESS; + ObArray fragments_need_to_read; + if (OB_FAIL(appendable_obj_meta.get_needed_fragments(start, end, fragments_need_to_read))) { + OB_LOG(WARN, "fail to get needed fragments", K(ret), K(start), K(end)); + } else if (n_expected_fragments != fragments_need_to_read.count()) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "not expected value", K(ret), K(n_expected_fragments), K(fragments_need_to_read.count())); + } + + for (int64_t i = 0; OB_SUCC(ret) && i < n_expected_fragments; i++) { + if (!fragments_need_to_read[i].is_data() || !fragments_need_to_read[i].is_valid() || + expected_start[i] != fragments_need_to_read[i].start_ || + expected_end[i] != fragments_need_to_read[i].end_) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "not expected value", K(ret), K(i), K(fragments_need_to_read[i]), K(expected_start[i]), K(expected_end[i])); + } + } + return ret; +} + +TEST_F(TestObjectStorage, test_appendable_object_util) +{ + int ret = OB_SUCCESS; + if (enable_test) { + { + ListAppendableObjectFragmentOp op; + dirent entry; + + // meta file + ASSERT_EQ(OB_SUCCESS, databuff_printf(entry.d_name, sizeof(entry.d_name), "%s%s", + OB_S3_APPENDABLE_FRAGMENT_PREFIX, + OB_S3_APPENDABLE_SEAL_META)); + ASSERT_EQ(OB_SUCCESS, op.func(&entry)); + // foramt file + ASSERT_EQ(OB_SUCCESS, databuff_printf(entry.d_name, sizeof(entry.d_name), "%s%s", + OB_S3_APPENDABLE_FRAGMENT_PREFIX, + OB_S3_APPENDABLE_FORMAT_META)); + ASSERT_EQ(OB_SUCCESS, op.func(&entry)); + + // invalid fragment name + const char *invalid_fragments[] = {"-2--1", "-1-1", "2--2", "3-2", "xxx"}; + for (int64_t i = 0; i < sizeof(invalid_fragments) / sizeof(char *); i++) { + ASSERT_TRUE(sizeof(entry.d_name) >= strlen(invalid_fragments[i]) + 1); + STRCPY(entry.d_name, invalid_fragments[i]); + ASSERT_EQ(OB_SUCCESS, databuff_printf(entry.d_name, sizeof(entry.d_name), "%s%s", + OB_S3_APPENDABLE_FRAGMENT_PREFIX, + invalid_fragments[i])); + ASSERT_EQ(OB_INVALID_ARGUMENT, op.func(&entry)); + } + } + + { + // empty appendable object + const char *fragments[] = {}; + int64_t n_fragments = sizeof(fragments) / sizeof(char *); + int64_t expected_start[] = {}; + int64_t expected_end[] = {}; + int64_t n_remained_fragments = sizeof(expected_start) / sizeof(int64_t); + int64_t expected_file_length = 0; + ObStorageObjectMeta appendable_obj_meta; + ASSERT_EQ(OB_SUCCESS, test_gen_object_meta(fragments, n_fragments, n_remained_fragments, + expected_start, expected_end, expected_file_length, appendable_obj_meta)); + + ObArray fragments_need_to_read; + ASSERT_EQ(OB_INVALID_ARGUMENT, + appendable_obj_meta.get_needed_fragments(-1, 1, fragments_need_to_read)); + ASSERT_EQ(OB_INVALID_ARGUMENT, + appendable_obj_meta.get_needed_fragments(5, 4, fragments_need_to_read)); + ASSERT_EQ(OB_INVALID_ARGUMENT, + appendable_obj_meta.get_needed_fragments(5, 5, fragments_need_to_read)); + ASSERT_EQ(OB_ERR_UNEXPECTED, + appendable_obj_meta.get_needed_fragments(0, 1, fragments_need_to_read)); + } + + { + // one fragment + const char *fragments[] = {"10-20.back"}; + int64_t n_fragments = sizeof(fragments) / sizeof(char *); + int64_t expected_start[] = {10}; + int64_t expected_end[] = {20}; + int64_t n_remained_fragments = sizeof(expected_start) / sizeof(int64_t); + int64_t expected_file_length = 20; + ObStorageObjectMeta appendable_obj_meta; + ASSERT_EQ(OB_SUCCESS, test_gen_object_meta(fragments, n_fragments, n_remained_fragments, + expected_start, expected_end, expected_file_length, appendable_obj_meta)); + + ObArray fragments_need_to_read; + ASSERT_EQ(OB_ERR_UNEXPECTED, + appendable_obj_meta.get_needed_fragments(5, 8, fragments_need_to_read)); + ASSERT_EQ(OB_ERR_UNEXPECTED, + appendable_obj_meta.get_needed_fragments(5, 10, fragments_need_to_read)); + ASSERT_EQ(OB_ERR_UNEXPECTED, + appendable_obj_meta.get_needed_fragments(5, 15, fragments_need_to_read)); + ASSERT_EQ(OB_INVALID_ARGUMENT, + appendable_obj_meta.get_needed_fragments(11, 11, fragments_need_to_read)); + + { + int64_t start = 10; + int64_t end = 18; + int64_t expected_start[] = {10}; + int64_t expected_end[] = {20}; + int64_t n_expected_fragments = sizeof(expected_start) / sizeof(int64_t); + ASSERT_EQ(OB_SUCCESS, test_get_needed_fragments(start, end, n_expected_fragments, expected_start, + expected_end, appendable_obj_meta)); + } + { + int64_t start = 15; + int64_t end = 18; + int64_t expected_start[] = {10}; + int64_t expected_end[] = {20}; + int64_t n_expected_fragments = sizeof(expected_start) / sizeof(int64_t); + ASSERT_EQ(OB_SUCCESS, test_get_needed_fragments(start, end, n_expected_fragments, expected_start, + expected_end, appendable_obj_meta)); + } + { + int64_t start = 19; + int64_t end = 20; + int64_t expected_start[] = {10}; + int64_t expected_end[] = {20}; + int64_t n_expected_fragments = sizeof(expected_start) / sizeof(int64_t); + ASSERT_EQ(OB_SUCCESS, test_get_needed_fragments(start, end, n_expected_fragments, expected_start, + expected_end, appendable_obj_meta)); + } + { + int64_t start = 19; + int64_t end = 21; + int64_t expected_start[] = {10}; + int64_t expected_end[] = {20}; + int64_t n_expected_fragments = sizeof(expected_start) / sizeof(int64_t); + ASSERT_EQ(OB_SUCCESS, test_get_needed_fragments(start, end, n_expected_fragments, expected_start, + expected_end, appendable_obj_meta)); + } + { + int64_t start = 20; + int64_t end = 21; + int64_t expected_start[] = {}; + int64_t expected_end[] = {}; + int64_t n_expected_fragments = sizeof(expected_start) / sizeof(int64_t); + ASSERT_EQ(OB_SUCCESS, test_get_needed_fragments(start, end, n_expected_fragments, expected_start, + expected_end, appendable_obj_meta)); + } + } + + { + // valid fragment name + const char *valid_fragments[] = { + OB_S3_APPENDABLE_FORMAT_META, OB_S3_APPENDABLE_SEAL_META, + "1-7", "2-5", "3-6", "4-7", "1-7", "1-7", "1-5", // covered by "1-7" + "0-3", "0-3", "0-1", "1-2", "2-3", // covered "0-3" + "7-8", "8-9", "9-10", "10-11", "11-12", "12-20", // no gap + // "22-25" & "21-24" are covered by "15-25", and there is a gap from "15-25" to "26-30" + "15-25", "22-25", "21-24", "26-30", + "30-1234567", "10000-1234567", "28-1234566" + }; + int64_t n_fragments = sizeof(valid_fragments) / sizeof(char *); + int64_t expected_start[] = {0, 1, 7, 8, 9, 10, 11, 12, 15, 26, 30}; + int64_t expected_end[] = {3, 7, 8, 9, 10, 11, 12, 20, 25, 30, 1234567}; + int64_t n_remained_fragments = sizeof(expected_start) / sizeof(int64_t); + int64_t expected_file_length = 1234567; + ObStorageObjectMeta appendable_obj_meta; + // (ObStorageObjectType::OB_OBJ_SIMULATE_APPEND); + ASSERT_EQ(OB_SUCCESS, test_gen_object_meta(valid_fragments, n_fragments, n_remained_fragments, + expected_start, expected_end, expected_file_length, appendable_obj_meta)); + + { + int64_t start = 0; + int64_t end = 3; + int64_t expected_start[] = {0}; + int64_t expected_end[] = {3}; + int64_t n_expected_fragments = sizeof(expected_start) / sizeof(int64_t); + ASSERT_EQ(OB_SUCCESS, test_get_needed_fragments(start, end, n_expected_fragments, expected_start, + expected_end, appendable_obj_meta)); + } + { + int64_t start = 1; + int64_t end = 4; + int64_t expected_start[] = {1}; + int64_t expected_end[] = {7}; + int64_t n_expected_fragments = sizeof(expected_start) / sizeof(int64_t); + ASSERT_EQ(OB_SUCCESS, test_get_needed_fragments(start, end, n_expected_fragments, expected_start, + expected_end, appendable_obj_meta)); + } + { + int64_t start = 0; + int64_t end = 15; + int64_t expected_start[] = {0, 1, 7, 8, 9, 10, 11, 12}; + int64_t expected_end[] = {3, 7, 8, 9, 10, 11, 12, 20}; + int64_t n_expected_fragments = sizeof(expected_start) / sizeof(int64_t); + ASSERT_EQ(OB_SUCCESS, test_get_needed_fragments(start, end, n_expected_fragments, expected_start, + expected_end, appendable_obj_meta)); + } + { + ObArray fragments_need_to_read; + ASSERT_EQ(OB_ERR_UNEXPECTED, + appendable_obj_meta.get_needed_fragments(15, 30, fragments_need_to_read)); + } + { + int64_t start = 28; + int64_t end = 2345678; + int64_t expected_start[] = {26, 30}; + int64_t expected_end[] = {30, 1234567}; + int64_t n_expected_fragments = sizeof(expected_start) / sizeof(int64_t); + ASSERT_EQ(OB_SUCCESS, test_get_needed_fragments(start, end, n_expected_fragments, expected_start, + expected_end, appendable_obj_meta)); + } + } + } +} + +TEST_F(TestObjectStorage, test_check_storage_obj_meta) +{ + int ret = OB_SUCCESS; + if (enable_test) { + ObStorageUtil util; + ASSERT_EQ(OB_SUCCESS, util.open(&info_base)); + const char *tmp_append_dir = "test_check_storage_obj_meta"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_append_dir, ts)); + + { + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), + "%s/test_appendable_%ld.back", + dir_uri, ObTimeUtility::current_time())); + // util before append + ObStorageAppender appender; + ASSERT_EQ(OB_SUCCESS, appender.open(uri, &info_base)); + const char *content = "123456789ABC"; + bool is_obj_exist = true; + ASSERT_EQ(OB_SUCCESS, util.is_exist(uri, false/*is_adaptive*/, is_obj_exist)); + ASSERT_FALSE(is_obj_exist); + is_obj_exist = true; + ASSERT_EQ(OB_SUCCESS, util.is_exist(uri, true/*is_adaptive*/, is_obj_exist)); + ASSERT_FALSE(is_obj_exist); + + // util after append + ASSERT_EQ(OB_SUCCESS, appender.pwrite(content, strlen(content), 0)); + is_obj_exist = true; + ASSERT_EQ(OB_SUCCESS, util.is_exist(uri, false/*is_adaptive*/, is_obj_exist)); + if (info_base.get_type() == ObStorageType::OB_STORAGE_S3) { + ASSERT_FALSE(is_obj_exist); + } else { + ASSERT_TRUE(is_obj_exist); + is_obj_exist = false; + } + ASSERT_EQ(OB_SUCCESS, util.is_exist(uri, true/*is_adaptive*/, is_obj_exist)); + ASSERT_TRUE(is_obj_exist); + + int64_t file_length = -1; + if (info_base.get_type() == ObStorageType::OB_STORAGE_S3) { + ASSERT_EQ(OB_BACKUP_FILE_NOT_EXIST, util.get_file_length(uri, false/*is_adaptive*/, file_length)); + } else { + ASSERT_EQ(OB_SUCCESS, util.get_file_length(uri, false/*is_adaptive*/, file_length)); + ASSERT_EQ(strlen(content), file_length); + file_length = -1; + } + ASSERT_EQ(OB_SUCCESS, util.get_file_length(uri, true/*is_adaptive*/, file_length)); + ASSERT_EQ(strlen(content), file_length); + + ASSERT_EQ(OB_SUCCESS, util.del_file(uri, true)); + ASSERT_EQ(OB_SUCCESS, util.is_exist(uri, false/*is_adaptive*/, is_obj_exist)); + ASSERT_FALSE(is_obj_exist); + is_obj_exist = true; + ASSERT_EQ(OB_SUCCESS, util.is_exist(uri, true/*is_adaptive*/, is_obj_exist)); + ASSERT_FALSE(is_obj_exist); + + // open twice + ASSERT_EQ(OB_INIT_TWICE, util.open(&info_base)); + util.close(); + + // invalid storage info + ASSERT_EQ(OB_INVALID_ARGUMENT, util.open(NULL)); + } + } +} + +TEST_F(TestObjectStorage, test_util_list_files) +{ + int ret = OB_SUCCESS; + if (enable_test) { + ObStorageUtil util; + const char *tmp_util_dir = "test_util_list_files"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_util_dir, ts)); + ASSERT_EQ(OB_SUCCESS, util.open(&info_base)); + + { + // wrong uri + uri[0] = '\0'; + TestObjectStorageListOp op; + ASSERT_EQ(OB_INVALID_BACKUP_DEST, util.list_files(uri, op)); + } + + int64_t file_num = 1001; + const char *write_content = "0123456789"; + + // list objects + { + const char *format = "%s/%ld_%ld"; + std::set files; + for (int64_t file_idx = 0; file_idx < file_num; file_idx++) { + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), format, + dir_uri, file_idx, file_idx)); + files.emplace(std::string(uri + strlen(dir_uri) + 1)); + ASSERT_EQ(OB_SUCCESS, util.write_single_file(uri, write_content, strlen(write_content))); + } + + // list and check and clean + TestObjectStorageListOp op; + ASSERT_EQ(OB_SUCCESS, util.list_files(dir_uri, false, op)); + ASSERT_EQ(file_num, op.object_names_.size()); + ASSERT_EQ(files, op.object_names_); + for (int64_t file_idx = 0; file_idx < file_num; file_idx++) { + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), format, + dir_uri, file_idx, file_idx)); + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + } + } + + // list subfolders' objects + { + const char *format = "%s/%ld/%ld"; + std::set files; + for (int64_t file_idx = 0; file_idx < file_num; file_idx++) { + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), format, + dir_uri, file_idx, file_idx)); + files.emplace(std::string(uri + strlen(dir_uri) + 1)); + ASSERT_EQ(OB_SUCCESS, util.write_single_file(uri, write_content, strlen(write_content))); + } + + // list and check and clean + TestObjectStorageListOp op; + ASSERT_EQ(OB_SUCCESS, util.list_files(dir_uri, false, op)); + ASSERT_EQ(file_num, op.object_names_.size()); + ASSERT_EQ(files, op.object_names_); + for (int64_t file_idx = 0; file_idx < file_num; file_idx++) { + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), format, + dir_uri, file_idx, file_idx)); + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + } + } + + { + // list empty dir, now dir_uri should be empty after delete + TestObjectStorageListOp list_empty_op; + ASSERT_EQ(OB_SUCCESS, util.list_files(dir_uri, false, list_empty_op)); + ASSERT_EQ(0, list_empty_op.object_names_.size()); + } + + util.close(); + } +} + +TEST_F(TestObjectStorage, test_util_list_directories) +{ + int ret = OB_SUCCESS; + if (enable_test) { + ObStorageUtil util; + const char *tmp_util_dir = "test_util_list_directories"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_util_dir, ts)); + ASSERT_EQ(OB_SUCCESS, util.open(&info_base)); + + { + // wrong uri + uri[0] = '\0'; + TestObjectStorageListOp op; + ASSERT_EQ(OB_INVALID_BACKUP_DEST, util.list_directories(uri, false/*is_adaptive*/, op)); + } + + int64_t file_num = 1001; + const char *write_content = "0123456789"; + + // list objects + { + const char *format = "%s/%ld/%ld"; + std::set files; + for (int64_t file_idx = 0; file_idx < file_num; file_idx++) { + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), format, + dir_uri, file_idx, file_idx)); + files.emplace(std::string(std::to_string(file_idx))); + ASSERT_EQ(OB_SUCCESS, util.write_single_file(uri, write_content, strlen(write_content))); + } + + // list and check and clean + TestObjectStorageListOp op; + ASSERT_EQ(OB_SUCCESS, util.list_directories(dir_uri, false, op)); + ASSERT_EQ(file_num, op.object_names_.size()); + ASSERT_EQ(files, op.object_names_); + for (int64_t file_idx = 0; file_idx < file_num; file_idx++) { + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), format, + dir_uri, file_idx, file_idx)); + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + } + } + + // no sub dirs + { + const char *format = "%s/%ld_%ld"; + for (int64_t file_idx = 0; file_idx < file_num; file_idx++) { + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), format, + dir_uri, file_idx, file_idx)); + ASSERT_EQ(OB_SUCCESS, util.write_single_file(uri, write_content, strlen(write_content))); + } + + // list and check and clean + TestObjectStorageListOp op; + ASSERT_EQ(OB_SUCCESS, util.list_directories(dir_uri, false, op)); + ASSERT_EQ(0, op.object_names_.size()); + for (int64_t file_idx = 0; file_idx < file_num; file_idx++) { + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), format, + dir_uri, file_idx, file_idx)); + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + } + } + + util.close(); + } +} + +TEST_F(TestObjectStorage, test_util_list_adaptive_files) +{ + int ret = OB_SUCCESS; + if (enable_test) { + ObStorageUtil util; + ASSERT_EQ(OB_SUCCESS, util.open(&info_base)); + const char *tmp_append_dir = "test_util_list_files"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_append_dir, ts)); + + { + std::set files; + ObStorageAppender appender; + const char *write_content = "012345678"; + auto write_group_files = [&](const int64_t file_num, const char *prefix) { + for (int64_t file_idx = 0; file_idx < file_num; file_idx++) { + std::string file_name; + // normal file + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/%s%ld-normal", + dir_uri, prefix, file_idx)); + files.emplace(std::string(uri + strlen(dir_uri) + 1)); + ASSERT_EQ(OB_SUCCESS, util.write_single_file(uri, write_content, strlen(write_content))); + + // appendable_file + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/%s%ld-appendable", + dir_uri, prefix, file_idx)); + files.emplace(std::string(uri + strlen(dir_uri) + 1)); + ASSERT_EQ(OB_SUCCESS, appender.open(uri, &info_base)); + ASSERT_EQ(OB_SUCCESS, appender.pwrite(write_content, strlen(write_content), 0)); + ASSERT_EQ(OB_SUCCESS, appender.seal_for_adaptive()); + ASSERT_EQ(OB_SUCCESS, appender.close()); + + // appendable_file with suffix + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/%s%ld-appendable.back", + dir_uri, prefix, file_idx)); + files.emplace(std::string(uri + strlen(dir_uri) + 1)); + ASSERT_EQ(OB_SUCCESS, appender.open(uri, &info_base)); + ASSERT_EQ(OB_SUCCESS, appender.pwrite(write_content, strlen(write_content), 0)); + ASSERT_EQ(OB_SUCCESS, appender.seal_for_adaptive()); + ASSERT_EQ(OB_SUCCESS, appender.close()); + } + }; + + // 文件层级 + // 第一级：0-normal, 0-appendable, 0-appendable.back .... 9-normal, 9-appendable, 9-appendable.back + // 第二级：0/0-normal, 0/0-appendable, 0/0-appendable.back, ... 9/9-normal, 9/9-appendable, 9/9-appendable.back + // 第三级：0/0/0-normal, 0/0/0-appendable, 0/0/0-appendable.back, ... 9/4/4-normal, 9/4/4-appendable, 9/4/4-appendable.back、 + write_group_files(10, ""); + std::string prefix; + for (int64_t i = 0; i < 10; i++) { + prefix = std::to_string(i) + "/"; + write_group_files(10, prefix.c_str()); + } + for (int64_t i = 0; i < 10; i++) { + for (int64_t j = 0; j < 5; j++) { + prefix = std::to_string(i) + "/" + std::to_string(j) + "/"; + write_group_files(5, prefix.c_str()); + } + } + + // list and check and clean + TestObjectStorageListOp op(strlen(write_content)); + ASSERT_EQ(OB_SUCCESS, util.list_files(dir_uri, true/*is_adaptive*/, op)); + ASSERT_EQ(files, op.object_names_); + + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/", dir_uri)); + op.object_names_.clear(); + ASSERT_EQ(OB_SUCCESS, util.list_files(uri, true/*is_adaptive*/, op)); + ASSERT_EQ(files, op.object_names_); + + for (auto &file : files) { + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/%s", dir_uri, file.c_str())); + ASSERT_EQ(OB_SUCCESS, util.del_file(uri, true)); + } + } + } +} + +void test_read_appendable_object(const char *content, const int64_t content_length, + const int64_t n_run, const int64_t *read_start, const int64_t *read_end, + ObStorageAdaptiveReader &reader) +{ + int64_t read_size = -1; + int64_t expected_read_size = -2; + char buf[content_length]; + for (int64_t i = 0; i < n_run; i++) { + read_size = -1; + expected_read_size = MIN(read_end[i], content_length) - MIN(read_start[i], content_length); + ASSERT_TRUE(read_start[i] <= read_end[i]); + ASSERT_EQ(OB_SUCCESS, + reader.pread(buf, read_end[i] - read_start[i], read_start[i], read_size)); + ASSERT_EQ(expected_read_size, read_size); + ASSERT_EQ(0, memcmp(buf, content + read_start[i], read_size)); + } +} + +TEST_F(TestObjectStorage, test_append_rw) +{ + int ret = OB_SUCCESS; + if (enable_test) { + ObStorageUtil util; + ASSERT_EQ(OB_SUCCESS, util.open(&info_base)); + const char *tmp_append_dir = "test_append"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_append_dir, ts)); + + { + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/test_append_file_%ld.back", + dir_uri, ObTimeUtility::current_time())); + ObStorageAppender appender; + ASSERT_EQ(OB_SUCCESS, appender.open(uri, &info_base)); + ASSERT_EQ(OB_INVALID_ARGUMENT, appender.pwrite(NULL, 10, 0)); + ASSERT_EQ(OB_INVALID_ARGUMENT, appender.pwrite("1", 0, 0)); + ASSERT_EQ(OB_INVALID_ARGUMENT, appender.pwrite("1", 1, -1)); + + ObStorageAdaptiveReader reader; + ASSERT_EQ(OB_BACKUP_FILE_NOT_EXIST, reader.open(uri, &info_base)); + } + + { + ObStorageAppender appender; + ASSERT_EQ(OB_SUCCESS, appender.open(uri, &info_base)); + // first append + const char first_write[] = "123"; + ASSERT_EQ(OB_SUCCESS, appender.pwrite(first_write, strlen(first_write), 0)); + ASSERT_EQ(strlen(first_write), appender.get_length()); + + // second append + const char second_write[] = "4567"; + ASSERT_EQ(OB_SUCCESS, appender.pwrite(second_write, strlen(second_write), strlen(first_write))); + ASSERT_EQ(strlen(first_write) + strlen(second_write), appender.get_length()); + + // check size + int64_t file_length = 0; + ASSERT_EQ(OB_SUCCESS, util.get_file_length(uri, true/*is_adaptive*/, file_length)); + ASSERT_EQ(strlen(first_write) + strlen(second_write), file_length); + + // check data and clean + ObStorageAdaptiveReader reader; + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &info_base)); + ASSERT_EQ(strlen(first_write) + strlen(second_write), reader.get_length()); + char read_buf[5] = {0}; + int64_t read_size = 0; + ASSERT_EQ(OB_SUCCESS, reader.pread(read_buf, 5, 2, read_size)); + ASSERT_EQ('3', read_buf[0]); + ASSERT_EQ('7', read_buf[4]); + ASSERT_EQ(5, read_size); + ASSERT_EQ(OB_SUCCESS, reader.close()); + ASSERT_EQ(OB_SUCCESS, appender.seal_for_adaptive()); + ASSERT_EQ(OB_SUCCESS, appender.close()); + ASSERT_EQ(OB_SUCCESS, util.del_file(uri, true)); + ASSERT_FALSE(appender.is_opened()); + } + + if (info_base.get_type() == ObStorageType::OB_STORAGE_S3) { + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/test_append_file_%ld.back", + dir_uri, ObTimeUtility::current_time())); + + ObStorageAppender appender; + ASSERT_EQ(OB_SUCCESS, appender.open(uri, &info_base)); + + const int64_t content_length = 100; + char content[content_length] = { 0 }; + for (int64_t i = 0; i < content_length; i++) { + content[i] = '0' + (i % 10); + } + // "1-7", "2-5", "3-6", "4-7", "1-7", "1-7", "1-5", // covered by "1-7" + // "0-3", "0-3", "0-1", "1-2", "2-3", // covered "0-3" + // "7-8", "8-9", "9-10", "10-11", "11-12", "12-20", // no gap + // "22-25" & "21-24" are covered by "15-25", and there is a gap from "15-25" to "26-30" + // "15-25", "22-25", "21-24", "26-30", "30-100", "28-100" + int64_t fragment_start[] = {1,2,3,4,1,1,1,0,0,0,1,2,7,8,9,10,11,12,15,22,21,26,30,28}; + int64_t fragment_end[] = {7,5,6,7,7,7,5,3,3,1,2,3,8,9,10,11,12,20,25,25,24,30,100,100}; + ASSERT_EQ(sizeof(fragment_start), sizeof(fragment_end)); + for (int64_t i = 0; i < sizeof(fragment_start) / sizeof(int64_t); i++) { + ASSERT_TRUE(fragment_start[i] < fragment_end[i]); + ASSERT_EQ(OB_SUCCESS, appender.pwrite(content + fragment_start[i], + fragment_end[i] - fragment_start[i], fragment_start[i])); + } + ASSERT_EQ(content_length, appender.get_length()); + + // read before close + ObStorageAdaptiveReader reader; + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &info_base)); + ASSERT_EQ(content_length, reader.get_length()); + int64_t read_start[] = {0, 1, 0, 4, 26, 30, 26}; + int64_t read_end[] = {3, 4, 15, 25, 27, 100, 101}; + ASSERT_EQ(sizeof(read_start), sizeof(read_end)); + test_read_appendable_object(content, content_length, + sizeof(read_start) / sizeof(int64_t), read_start, read_end, reader); + + char buf[content_length]; + int64_t read_size = -1; + ASSERT_EQ(OB_ERR_UNEXPECTED, + reader.pread(buf, content_length, 0, read_size)); + ASSERT_EQ(content_length, reader.get_length()); + { + ASSERT_EQ(OB_INVALID_ARGUMENT, reader.pread(buf, 0, 0, read_size)); + ASSERT_EQ(OB_INVALID_ARGUMENT, reader.pread(buf, 1, -1, read_size)); + ASSERT_EQ(OB_INVALID_ARGUMENT, reader.pread(NULL, 1, 0, read_size)); + } + + OB_LOG(INFO, "-=-=-=-=-===========-=-=====-=-=-=-=-=-=-=-=-=-=-=-="); + ASSERT_EQ(OB_SUCCESS, appender.close()); + // open before close, read after close + test_read_appendable_object(content, content_length, + sizeof(read_start) / sizeof(int64_t), read_start, read_end, reader); + ASSERT_EQ(content_length, reader.get_length()); + ASSERT_EQ(OB_SUCCESS, reader.close()); + OB_LOG(INFO, "-=-=-=-=-===========-=-=====-=-=-=-=-=-=-=-=-=-=-=-="); + + // open/read after close + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &info_base)); + test_read_appendable_object(content, content_length, + sizeof(read_start) / sizeof(int64_t), read_start, read_end, reader); + ASSERT_EQ(content_length, reader.get_length()); + OB_LOG(INFO, "-=-=-=-=-===========-=-=====-=-=-=-=-=-=-=-=-=-=-=-="); + + // read after seal + ASSERT_EQ(OB_SUCCESS, appender.open(uri, &info_base)); + ASSERT_EQ(OB_SUCCESS, appender.seal_for_adaptive()); + test_read_appendable_object(content, content_length, + sizeof(read_start) / sizeof(int64_t), read_start, read_end, reader); + ASSERT_EQ(content_length, reader.get_length()); + ASSERT_EQ(OB_SUCCESS, reader.close()); + ASSERT_EQ(OB_SUCCESS, appender.close()); + OB_LOG(INFO, "-=-=-=-=-===========-=-=====-=-=-=-=-=-=-=-=-=-=-=-="); + + // open/read after seal + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &info_base)); + test_read_appendable_object(content, content_length, + sizeof(read_start) / sizeof(int64_t), read_start, read_end, reader); + ASSERT_EQ(content_length, reader.get_length()); + ASSERT_EQ(OB_SUCCESS, reader.close()); + + ASSERT_EQ(OB_SUCCESS, util.del_file(uri, true)); + } + + if (info_base.get_type() == ObStorageType::OB_STORAGE_S3) { + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/test_append_file_%ld.back", + dir_uri, ObTimeUtility::current_time())); + + ObStorageAppender appender; + ASSERT_EQ(OB_SUCCESS, appender.open(uri, &info_base)); + + const int64_t content_length = 100; + char content[content_length] = { 0 }; + for (int64_t i = 0; i < content_length; i++) { + content[i] = '0' + (i % 10); + } + // "1-7", "2-5", "3-6", "4-7", "1-7", "1-7", "1-5", // covered by "1-7" + // "0-3", "0-3", "0-1", "1-2", "2-3", // covered "0-3" + // "7-8", "8-9", "9-10", "10-11", "11-12", "12-20", // no gap + // "22-25" & "21-24" are covered by "15-25", and there is a gap from "15-25" to "26-30" + // "15-25", "22-25", "21-24", "26-30", "30-100", "28-100" + int64_t fragment_start[] = {1,2,3,4,1,1,1,0,0,0,1,2,7,8,9,10,11,12,15,22,21,26,30,28}; + int64_t fragment_end[] = {7,5,6,7,7,7,5,3,3,1,2,3,8,9,10,11,12,20,25,25,24,30,100,100}; + ASSERT_EQ(sizeof(fragment_start), sizeof(fragment_end)); + for (int64_t i = 0; i < sizeof(fragment_start) / sizeof(int64_t); i++) { + ASSERT_TRUE(fragment_start[i] < fragment_end[i]); + ASSERT_EQ(OB_SUCCESS, appender.pwrite(content + fragment_start[i], + fragment_end[i] - fragment_start[i], fragment_start[i])); + } + ASSERT_EQ(content_length, appender.get_length()); + + // read before close + ObStorageAdaptiveReader reader; + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &info_base)); + ASSERT_EQ(content_length, reader.get_length()); + + char buf[content_length]; + int64_t read_size = -1; + ASSERT_EQ(OB_ERR_UNEXPECTED, + reader.pread(buf, content_length, 0, read_size)); + ASSERT_EQ(OB_SUCCESS, reader.close()); + + // fill gap + ASSERT_EQ(OB_SUCCESS, appender.pwrite(content + 25, 1, 25)); + + // now gap is filled + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &info_base)); + ASSERT_EQ(content_length, reader.get_length()); + ASSERT_EQ(OB_SUCCESS, + reader.pread(buf, content_length, 0, read_size)); + ASSERT_EQ(OB_SUCCESS, reader.close()); + + // test appendable put is valid + // ObStorageUtil s3_util; + // ObStorageObjectMeta appendable_obj_meta; + // ASSERT_EQ(OB_SUCCESS, s3_util.open(&info_base)); + // ASSERT_EQ(OB_SUCCESS, s3_util.list_appendable_file_fragments(uri, appendable_obj_meta)); + // ASSERT_EQ(1, appendable_obj_meta.fragment_metas_.count()); + // ASSERT_EQ(0, appendable_obj_meta.fragment_metas_[0].start_); + // ASSERT_EQ(9223372036854775807, appendable_obj_meta.fragment_metas_[0].end_); + // ASSERT_EQ(content_length, appendable_obj_meta.length_); + + ASSERT_EQ(OB_SUCCESS, appender.close()); + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &info_base)); + // open before close, read after close + int64_t read_start[] = {0, 1, 0, 4, 26, 30, 26}; + int64_t read_end[] = {3, 4, 15, 25, 27, 100, 101}; + test_read_appendable_object(content, content_length, + sizeof(read_start) / sizeof(int64_t), read_start, read_end, reader); + ASSERT_EQ(content_length, reader.get_length()); + ASSERT_EQ(OB_SUCCESS, reader.close()); + + // open/read after close + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &info_base)); + // ASSERT_EQ(content_length - 1, reader.get_appendable_object_size()); + test_read_appendable_object(content, content_length, + sizeof(read_start) / sizeof(int64_t), read_start, read_end, reader); + ASSERT_EQ(content_length, reader.get_length()); + + // read after seal + ASSERT_EQ(OB_SUCCESS, appender.open(uri, &info_base)); + ASSERT_EQ(OB_SUCCESS, appender.seal_for_adaptive()); + test_read_appendable_object(content, content_length, + sizeof(read_start) / sizeof(int64_t), read_start, read_end, reader); + ASSERT_EQ(content_length, reader.get_length()); + ASSERT_EQ(OB_SUCCESS, reader.close()); + ASSERT_EQ(OB_SUCCESS, appender.close()); + + // open/read after seal + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &info_base)); + // ASSERT_EQ(content_length - 1, reader.get_appendable_object_size()); + test_read_appendable_object(content, content_length, + sizeof(read_start) / sizeof(int64_t), read_start, read_end, reader); + ASSERT_EQ(content_length, reader.get_length()); + ASSERT_EQ(OB_SUCCESS, reader.close()); + + ASSERT_EQ(OB_SUCCESS, util.del_file(uri, true)); + } + + if (info_base.get_type() == ObStorageType::OB_STORAGE_S3) { + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/test_append_file_%ld.back", + dir_uri, ObTimeUtility::current_time())); + + ObStorageAppender appender_a; + ObStorageAppender appender_b; + ObStorageAppender appender_c; + ASSERT_EQ(OB_SUCCESS, appender_a.open(uri, &info_base)); + ASSERT_EQ(OB_SUCCESS, appender_b.open(uri, &info_base)); + ASSERT_EQ(OB_SUCCESS, appender_c.open(uri, &info_base)); + + const int64_t content_length = 100; + char content[content_length] = { 0 }; + for (int64_t i = 0; i < content_length; i++) { + content[i] = '0' + (i % 10); + } + // "1-7", "2-5", "3-6", "4-7", "1-7", "1-7", "1-5", // covered by "1-7" + // "0-3", "0-3", "0-1", "1-2", "2-3", // covered "0-3" + // "7-8", "8-9", "9-10", "10-11", "11-12", "12-20", // no gap + // "22-25" & "21-24" are covered by "15-25" + // "15-25", "22-25", "21-24", "25-30", "30-100", "28-100" + int64_t fragment_start[] = {1,2,3,4,1,1,1,0,0,0,1,2,7,8,9,10,11,12,15,22,21,25,30,28}; + int64_t fragment_end[] = {7,5,6,7,7,7,5,3,3,1,2,3,8,9,10,11,12,20,25,25,24,30,100,100}; + ASSERT_EQ(sizeof(fragment_start), sizeof(fragment_end)); + for (int64_t i = 0; i < sizeof(fragment_start) / sizeof(int64_t); i++) { + ASSERT_TRUE(fragment_start[i] < fragment_end[i]); + if (i % 3 == 0) { + ASSERT_EQ(OB_SUCCESS, appender_a.pwrite(content + fragment_start[i], + fragment_end[i] - fragment_start[i], fragment_start[i])); + } else if (i % 3 == 1) { + ASSERT_EQ(OB_SUCCESS, appender_b.pwrite(content + fragment_start[i], + fragment_end[i] - fragment_start[i], fragment_start[i])); + } else { + ASSERT_EQ(OB_SUCCESS, appender_c.pwrite(content + fragment_start[i], + fragment_end[i] - fragment_start[i], fragment_start[i])); + } + } + + // read before close + ObStorageAdaptiveReader reader_a; + ObStorageAdaptiveReader reader_b; + ASSERT_EQ(OB_SUCCESS, reader_a.open(uri, &info_base)); + ASSERT_EQ(OB_SUCCESS, reader_b.open(uri, &info_base)); + int64_t read_start[] = {0, 1, 0, 4, 26, 30, 26, 0}; + int64_t read_end[] = {3, 4, 15, 25, 27, 100, 101, 100}; + ASSERT_EQ(sizeof(read_start), sizeof(read_end)); + int64_t n_run = sizeof(read_start) / sizeof(int64_t); + std::thread read_thread_a(test_read_appendable_object, content, content_length, + n_run, read_start, read_end, std::ref(reader_a)); + std::thread read_thread_b(test_read_appendable_object, content, content_length, + n_run, read_start, read_end, std::ref(reader_b)); + read_thread_a.join(); + read_thread_b.join(); + ASSERT_EQ(content_length, reader_a.get_length()); + ASSERT_EQ(content_length, reader_b.get_length()); + + // open before close, read after close + ASSERT_EQ(OB_SUCCESS, appender_a.close()); + ASSERT_EQ(OB_SUCCESS, appender_b.close()); + ASSERT_EQ(OB_SUCCESS, appender_c.close()); + std::thread read_thread_c(test_read_appendable_object, content, content_length, + n_run, read_start, read_end, std::ref(reader_a)); + std::thread read_thread_d(test_read_appendable_object, content, content_length, + n_run, read_start, read_end, std::ref(reader_b)); + read_thread_c.join(); + read_thread_d.join(); + ASSERT_EQ(content_length, reader_a.get_length()); + ASSERT_EQ(content_length, reader_b.get_length()); + ASSERT_EQ(OB_SUCCESS, reader_a.close()); + ASSERT_EQ(OB_SUCCESS, reader_b.close()); + + // open/read after close + ASSERT_EQ(OB_SUCCESS, reader_a.open(uri, &info_base)); + ASSERT_EQ(OB_SUCCESS, reader_b.open(uri, &info_base)); + std::thread read_thread_e(test_read_appendable_object, content, content_length, + n_run, read_start, read_end, std::ref(reader_a)); + std::thread read_thread_f(test_read_appendable_object, content, content_length, + n_run, read_start, read_end, std::ref(reader_b)); + read_thread_e.join(); + read_thread_f.join(); + ASSERT_EQ(content_length, reader_a.get_length()); + ASSERT_EQ(content_length, reader_b.get_length()); + + // read after seal + ASSERT_EQ(OB_SUCCESS, appender_a.open(uri, &info_base)); + ASSERT_EQ(OB_SUCCESS, appender_a.seal_for_adaptive()); + std::thread read_thread_g(test_read_appendable_object, content, content_length, + n_run, read_start, read_end, std::ref(reader_a)); + std::thread read_thread_h(test_read_appendable_object, content, content_length, + n_run, read_start, read_end, std::ref(reader_b)); + read_thread_g.join(); + read_thread_h.join(); + ASSERT_EQ(content_length, reader_a.get_length()); + ASSERT_EQ(content_length, reader_b.get_length()); + ASSERT_EQ(OB_SUCCESS, reader_a.close()); + ASSERT_EQ(OB_SUCCESS, reader_b.close()); + ASSERT_EQ(OB_SUCCESS, appender_a.close()); + + // open/read after seal + ASSERT_EQ(OB_SUCCESS, reader_a.open(uri, &info_base)); + ASSERT_EQ(OB_SUCCESS, reader_b.open(uri, &info_base)); + std::thread read_thread_i(test_read_appendable_object, content, content_length, + n_run, read_start, read_end, std::ref(reader_a)); + std::thread read_thread_j(test_read_appendable_object, content, content_length, + n_run, read_start, read_end, std::ref(reader_b)); + read_thread_i.join(); + read_thread_j.join(); + ASSERT_EQ(content_length, reader_a.get_length()); + ASSERT_EQ(content_length, reader_b.get_length()); + ASSERT_EQ(OB_SUCCESS, reader_a.close()); + ASSERT_EQ(OB_SUCCESS, reader_b.close()); + + ASSERT_EQ(OB_SUCCESS, util.del_file(uri, true)); + } + } +} + +TEST_F(TestObjectStorage, test_cross_testing) +{ + char * a = NULL; + printf("testtttt %s\n", a); + int ret = OB_SUCCESS; + ObObjectStorageInfo cross_info; + if (enable_test) { + set_storage_info(bucket, endpoint, + secretid, secretkey, appid, region, cross_info); + ObStorageUtil util; + ASSERT_EQ(OB_SUCCESS, util.open(&cross_info)); + + // constuct obj name, should be compatible with nfs + const char *tmp_dir = "test_cross_appendable"; + const int64_t ts = ObTimeUtility::current_time(); + if (cross_info.get_type() == ObStorageType::OB_STORAGE_FILE) { + ASSERT_EQ(OB_SUCCESS, + databuff_printf(uri, sizeof(uri), "file://%s/%s_%ld.back", get_current_dir_name(), tmp_dir, ts)); + ASSERT_EQ(OB_SUCCESS, util.mkdir(uri)); + } else { + ASSERT_EQ(OB_SUCCESS, + databuff_printf(uri, sizeof(uri), "%s/%s/%s_%ld", bucket, dir_name, tmp_dir, ts)); + } + + // simulate append + bool is_obj_exist = true; + ASSERT_EQ(OB_SUCCESS, util.is_exist(uri, true/*is_adaptive*/, is_obj_exist)); + ASSERT_FALSE(is_obj_exist); + + // write format + int64_t file_length = -1; + char fragment_name[OB_MAX_URI_LENGTH] = { 0 }; + ASSERT_EQ(OB_SUCCESS, + construct_fragment_full_name(uri, OB_S3_APPENDABLE_FORMAT_META, fragment_name, sizeof(fragment_name))); + ASSERT_EQ(OB_SUCCESS, util.write_single_file(fragment_name, OB_S3_APPENDABLE_FORMAT_CONTENT_V1, + strlen(OB_S3_APPENDABLE_FORMAT_CONTENT_V1))); + ASSERT_EQ(OB_SUCCESS, util.is_exist(uri, true/*is_adaptive*/, is_obj_exist)); + ASSERT_TRUE(is_obj_exist); + ASSERT_EQ(OB_SUCCESS, util.get_file_length(uri, true/*is_adaptive*/, file_length)); + ASSERT_EQ(0, file_length); + + // put fragments + // "1-7", "2-5", "3-6", "4-7", "1-7", "1-7", "1-5", // covered by "1-7" + // "0-3", "0-3", "0-1", "1-2", "2-3", // covered "0-3" + // "7-8", "8-9", "9-10", "10-11", "11-12", "12-20", // no gap + // "22-25" & "21-24" are covered by "15-25" + // "15-25", "22-25", "21-24", "25-30", "30-100", "28-100" + int64_t fragment_start[] = {1,2,3,4,1,1,1,0,0,0,1,2,7,8,9,10,11,12,15,22,21,25,30,28}; + int64_t fragment_end[] = {7,5,6,7,7,7,5,3,3,1,2,3,8,9,10,11,12,20,25,25,24,30,100,100}; + const int64_t content_length = 100; + char content[content_length + 5] = { 0 }; + for (int64_t i = 0; i < content_length + 5; i++) { + content[i] = '0' + (i % 10); + } + ASSERT_EQ(sizeof(fragment_start), sizeof(fragment_end)); + for (int64_t i = 0; i < sizeof(fragment_start) / sizeof(int64_t); i++) { + ASSERT_TRUE(fragment_start[i] < fragment_end[i]); + ASSERT_EQ(OB_SUCCESS, + construct_fragment_full_name(uri, fragment_start[i], fragment_end[i], fragment_name, sizeof(fragment_name))); + ASSERT_EQ(OB_SUCCESS, util.write_single_file(fragment_name, content + fragment_start[i], + fragment_end[i] - fragment_start[i])); + } + ASSERT_EQ(OB_SUCCESS, util.get_file_length(uri, true/*is_adaptive*/, file_length)); + ASSERT_EQ(content_length, file_length); + + // open before close, read after close + ObStorageAdaptiveReader reader; + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &cross_info)); + int64_t read_start[] = {0, 1, 0, 4, 26, 30, 26}; + int64_t read_end[] = {3, 4, 15, 25, 27, 100, 101}; + test_read_appendable_object(content, content_length, + sizeof(read_start) / sizeof(int64_t), read_start, read_end, reader); + ASSERT_EQ(content_length, reader.get_length()); + ASSERT_EQ(OB_SUCCESS, reader.close()); + + // append a fragment and seal + ASSERT_EQ(OB_SUCCESS, + construct_fragment_full_name(uri, 100, 105, fragment_name, sizeof(fragment_name))); + ASSERT_EQ(OB_SUCCESS, util.write_single_file(fragment_name, content, 5)); + char *buf = NULL; + char seal_meta_uri[OB_MAX_URI_LENGTH] = { 0 }; + ListAppendableObjectFragmentOp op; + ObStorageObjectMeta appendable_obj_meta; + ObArenaAllocator allocator; + int64_t pos = 0; // for serializing appendable_obj_meta + int64_t serialize_size = -1; // for serializing appendable_obj_meta + ASSERT_EQ(OB_SUCCESS, util.list_appendable_file_fragments(uri, appendable_obj_meta)); + serialize_size = appendable_obj_meta.get_serialize_size() + 1; + buf = static_cast(allocator.alloc(serialize_size)); + ASSERT_TRUE(OB_NOT_NULL(buf)); + ASSERT_EQ(OB_SUCCESS, appendable_obj_meta.serialize(buf, serialize_size, pos)); + ASSERT_EQ(OB_SUCCESS, construct_fragment_full_name(uri, OB_S3_APPENDABLE_SEAL_META, + seal_meta_uri, sizeof(seal_meta_uri))); + ASSERT_EQ(OB_SUCCESS, util.write_single_file(seal_meta_uri, buf, pos)); + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &cross_info)); + ASSERT_EQ(105, reader.get_length()); + test_read_appendable_object(content, 105, + sizeof(read_start) / sizeof(int64_t), read_start, read_end, reader); + ASSERT_EQ(OB_SUCCESS, reader.close()); + + if (cross_info.get_type() != ObStorageType::OB_STORAGE_FILE) { + ASSERT_EQ(OB_SUCCESS, util.del_file(uri, true)); + } + } +} + +TEST_F(TestObjectStorage, test_multipart_write) +{ + int ret = OB_SUCCESS; + if (enable_test) { + ObStorageUtil util; + ASSERT_EQ(OB_SUCCESS, util.open(&info_base)); + const char *tmp_dir = "test_multipart_write_files"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_dir, ts)); + + const int64_t content_size = 20 * 1024 * 1024L + 7; // 20M + 7B + ObArenaAllocator allocator; + char *write_buf = (char *)allocator.alloc(content_size); + static const char alphanum[] = + "0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; + for (int64_t i = 0; i < content_size - 1; i++) { + write_buf[i] = alphanum[ObRandom::rand(0, sizeof(alphanum) - 2)]; + } + write_buf[content_size - 1] = '\0'; + + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/test_multipart", dir_uri)); + ObStorageMultiPartWriter writer; + ASSERT_EQ(OB_SUCCESS, writer.open(uri, &info_base)); + ASSERT_EQ(OB_SUCCESS, writer.write(write_buf, content_size)); + ASSERT_EQ(content_size, writer.get_length()); + ASSERT_EQ(OB_SUCCESS, writer.close()); + + ObStorageReader reader; + int64_t read_size = -1; + const int64_t read_start = 1024; + const int64_t read_buf_size = 1024; + char read_buf[read_buf_size]; + memset(read_buf, 0, sizeof(read_buf)); + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &info_base)); + ASSERT_EQ(content_size, reader.get_length()); + ASSERT_EQ(OB_SUCCESS, reader.pread(read_buf, read_buf_size, read_start, read_size)); + ASSERT_EQ(read_buf_size, read_size); + ASSERT_EQ(0, memcmp(write_buf + read_start, read_buf, read_size)); + ASSERT_EQ(OB_SUCCESS, reader.close()); + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + } +} + +TEST_F(TestObjectStorage, test_del_unmerged_parts) +{ + int ret = OB_SUCCESS; + if (enable_test && false) { + ObStorageUtil util; + ASSERT_EQ(OB_SUCCESS, util.open(&info_base)); + const char *tmp_dir = "test_del_unmerged_parts"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_dir, ts)); + + const int64_t content_size = 20 * 1024 * 1024L + 7; // 20M + 7B + ObArenaAllocator allocator; + char *write_buf = (char *)allocator.alloc(content_size); + static const char alphanum[] = + "0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; + for (int64_t i = 0; i < content_size - 1; i++) { + write_buf[i] = alphanum[ObRandom::rand(0, sizeof(alphanum) - 2)]; + } + write_buf[content_size - 1] = '\0'; + + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/%s", dir_uri, tmp_dir)); + ObStorageMultiPartWriter writer; + ASSERT_EQ(OB_SUCCESS, writer.open(uri, &info_base)); + ASSERT_EQ(OB_SUCCESS, writer.write(write_buf, content_size)); + ASSERT_EQ(content_size, writer.get_length()); + + ASSERT_EQ(OB_SUCCESS, util.del_unmerged_parts(uri)); + if (info_base.get_type() == ObStorageType::OB_STORAGE_OSS) { + ASSERT_EQ(OB_OSS_ERROR, writer.close()); + } else if (info_base.get_type() == ObStorageType::OB_STORAGE_S3) { + ASSERT_EQ(OB_S3_ERROR, writer.close()); + } else if (info_base.get_type() == ObStorageType::OB_STORAGE_COS) { + ASSERT_EQ(OB_COS_ERROR, writer.close()); + } + + } +} + +int main(int argc, char **argv) +{ + system("rm -f test_object_storage.log*"); + OB_LOGGER.set_file_name("test_object_storage.log", true, true); + OB_LOGGER.set_log_level("DEBUG"); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/deps/oblib/unittest/lib/restore/test_object_storage.h b/deps/oblib/unittest/lib/restore/test_object_storage.h new file mode 100644 index 000000000..9c9fce66a --- /dev/null +++ b/deps/oblib/unittest/lib/restore/test_object_storage.h @@ -0,0 +1,24 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef TEST_OBJECT_STORAGE_H_ +#define TEST_OBJECT_STORAGE_H_ + +const bool enable_test = false; +const char *bucket = "xxx"; +const char *region = "xxx"; +const char *endpoint = "xxx"; +const char *secretid = "xxx"; +const char *secretkey = "xxx"; +const char *appid = "xxx"; + +#endif \ No newline at end of file diff --git a/deps/oblib/unittest/lib/restore/test_storage_cos.cpp b/deps/oblib/unittest/lib/restore/test_storage_cos.cpp index 60d169f92..77a047254 100644 --- a/deps/oblib/unittest/lib/restore/test_storage_cos.cpp +++ b/deps/oblib/unittest/lib/restore/test_storage_cos.cpp @@ -671,6 +671,44 @@ TEST_F(TestStorageCos, test_util_is_tagging) } } +TEST_F(TestStorageCos, test_multipartupload) +{ + int ret = OB_SUCCESS; + if (enable_test_) { + ObStorageCosMultiPartWriter multi_upload; + ObStorageUtil util; + ASSERT_EQ(OB_SUCCESS, util.open(&cos_base)); + + const char *tmp_multi_dir = "test_multipartupload"; + const int64_t content_size = 20 * 1024 * 1024L;//20MB + char *content = new char[content_size]; + memset(content, 'a', content_size); + + // operate before open + ASSERT_EQ(OB_COS_ERROR, multi_upload.write(content, content_size)); + + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_multi_dir, ts)); + + // operate correctly + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/multipartupload", dir_uri)); + ASSERT_EQ(OB_SUCCESS, multi_upload.open(uri, &cos_base)); + ASSERT_EQ(true, multi_upload.is_opened()); + ASSERT_EQ(OB_SUCCESS, multi_upload.write(content, content_size)); + ASSERT_EQ(OB_SUCCESS, multi_upload.close()); + + // check multipartupload valid + ASSERT_EQ(content_size, multi_upload.get_length()); + bool is_exist = false; + ASSERT_EQ(OB_SUCCESS, util.is_exist(uri, is_exist)); + ASSERT_TRUE(is_exist); + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + + delete[] content; + } +} + int main(int argc, char **argv) { system("rm -f test_storage_cos.log*"); diff --git a/deps/oblib/unittest/lib/restore/test_storage_info.cpp b/deps/oblib/unittest/lib/restore/test_storage_info.cpp index 6ddbf50bb..764b49a6d 100644 --- a/deps/oblib/unittest/lib/restore/test_storage_info.cpp +++ b/deps/oblib/unittest/lib/restore/test_storage_info.cpp @@ -104,6 +104,31 @@ TEST(ObObjectStorageInfo, cos) ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info)); } +TEST(ObObjectStorageInfo, s3) +{ + const char *uri = "s3://backup_dir?host=xxx.com&access_id=111&access_key=222&s3_region=333"; + ObObjectStorageInfo info1; + + const char *storage_info = ""; + ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info)); + storage_info = "host=xxx.com&access_id=111&access_key=222"; + ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info)); + + storage_info = "host=xxx.com&access_id=111&access_key=222&s3_region=333"; + ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info)); + ASSERT_EQ(0, ::strcmp("s3_region=333", info1.extension_)); + + char buf[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 }; + ASSERT_EQ(OB_SUCCESS, info1.get_storage_info_str(buf, sizeof(buf))); + ASSERT_STREQ(storage_info, buf); + + storage_info = "host=xxx.com&access_id=111&access_key=222&s3_region=333&delete_mode=delete"; + info1.reset(); + ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info)); + ASSERT_EQ(OB_SUCCESS, info1.get_storage_info_str(buf, sizeof(buf))); + ASSERT_STREQ(storage_info, buf); +} + int main(int argc, char **argv) { system("rm -f test_storage_info.log*"); diff --git a/deps/oblib/unittest/lib/restore/test_storage_s3.cpp b/deps/oblib/unittest/lib/restore/test_storage_s3.cpp new file mode 100644 index 000000000..1d3c03e57 --- /dev/null +++ b/deps/oblib/unittest/lib/restore/test_storage_s3.cpp @@ -0,0 +1,1108 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#include "lib/utility/ob_test_util.h" +#include "lib/restore/ob_storage.h" +#include "lib/restore/ob_storage_s3_base.h" +#include "lib/allocator/page_arena.h" +#include "test_storage_s3.h" +#include "lib/container/ob_array_serialization.h" +#include + +#include + +using namespace oceanbase::common; + +class TestStorageS3Common { +public: +TestStorageS3Common() {} +~TestStorageS3Common() {} + +void init() +{ + ASSERT_EQ(OB_SUCCESS, + databuff_printf(account, sizeof(account), + "s3_region=%s&host=%s&access_id=%s&access_key=%s", + region, endpoint, secretid, secretkey)); + //build s3_base + const ObString s3_storage_info(account); + ASSERT_EQ(OB_SUCCESS, s3_base.set(ObStorageType::OB_STORAGE_S3, s3_storage_info.ptr())); +} +void destory() +{ +} +protected: + char account[OB_MAX_URI_LENGTH]; + const char *dir_name = "s3_unittest_dir"; + char uri[OB_MAX_URI_LENGTH]; + char dir_uri[OB_MAX_URI_LENGTH]; + ObObjectStorageInfo s3_base; + + int object_prefix_len = 5; +}; + +class TestStorageS3: public ::testing::Test, public TestStorageS3Common +{ +public: + TestStorageS3() : enable_test_(enable_test) {} + virtual ~TestStorageS3(){} + virtual void SetUp() + { + init(); + } + virtual void TearDown() + { + destory(); + } + + static void SetUpTestCase() + { + init_s3_env(); + } + + static void TearDownTestCase() + { + fin_s3_env(); + } + +private: + // disallow copy + DISALLOW_COPY_AND_ASSIGN(TestStorageS3); +protected: + bool enable_test_; +}; + +#define WRITE_SINGLE_FILE(file_suffix, file_content) \ + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%stest_write_file_%ld.back", dir_uri, file_suffix)); \ + ASSERT_EQ(OB_SUCCESS, writer.open(uri, &s3_base)); \ + const char write_content[] = file_content; \ + ASSERT_EQ(OB_SUCCESS, writer.write(write_content, strlen(write_content))); \ + ASSERT_EQ(OB_SUCCESS, writer.close()); \ + +TEST_F(TestStorageS3, test_basic_rw) +{ + std::vector arr{1,2,3,4}; + auto it = std::upper_bound(arr.begin(), arr.end(), 10); + std::cout << it - arr.begin() << std::endl; + + int ret = OB_SUCCESS; + if (enable_test_) { + ObStorageWriter writer; + ObStorageReader reader; + ObStorageUtil util; + ASSERT_EQ(OB_SUCCESS, util.open(&s3_base)); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/", bucket, dir_name)); + + // operate before open + char tmp_buf[10] = {0}; + int64_t tmp_read_size = 0; + ASSERT_EQ(OB_NOT_INIT, writer.write(tmp_buf, sizeof(tmp_buf))); + ASSERT_EQ(OB_NOT_INIT, writer.close()); + ASSERT_EQ(OB_NOT_INIT, reader.pread(tmp_buf, sizeof(tmp_buf), 0, tmp_read_size)); + ASSERT_EQ(OB_NOT_INIT, reader.close()); + + // wrong uri + uri[0] = '\0'; + ASSERT_EQ(OB_INVALID_BACKUP_DEST, writer.open(uri, NULL)); + ASSERT_EQ(OB_NOT_INIT, writer.write(tmp_buf, sizeof(tmp_buf))); + ASSERT_EQ(OB_NOT_INIT, writer.close()); + ASSERT_EQ(OB_INVALID_BACKUP_DEST, reader.open(uri, NULL)); + ASSERT_EQ(OB_NOT_INIT, reader.pread(tmp_buf, sizeof(tmp_buf), 0, tmp_read_size)); + ASSERT_EQ(OB_NOT_INIT, reader.close()); + + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%stest_write_file_////", dir_uri)); + ASSERT_EQ(OB_INVALID_ARGUMENT, writer.open(uri, &s3_base)); + ASSERT_EQ(OB_NOT_INIT, writer.write(tmp_buf, sizeof(tmp_buf))); + ASSERT_EQ(OB_NOT_INIT, writer.close()); + ASSERT_EQ(OB_INVALID_ARGUMENT, reader.open(uri, &s3_base)); + ASSERT_EQ(OB_NOT_INIT, reader.pread(tmp_buf, sizeof(tmp_buf), 0, tmp_read_size)); + ASSERT_EQ(OB_NOT_INIT, reader.close()); + + const int64_t tmp_ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%stest_write_file_%ld", dir_uri, tmp_ts)); + ASSERT_EQ(OB_SUCCESS, writer.open(uri, &s3_base)); + ASSERT_EQ(OB_SUCCESS, writer.close()); + + // invalid storage info + ASSERT_EQ(OB_INVALID_ARGUMENT, writer.open(uri, NULL)); + ASSERT_EQ(OB_NOT_INIT, writer.write(tmp_buf, sizeof(tmp_buf))); + ASSERT_EQ(OB_NOT_INIT, writer.close()); + ASSERT_EQ(OB_INVALID_ARGUMENT, reader.open(uri, NULL)); + ASSERT_EQ(OB_NOT_INIT, reader.pread(tmp_buf, sizeof(tmp_buf), 0, tmp_read_size)); + ASSERT_EQ(OB_NOT_INIT, reader.close()); + + { + // rw empty object + const int64_t ts = ObTimeUtility::current_time(); + WRITE_SINGLE_FILE(ts, ""); + + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &s3_base)); + ASSERT_EQ(strlen(write_content), reader.get_length()); + ASSERT_EQ(OB_SUCCESS, reader.close()); + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + } + + { + // ObStorageWriter writer; + const int64_t ts = ObTimeUtility::current_time(); + WRITE_SINGLE_FILE(ts, "123456789"); + + // ObStorageReader reader; + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &s3_base)); + ASSERT_EQ(strlen(write_content), reader.get_length()); + char read_buf[7] = {0}; + int64_t read_size = 0; + ASSERT_EQ(OB_SUCCESS, reader.pread(read_buf, 7, 2, read_size)); + ASSERT_EQ('3', read_buf[0]); + ASSERT_EQ('9', read_buf[6]); + ASSERT_EQ(7, read_size); + ASSERT_EQ(OB_SUCCESS, reader.close()); + + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + } + + { + // ObStorageWriter writer; + const int64_t ts = ObTimeUtility::current_time(); + WRITE_SINGLE_FILE(ts, "123456789ABCDEF"); + + // ObStorageReader reader; + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &s3_base)); + ASSERT_EQ(strlen(write_content), reader.get_length()); + char read_buf[10] = {0}; + int64_t read_size = 0; + ASSERT_EQ(OB_SUCCESS, reader.pread(read_buf, 10, 9, read_size)); + ASSERT_EQ('A', read_buf[0]); + ASSERT_EQ('F', read_buf[5]); + ASSERT_EQ(6, read_size); + ASSERT_EQ(OB_SUCCESS, reader.close()); + + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + } + + { + // read not exist object + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%snot_exist_%ld", dir_uri, ts)); + ASSERT_EQ(OB_BACKUP_FILE_NOT_EXIST, reader.open(uri, &s3_base)); + ASSERT_EQ(OB_NOT_INIT, reader.close()); + + // open after fail + WRITE_SINGLE_FILE(ts, "123456789ABCDEF"); + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &s3_base)); + ASSERT_EQ(OB_SUCCESS, reader.close()); + } + + // open twice + ASSERT_EQ(OB_SUCCESS, writer.open(uri, &s3_base)); + ASSERT_EQ(OB_INIT_TWICE, writer.open(uri, &s3_base)); + ASSERT_EQ(OB_NOT_INIT, writer.close()); // reader/writer will be closed if init twice + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &s3_base)); + ASSERT_EQ(OB_INIT_TWICE, reader.open(uri, &s3_base)); + ASSERT_EQ(OB_NOT_INIT, reader.close()); + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + + // operate after close + ASSERT_EQ(OB_NOT_INIT, writer.write(tmp_buf, sizeof(tmp_buf))); + ASSERT_EQ(OB_NOT_INIT, writer.close()); + ASSERT_EQ(OB_NOT_INIT, reader.pread(tmp_buf, sizeof(tmp_buf), 0, tmp_read_size)); + ASSERT_EQ(OB_NOT_INIT, reader.close()); + + util.close(); + } +} + +TEST_F(TestStorageS3, test_util) +{ + int ret = OB_SUCCESS; + if (enable_test_) { + ObStorageUtil util; + const char *tmp_util_dir = "test_util"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld/", + bucket, dir_name, tmp_util_dir, ts)); + ASSERT_EQ(OB_SUCCESS, util.open(&s3_base)); + + ObStorageWriter writer; + { + WRITE_SINGLE_FILE(1L, "123456789ABC"); + + bool is_obj_exist = false; + ASSERT_EQ(OB_SUCCESS, util.is_exist(uri, is_obj_exist)); + ASSERT_TRUE(is_obj_exist); + + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + ASSERT_EQ(OB_SUCCESS, util.is_exist(uri, is_obj_exist)); + ASSERT_FALSE(is_obj_exist); + } + { + WRITE_SINGLE_FILE(2L, "123456789ABCDEF"); + + int64_t file_length = 0; + ASSERT_EQ(OB_SUCCESS, util.get_file_length(uri, file_length)); + ASSERT_EQ(strlen(write_content), file_length); + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + } + + // open twice + ASSERT_EQ(OB_INIT_TWICE, util.open(&s3_base)); + util.close(); + + // invalid storage info + ASSERT_EQ(OB_INVALID_ARGUMENT, util.open(NULL)); + } +} + +TEST_F(TestStorageS3, test_util_is_tagging) +{ + int ret = OB_SUCCESS; + if (enable_test_) { + ObStorageUtil util; + const char *tmp_util_dir = "test_util_is_tagging"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_util_dir, ts)); + + bool is_tagging = true; + char tmp_account[OB_MAX_URI_LENGTH]; + ObObjectStorageInfo tmp_s3_base; + const char *write_content = "123456789ABCDEF"; + + // wrong tag mode + ASSERT_EQ(OB_SUCCESS, + databuff_printf(tmp_account, sizeof(tmp_account), + "s3_region=%s&host=%s&access_id=%s&access_key=%s&delete_mode=tag", + region, endpoint, secretid, secretkey)); + ASSERT_EQ(OB_INVALID_ARGUMENT, tmp_s3_base.set(ObStorageType::OB_STORAGE_S3, tmp_account)); + tmp_s3_base.reset(); + + ASSERT_EQ(OB_SUCCESS, + databuff_printf(tmp_account, sizeof(tmp_account), + "s3_region=%s&host=%s&access_id=%s&access_key=%s&delete_mode=delete_delete", + region, endpoint, secretid, secretkey)); + ASSERT_EQ(OB_INVALID_ARGUMENT, tmp_s3_base.set(ObStorageType::OB_STORAGE_S3, tmp_account)); + tmp_s3_base.reset(); + + // delete mode + ASSERT_EQ(OB_SUCCESS, + databuff_printf(tmp_account, sizeof(tmp_account), + "s3_region=%s&host=%s&access_id=%s&access_key=%s&delete_mode=delete", + region, endpoint, secretid, secretkey)); + ASSERT_EQ(OB_SUCCESS, tmp_s3_base.set(ObStorageType::OB_STORAGE_S3, tmp_account)); + + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/delete_mode", dir_uri)); + ASSERT_EQ(OB_SUCCESS, util.open(&tmp_s3_base)); + ASSERT_EQ(OB_SUCCESS, util.write_single_file(uri, write_content, strlen(write_content))); + + ASSERT_EQ(OB_SUCCESS, util.is_tagging(uri, is_tagging)); + ASSERT_FALSE(is_tagging); + + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + ASSERT_EQ(OB_BACKUP_FILE_NOT_EXIST, util.is_tagging(uri, is_tagging)); + tmp_s3_base.reset(); + util.close(); + + // tagging mode + ASSERT_EQ(OB_SUCCESS, + databuff_printf(tmp_account, sizeof(tmp_account), + "s3_region=%s&host=%s&access_id=%s&access_key=%s&delete_mode=tagging", + region, endpoint, secretid, secretkey)); + ASSERT_EQ(OB_SUCCESS, tmp_s3_base.set(ObStorageType::OB_STORAGE_S3, tmp_account)); + + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/tagging_mode", dir_uri)); + ASSERT_EQ(OB_SUCCESS, util.open(&tmp_s3_base)); + ASSERT_EQ(OB_SUCCESS, util.write_single_file(uri, write_content, strlen(write_content))); + + is_tagging = true; + ASSERT_EQ(OB_SUCCESS, util.is_tagging(uri, is_tagging)); + ASSERT_FALSE(is_tagging); + + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + ASSERT_EQ(OB_SUCCESS, util.is_tagging(uri, is_tagging)); + ASSERT_TRUE(is_tagging); + tmp_s3_base.reset(); + util.close(); + + // clean + ASSERT_EQ(OB_SUCCESS, + databuff_printf(tmp_account, sizeof(tmp_account), + "s3_region=%s&host=%s&access_id=%s&access_key=%s", + region, endpoint, secretid, secretkey)); + ASSERT_EQ(OB_SUCCESS, tmp_s3_base.set(ObStorageType::OB_STORAGE_S3, tmp_account)); + + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/tagging_mode", dir_uri)); + ASSERT_EQ(OB_SUCCESS, util.open(&tmp_s3_base)); + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + ASSERT_EQ(OB_BACKUP_FILE_NOT_EXIST, util.is_tagging(uri, is_tagging)); + util.close(); + } +} + +class TestS3ListOp : public ObBaseDirEntryOperator +{ +public: + TestS3ListOp() {} + virtual ~TestS3ListOp() {} + int func(const dirent *entry) override; + + ObArray object_names_; +private : + ObArenaAllocator allocator_; +}; + +int TestS3ListOp::func(const dirent *entry) +{ + int ret = OB_SUCCESS; + OB_LOG(INFO, "======="); + char *tmp_object_name = (char *)allocator_.alloc(strlen(entry->d_name) + 1); + if (OB_ISNULL(tmp_object_name)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN,"fail to alloc buf for tmp object name", K(ret)); + } else { + STRCPY(tmp_object_name, entry->d_name); + object_names_.push_back(tmp_object_name); + OB_LOG(DEBUG, "list entry", K(tmp_object_name)); + } + return ret; +} + +#define UTIL_WRITE_FILES(format, object_prefix_len, file_num, file_content) \ + for (int64_t file_idx = 0; file_idx < file_num; file_idx++) { \ + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), format, \ + dir_uri, object_prefix_len, file_idx, file_idx)); \ + ASSERT_EQ(OB_SUCCESS, util.write_single_file(uri, file_content, strlen(file_content))); \ + } \ + +#define UTIL_DELETE_FILES(format, object_prefix_len, file_num) \ + for (int64_t file_idx = 0; file_idx < file_num; file_idx++) { \ + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), format, \ + dir_uri, object_prefix_len, file_idx, file_idx)); \ + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); \ + } \ + +TEST_F(TestStorageS3, test_util_list_files) +{ + int ret = OB_SUCCESS; + if (enable_test_) { + ObStorageUtil util; + const char *tmp_util_dir = "test_util_list_files"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_util_dir, ts)); + ASSERT_EQ(OB_SUCCESS, util.open(&s3_base)); + + { + // wrong uri + uri[0] = '\0'; + TestS3ListOp op; + ASSERT_EQ(OB_INVALID_BACKUP_DEST, util.list_files(uri, op)); + } + + int64_t file_num = 11; + const char *write_content = "0123456789"; + + // list objects + { + const char *format = "%s/%0*ld_%ld"; + UTIL_WRITE_FILES(format, object_prefix_len, file_num, write_content); + + // list and check and clean + TestS3ListOp op; + ASSERT_EQ(OB_SUCCESS, util.list_files(dir_uri, op)); + ASSERT_EQ(file_num, op.object_names_.size()); + for (int64_t i = 0; i < file_num; i++) { + // listed files do not contain prefix + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%0*ld_%ld", object_prefix_len, i, i)); + ASSERT_STREQ(uri, op.object_names_[i]); + } + UTIL_DELETE_FILES(format, object_prefix_len, file_num); + } + + // list subfolders' objects + { + const char *format = "%s/%0*ld/%ld"; + UTIL_WRITE_FILES(format, object_prefix_len, file_num, write_content); + + // list and check and clean + TestS3ListOp op; + ASSERT_EQ(OB_SUCCESS, util.list_files(dir_uri, op)); + ASSERT_EQ(file_num, op.object_names_.size()); + for (int64_t i = 0; i < file_num; i++) { + // listed files do not contain prefix + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%0*ld/%ld", object_prefix_len, i, i)); + ASSERT_STREQ(uri, op.object_names_[i]); + } + UTIL_DELETE_FILES(format, object_prefix_len, file_num); + } + + // list empty dir, now dir_uri should be empty after delete + TestS3ListOp list_empty_op; + ASSERT_EQ(OB_SUCCESS, util.list_files(dir_uri, list_empty_op)); + ASSERT_EQ(0, list_empty_op.object_names_.size()); + + util.close(); + } +} + +TEST_F(TestStorageS3, test_util_list_directories) +{ + int ret = OB_SUCCESS; + if (enable_test_) { + ObStorageUtil util; + const char *tmp_util_dir = "test_util_list_directories"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_util_dir, ts)); + ASSERT_EQ(OB_SUCCESS, util.open(&s3_base)); + + { + // wrong uri + uri[0] = '\0'; + TestS3ListOp op; + ASSERT_EQ(OB_INVALID_ARGUMENT, util.list_directories(uri, op)); + } + + int64_t file_num = 11; + const char *write_content = "0123456789"; + + { + const char *format = "%s/%0*ld_%ld"; + UTIL_WRITE_FILES(format, object_prefix_len, file_num, write_content); + format = "%s/%0*ld/%ld"; + UTIL_WRITE_FILES(format, object_prefix_len, file_num, write_content); + + // list and check and clean + TestS3ListOp op; + ASSERT_EQ(OB_SUCCESS, util.list_directories(dir_uri, op)); + ASSERT_EQ(file_num, op.object_names_.size()); + for (int64_t i = 0; i < file_num; i++) { + // listed directories do not contain prefix + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%0*ld", object_prefix_len, i)); + ASSERT_STREQ(uri, op.object_names_[i]); + } + UTIL_DELETE_FILES(format, object_prefix_len, file_num); + format = "%s/%0*ld_%ld"; + UTIL_DELETE_FILES(format, object_prefix_len, file_num); + } + + // no sub directories + { + const char *format = "%s/%0*ld_%ld"; + UTIL_WRITE_FILES(format, object_prefix_len, file_num, write_content); + + // list and check and clean + TestS3ListOp op; + ASSERT_EQ(OB_SUCCESS, util.list_directories(dir_uri, op)); + ASSERT_EQ(0, op.object_names_.size()); + UTIL_DELETE_FILES(format, object_prefix_len, file_num); + } + + util.close(); + } +} + +// TEST_F(TestStorageS3, test_util_list_adaptive_files) +// { +// int ret = OB_SUCCESS; +// if (enable_test_) { +// ObStorageUtil util; +// const char *tmp_util_dir = "test_util_list_adaptive_files"; +// const int64_t ts = ObTimeUtility::current_time(); +// ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", +// bucket, dir_name, tmp_util_dir, ts)); +// ASSERT_EQ(OB_SUCCESS, util.open(&s3_base)); + +// int64_t file_num = 11; +// const char *write_content = "0123456789"; + +// // list objects +// { +// const char *format = "%s/%0*ld_%ld"; +// UTIL_WRITE_FILES(format, object_prefix_len, file_num, write_content); +// format = "%s/%0*ld/%ld"; +// UTIL_WRITE_FILES(format, object_prefix_len, file_num, write_content); + +// // list and check and clean +// TestS3ListOp op; +// ASSERT_EQ(OB_SUCCESS, util.list_adaptive_files(dir_uri, op)); +// ASSERT_EQ(file_num * 2, op.object_names_.size()); +// // for (int64_t i = 0; i < file_num; i++) { +// // // listed files do not contain prefix +// // ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%0*ld_%ld", object_prefix_len, i, i)); +// // ASSERT_STREQ(uri, op.object_names_[i]); +// // } +// format = "%s/%0*ld_%ld"; +// UTIL_DELETE_FILES(format, object_prefix_len, file_num); +// format = "%s/%0*ld/%ld"; +// UTIL_DELETE_FILES(format, object_prefix_len, file_num); +// } + +// // list empty dir, now dir_uri should be empty after delete +// TestS3ListOp list_empty_op; +// ASSERT_EQ(OB_SUCCESS, util.list_files(dir_uri, list_empty_op)); +// ASSERT_EQ(0, list_empty_op.object_names_.size()); + +// util.close(); +// } +// } + +void test_read_appendable_object(const char *content, const int64_t content_length, + const int64_t n_run, const int64_t *read_start, const int64_t *read_end, + ObStorageAdaptiveReader &reader) +{ + int64_t read_size = -1; + int64_t expected_read_size = -2; + char buf[content_length]; + for (int64_t i = 0; i < n_run; i++) { + read_size = -1; + expected_read_size = MIN(read_end[i], content_length) - MIN(read_start[i], content_length); + ASSERT_TRUE(read_start[i] <= read_end[i]); + ASSERT_EQ(OB_SUCCESS, + reader.pread(buf, read_end[i] - read_start[i], read_start[i], read_size)); + ASSERT_EQ(expected_read_size, read_size); + ASSERT_EQ(0, memcmp(buf, content + read_start[i], read_size)); + } +} + +TEST_F(TestStorageS3, test_append_rw) +{ + const char *str = "012345678"; + char buf[5] = { 0 }; + int return_size = snprintf(buf, sizeof(buf), "%s", str); + std::cout << "return size = " << return_size << std::endl; + + int ret = OB_SUCCESS; + if (enable_test_) { + ObStorageUtil util; + ASSERT_EQ(OB_SUCCESS, util.open(&s3_base)); + const char *tmp_append_dir = "test_append"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_append_dir, ts)); + + { + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/test_append_file_%ld.back", + dir_uri, ObTimeUtility::current_time())); + + ObStorageAppender appender; + ASSERT_EQ(OB_SUCCESS, appender.open(uri, &s3_base)); + + const int64_t content_length = 100; + char content[content_length] = { 0 }; + for (int64_t i = 0; i < content_length; i++) { + content[i] = '0' + (i % 10); + } + // "1-7", "2-5", "3-6", "4-7", "1-7", "1-7", "1-5", // covered by "1-7" + // "0-3", "0-3", "0-1", "1-2", "2-3", // covered "0-3" + // "7-8", "8-9", "9-10", "10-11", "11-12", "12-20", // no gap + // "22-25" & "21-24" are covered by "15-25", and there is a gap from "15-25" to "26-30" + // "15-25", "22-25", "21-24", "26-30", "30-100", "28-100" + int64_t fragment_start[] = {1,2,3,4,1,1,1,0,0,0,1,2,7,8,9,10,11,12,15,22,21,26,30,28}; + int64_t fragment_end[] = {7,5,6,7,7,7,5,3,3,1,2,3,8,9,10,11,12,20,25,25,24,30,100,100}; + ASSERT_EQ(sizeof(fragment_start), sizeof(fragment_end)); + for (int64_t i = 0; i < sizeof(fragment_start) / sizeof(int64_t); i++) { + ASSERT_TRUE(fragment_start[i] < fragment_end[i]); + ASSERT_EQ(OB_SUCCESS, appender.pwrite(content + fragment_start[i], + fragment_end[i] - fragment_start[i], fragment_start[i])); + } + ASSERT_EQ(content_length, appender.get_length()); + + // read before close + ObStorageAdaptiveReader reader; + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &s3_base)); + ASSERT_EQ(content_length, reader.get_length()); + int64_t read_start[] = {0, 1, 0, 4, 26, 30, 26, 0}; + int64_t read_end[] = {3, 4, 15, 25, 27, 100, 101, 0}; + ASSERT_EQ(sizeof(read_start), sizeof(read_end)); + test_read_appendable_object(content, content_length, + sizeof(read_start) / sizeof(int64_t), read_start, read_end, reader); + + char buf[content_length]; + int64_t read_size = -1; + ASSERT_EQ(OB_ERR_UNEXPECTED, + reader.pread(buf, content_length, 0, read_size)); + ASSERT_EQ(content_length, reader.get_length()); + + OB_LOG(INFO, "-=-=-=-=-===========-=-=====-=-=-=-=-=-=-=-=-=-=-=-="); + ASSERT_EQ(OB_SUCCESS, appender.close()); + // open before close, read after close + test_read_appendable_object(content, content_length, + sizeof(read_start) / sizeof(int64_t), read_start, read_end, reader); + ASSERT_EQ(content_length, reader.get_length()); + ASSERT_EQ(OB_SUCCESS, reader.close()); + OB_LOG(INFO, "-=-=-=-=-===========-=-=====-=-=-=-=-=-=-=-=-=-=-=-="); + + // open/read after close + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &s3_base)); + // ASSERT_EQ(content_length - 1, reader.get_appendable_object_size()); + test_read_appendable_object(content, content_length, + sizeof(read_start) / sizeof(int64_t), read_start, read_end, reader); + ASSERT_EQ(content_length, reader.get_length()); + + // read after seal + ASSERT_EQ(OB_SUCCESS, appender.open(uri, &s3_base)); + ASSERT_EQ(OB_SUCCESS, appender.seal_for_adaptive()); + test_read_appendable_object(content, content_length, + sizeof(read_start) / sizeof(int64_t), read_start, read_end, reader); + ASSERT_EQ(content_length, reader.get_length()); + ASSERT_EQ(OB_SUCCESS, reader.close()); + ASSERT_EQ(OB_SUCCESS, appender.close()); + + // open/read after seal + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &s3_base)); + // ASSERT_EQ(content_length - 1, reader.get_appendable_object_size()); + test_read_appendable_object(content, content_length, + sizeof(read_start) / sizeof(int64_t), read_start, read_end, reader); + ASSERT_EQ(content_length, reader.get_length()); + ASSERT_EQ(OB_SUCCESS, reader.close()); + + // ASSERT_EQ(OB_SUCCESS, util.del_appendable_file(uri)); + } + + { + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/test_append_file_%ld.back", + dir_uri, ObTimeUtility::current_time())); + + ObStorageAppender appender; + ASSERT_EQ(OB_SUCCESS, appender.open(uri, &s3_base)); + + const int64_t content_length = 100; + char content[content_length] = { 0 }; + for (int64_t i = 0; i < content_length; i++) { + content[i] = '0' + (i % 10); + } + // "1-7", "2-5", "3-6", "4-7", "1-7", "1-7", "1-5", // covered by "1-7" + // "0-3", "0-3", "0-1", "1-2", "2-3", // covered "0-3" + // "7-8", "8-9", "9-10", "10-11", "11-12", "12-20", // no gap + // "22-25" & "21-24" are covered by "15-25", and there is a gap from "15-25" to "26-30" + // "15-25", "22-25", "21-24", "26-30", "30-100", "28-100" + int64_t fragment_start[] = {1,2,3,4,1,1,1,0,0,0,1,2,7,8,9,10,11,12,15,22,21,26,30,28}; + int64_t fragment_end[] = {7,5,6,7,7,7,5,3,3,1,2,3,8,9,10,11,12,20,25,25,24,30,100,100}; + ASSERT_EQ(sizeof(fragment_start), sizeof(fragment_end)); + for (int64_t i = 0; i < sizeof(fragment_start) / sizeof(int64_t); i++) { + ASSERT_TRUE(fragment_start[i] < fragment_end[i]); + ASSERT_EQ(OB_SUCCESS, appender.pwrite(content + fragment_start[i], + fragment_end[i] - fragment_start[i], fragment_start[i])); + } + ASSERT_EQ(content_length, appender.get_length()); + + // read before close + ObStorageAdaptiveReader reader; + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &s3_base)); + ASSERT_EQ(content_length, reader.get_length()); + + char buf[content_length]; + int64_t read_size = -1; + ASSERT_EQ(OB_ERR_UNEXPECTED, + reader.pread(buf, content_length, 0, read_size)); + ASSERT_EQ(OB_SUCCESS, reader.close()); + + // appender not finished, use appendable put + // ObStorageAdaptiveWriter writer; + // ASSERT_EQ(OB_SUCCESS, writer.open(uri, &s3_base)); + // ASSERT_EQ(OB_SUCCESS, writer.write(content, content_length)); + // ASSERT_EQ(OB_SUCCESS, writer.close()); + + // now gap is filled + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &s3_base)); + ASSERT_EQ(content_length, reader.get_length()); + ASSERT_EQ(OB_SUCCESS, + reader.pread(buf, content_length, 0, read_size)); + ASSERT_EQ(OB_SUCCESS, reader.close()); + + // test appendable put is valid + ObStorageUtil s3_util; + ObStorageObjectMeta appendable_obj_meta; + ASSERT_EQ(OB_SUCCESS, s3_util.open(&s3_base)); + ASSERT_EQ(OB_SUCCESS, s3_util.list_appendable_file_fragments(uri, appendable_obj_meta)); + ASSERT_EQ(1, appendable_obj_meta.fragment_metas_.count()); + ASSERT_EQ(0, appendable_obj_meta.fragment_metas_[0].start_); + ASSERT_EQ(9223372036854775807, appendable_obj_meta.fragment_metas_[0].end_); + ASSERT_EQ(content_length, appendable_obj_meta.length_); + + ASSERT_EQ(OB_SUCCESS, appender.close()); + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &s3_base)); + // open before close, read after close + int64_t read_start[] = {0, 1, 0, 4, 26, 30, 26, 0}; + int64_t read_end[] = {3, 4, 15, 25, 27, 100, 101, 0}; + test_read_appendable_object(content, content_length, + sizeof(read_start) / sizeof(int64_t), read_start, read_end, reader); + ASSERT_EQ(content_length, reader.get_length()); + ASSERT_EQ(OB_SUCCESS, reader.close()); + + // open/read after close + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &s3_base)); + // ASSERT_EQ(content_length - 1, reader.get_appendable_object_size()); + test_read_appendable_object(content, content_length, + sizeof(read_start) / sizeof(int64_t), read_start, read_end, reader); + ASSERT_EQ(content_length, reader.get_length()); + + // read after seal + ASSERT_EQ(OB_SUCCESS, appender.open(uri, &s3_base)); + ASSERT_EQ(OB_SUCCESS, appender.seal_for_adaptive()); + test_read_appendable_object(content, content_length, + sizeof(read_start) / sizeof(int64_t), read_start, read_end, reader); + ASSERT_EQ(content_length, reader.get_length()); + ASSERT_EQ(OB_SUCCESS, reader.close()); + ASSERT_EQ(OB_SUCCESS, appender.close()); + + // open/read after seal + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &s3_base)); + // ASSERT_EQ(content_length - 1, reader.get_appendable_object_size()); + test_read_appendable_object(content, content_length, + sizeof(read_start) / sizeof(int64_t), read_start, read_end, reader); + ASSERT_EQ(content_length, reader.get_length()); + ASSERT_EQ(OB_SUCCESS, reader.close()); + + // ASSERT_EQ(OB_SUCCESS, util.del_appendable_file(uri)); + } + + { + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/test_append_file_%ld.back", + dir_uri, ObTimeUtility::current_time())); + + ObStorageAppender appender_a; + ObStorageAppender appender_b; + ObStorageAppender appender_c; + ASSERT_EQ(OB_SUCCESS, appender_a.open(uri, &s3_base)); + ASSERT_EQ(OB_SUCCESS, appender_b.open(uri, &s3_base)); + ASSERT_EQ(OB_SUCCESS, appender_c.open(uri, &s3_base)); + + const int64_t content_length = 100; + char content[content_length] = { 0 }; + for (int64_t i = 0; i < content_length; i++) { + content[i] = '0' + (i % 10); + } + // "1-7", "2-5", "3-6", "4-7", "1-7", "1-7", "1-5", // covered by "1-7" + // "0-3", "0-3", "0-1", "1-2", "2-3", // covered "0-3" + // "7-8", "8-9", "9-10", "10-11", "11-12", "12-20", // no gap + // "22-25" & "21-24" are covered by "15-25" + // "15-25", "22-25", "21-24", "25-30", "30-100", "28-100" + int64_t fragment_start[] = {1,2,3,4,1,1,1,0,0,0,1,2,7,8,9,10,11,12,15,22,21,25,30,28}; + int64_t fragment_end[] = {7,5,6,7,7,7,5,3,3,1,2,3,8,9,10,11,12,20,25,25,24,30,100,100}; + ASSERT_EQ(sizeof(fragment_start), sizeof(fragment_end)); + for (int64_t i = 0; i < sizeof(fragment_start) / sizeof(int64_t); i++) { + ASSERT_TRUE(fragment_start[i] < fragment_end[i]); + if (i % 3 == 0) { + ASSERT_EQ(OB_SUCCESS, appender_a.pwrite(content + fragment_start[i], + fragment_end[i] - fragment_start[i], fragment_start[i])); + } else if (i % 3 == 1) { + ASSERT_EQ(OB_SUCCESS, appender_b.pwrite(content + fragment_start[i], + fragment_end[i] - fragment_start[i], fragment_start[i])); + } else { + ASSERT_EQ(OB_SUCCESS, appender_c.pwrite(content + fragment_start[i], + fragment_end[i] - fragment_start[i], fragment_start[i])); + } + } + + // read before close + ObStorageAdaptiveReader reader; + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &s3_base)); + int64_t read_start[] = {0, 1, 0, 4, 26, 30, 26, 0, 0}; + int64_t read_end[] = {3, 4, 15, 25, 27, 100, 101, 0, 100}; + ASSERT_EQ(sizeof(read_start), sizeof(read_end)); + int64_t n_run = sizeof(read_start) / sizeof(int64_t); + std::thread read_thread_a(test_read_appendable_object, content, content_length, + n_run, read_start, read_end, std::ref(reader)); + std::thread read_thread_b(test_read_appendable_object, content, content_length, + n_run, read_start, read_end, std::ref(reader)); + read_thread_a.join(); + read_thread_b.join(); + ASSERT_EQ(content_length, reader.get_length()); + + // open before close, read after close + ASSERT_EQ(OB_SUCCESS, appender_a.close()); + ASSERT_EQ(OB_SUCCESS, appender_b.close()); + ASSERT_EQ(OB_SUCCESS, appender_c.close()); + std::thread read_thread_c(test_read_appendable_object, content, content_length, + n_run, read_start, read_end, std::ref(reader)); + std::thread read_thread_d(test_read_appendable_object, content, content_length, + n_run, read_start, read_end, std::ref(reader)); + read_thread_c.join(); + read_thread_d.join(); + ASSERT_EQ(content_length, reader.get_length()); + ASSERT_EQ(OB_SUCCESS, reader.close()); + + // open/read after close + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &s3_base)); + // ASSERT_EQ(content_length, reader.get_appendable_object_size()); + std::thread read_thread_e(test_read_appendable_object, content, content_length, + n_run, read_start, read_end, std::ref(reader)); + std::thread read_thread_f(test_read_appendable_object, content, content_length, + n_run, read_start, read_end, std::ref(reader)); + read_thread_e.join(); + read_thread_f.join(); + ASSERT_EQ(content_length, reader.get_length()); + + // read after seal + ASSERT_EQ(OB_SUCCESS, appender_a.open(uri, &s3_base)); + ASSERT_EQ(OB_SUCCESS, appender_a.seal_for_adaptive()); + std::thread read_thread_g(test_read_appendable_object, content, content_length, + n_run, read_start, read_end, std::ref(reader)); + std::thread read_thread_h(test_read_appendable_object, content, content_length, + n_run, read_start, read_end, std::ref(reader)); + read_thread_g.join(); + read_thread_h.join(); + ASSERT_EQ(content_length, reader.get_length()); + ASSERT_EQ(OB_SUCCESS, reader.close()); + ASSERT_EQ(OB_SUCCESS, appender_a.close()); + + // open/read after seal + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &s3_base)); + std::thread read_thread_i(test_read_appendable_object, content, content_length, + n_run, read_start, read_end, std::ref(reader)); + std::thread read_thread_j(test_read_appendable_object, content, content_length, + n_run, read_start, read_end, std::ref(reader)); + read_thread_i.join(); + read_thread_j.join(); + ASSERT_EQ(content_length, reader.get_length()); + ASSERT_EQ(OB_SUCCESS, reader.close()); + + // ASSERT_EQ(OB_SUCCESS, util.del_appendable_file(uri)); + } + } +} + +void test_gen_object_meta(const char **fragments, const int64_t n_fragments, + const int64_t n_remained_fragments, const int64_t *expected_start, const int64_t *expected_end, + const int64_t expected_file_length, ObStorageObjectMeta &appendable_obj_meta) +{ + dirent entry; + ListAppendableObjectFragmentOp op; + appendable_obj_meta.reset(); + + for (int64_t i = 0; i < n_fragments; i++) { + ASSERT_TRUE(sizeof(entry.d_name) >= strlen(fragments[i]) + 1); + STRCPY(entry.d_name, fragments[i]); + // STRCPY(entry.d_name + strlen(entry.d_name), ".back"); + ASSERT_EQ(OB_SUCCESS, op.func(&entry)); + } + + ASSERT_EQ(OB_SUCCESS, op.gen_object_meta(appendable_obj_meta)); + OB_LOG(INFO, "--", K(appendable_obj_meta)); + ASSERT_EQ(expected_file_length, appendable_obj_meta.length_); + ASSERT_EQ(n_remained_fragments, appendable_obj_meta.fragment_metas_.count()); + for (int64_t i = 0; i < n_remained_fragments; i++) { + ASSERT_EQ(expected_start[i], appendable_obj_meta.fragment_metas_[i].start_); + ASSERT_EQ(expected_end[i], appendable_obj_meta.fragment_metas_[i].end_); + } +} + +void test_get_needed_fragments(const int64_t start, const int64_t end, + const int64_t n_expected_fragments, const int64_t *expected_start, const int64_t *expected_end, + ObStorageObjectMeta &appendable_obj_meta) +{ + ObArray fragments_need_to_read; + ASSERT_EQ(OB_SUCCESS, + appendable_obj_meta.get_needed_fragments(start, end, fragments_need_to_read)); + // OB_LOG(INFO, "*************************", K(start), K(end), K(n_expected_fragments), K(fragments_need_to_read), K(appendable_obj_meta)); + ASSERT_EQ(n_expected_fragments, fragments_need_to_read.count()); + for (int64_t i = 0; i < n_expected_fragments; i++) { + ASSERT_EQ(expected_start[i], fragments_need_to_read[i].start_); + ASSERT_EQ(expected_end[i], fragments_need_to_read[i].end_); + } +} + +TEST_F(TestStorageS3, test_appendable_object_util) +{ + int ret = OB_SUCCESS; + if (enable_test_) { + { + ListAppendableObjectFragmentOp op; + dirent entry; + + // meta file + ASSERT_TRUE(sizeof(entry.d_name) >= sizeof(OB_S3_APPENDABLE_SEAL_META)); + STRCPY(entry.d_name, OB_S3_APPENDABLE_SEAL_META); + ASSERT_EQ(OB_SUCCESS, op.func(&entry)); + + // invalid fragment name + const char *invalid_fragments[] = {"-2--1", "-1-1", "2--2", "3-2"}; + for (int64_t i = 0; i < sizeof(invalid_fragments) / sizeof(char *); i++) { + ASSERT_TRUE(sizeof(entry.d_name) >= strlen(invalid_fragments[i]) + 1); + STRCPY(entry.d_name, invalid_fragments[i]); + ASSERT_EQ(OB_INVALID_ARGUMENT, op.func(&entry)); + } + } + + { + // empty appendable object + const char *fragments[] = {}; + int64_t n_fragments = sizeof(fragments) / sizeof(char *); + int64_t expected_start[] = {}; + int64_t expected_end[] = {}; + int64_t n_remained_fragments = sizeof(expected_start) / sizeof(int64_t); + int64_t expected_file_length = 0; + ObStorageObjectMeta appendable_obj_meta; + test_gen_object_meta( + fragments, n_fragments, + n_remained_fragments, expected_start, expected_end, + expected_file_length, appendable_obj_meta); + + ObArray fragments_need_to_read; + ASSERT_EQ(OB_INVALID_ARGUMENT, + appendable_obj_meta.get_needed_fragments(-1, 1, fragments_need_to_read)); + ASSERT_EQ(OB_INVALID_ARGUMENT, + appendable_obj_meta.get_needed_fragments(5, 4, fragments_need_to_read)); + ASSERT_EQ(OB_INVALID_ARGUMENT, + appendable_obj_meta.get_needed_fragments(5, 5, fragments_need_to_read)); + ASSERT_EQ(OB_ERR_UNEXPECTED, + appendable_obj_meta.get_needed_fragments(0, 1, fragments_need_to_read)); + } + + { + // one fragment + const char *fragments[] = {"10-20.back"}; + int64_t n_fragments = sizeof(fragments) / sizeof(char *); + int64_t expected_start[] = {10}; + int64_t expected_end[] = {20}; + int64_t n_remained_fragments = sizeof(expected_start) / sizeof(int64_t); + int64_t expected_file_length = 20; + ObStorageObjectMeta appendable_obj_meta; + test_gen_object_meta( + fragments, n_fragments, + n_remained_fragments, expected_start, expected_end, + expected_file_length, appendable_obj_meta); + + ObArray fragments_need_to_read; + ASSERT_EQ(OB_ERR_UNEXPECTED, + appendable_obj_meta.get_needed_fragments(5, 8, fragments_need_to_read)); + ASSERT_EQ(OB_ERR_UNEXPECTED, + appendable_obj_meta.get_needed_fragments(5, 10, fragments_need_to_read)); + ASSERT_EQ(OB_ERR_UNEXPECTED, + appendable_obj_meta.get_needed_fragments(5, 15, fragments_need_to_read)); + ASSERT_EQ(OB_INVALID_ARGUMENT, + appendable_obj_meta.get_needed_fragments(11, 11, fragments_need_to_read)); + + { + int64_t start = 10; + int64_t end = 18; + int64_t expected_start[] = {10}; + int64_t expected_end[] = {20}; + int64_t n_expected_fragments = sizeof(expected_start) / sizeof(int64_t); + test_get_needed_fragments(start, end, + n_expected_fragments, expected_start, expected_end, appendable_obj_meta); + } + { + int64_t start = 15; + int64_t end = 18; + int64_t expected_start[] = {10}; + int64_t expected_end[] = {20}; + int64_t n_expected_fragments = sizeof(expected_start) / sizeof(int64_t); + test_get_needed_fragments(start, end, + n_expected_fragments, expected_start, expected_end, appendable_obj_meta); + } + { + int64_t start = 19; + int64_t end = 20; + int64_t expected_start[] = {10}; + int64_t expected_end[] = {20}; + int64_t n_expected_fragments = sizeof(expected_start) / sizeof(int64_t); + test_get_needed_fragments(start, end, + n_expected_fragments, expected_start, expected_end, appendable_obj_meta); + } + { + int64_t start = 19; + int64_t end = 21; + int64_t expected_start[] = {10}; + int64_t expected_end[] = {20}; + int64_t n_expected_fragments = sizeof(expected_start) / sizeof(int64_t); + test_get_needed_fragments(start, end, + n_expected_fragments, expected_start, expected_end, appendable_obj_meta); + } + { + int64_t start = 20; + int64_t end = 21; + int64_t expected_start[] = {}; + int64_t expected_end[] = {}; + int64_t n_expected_fragments = sizeof(expected_start) / sizeof(int64_t); + test_get_needed_fragments(start, end, + n_expected_fragments, expected_start, expected_end, appendable_obj_meta); + } + } + + { + // valid fragment name + const char *valid_fragments[] = { + OB_S3_APPENDABLE_SEAL_META, + "1-7", "2-5", "3-6", "4-7", "1-7", "1-7", "1-5", // covered by "1-7" + "0-3", "0-3", "0-1", "1-2", "2-3", // covered "0-3" + "7-8", "8-9", "9-10", "10-11", "11-12", "12-20", // no gap + // "22-25" & "21-24" are covered by "15-25", and there is a gap from "15-25" to "26-30" + "15-25", "22-25", "21-24", "26-30", + "30-1234567", "10000-1234567", "28-1234566" + }; + int64_t n_fragments = sizeof(valid_fragments) / sizeof(char *); + int64_t expected_start[] = {0, 1, 7, 8, 9, 10, 11, 12, 15, 26, 30}; + int64_t expected_end[] = {3, 7, 8, 9, 10, 11, 12, 20, 25, 30, 1234567}; + int64_t n_remained_fragments = sizeof(expected_start) / sizeof(int64_t); + int64_t expected_file_length = 1234567; + ObStorageObjectMeta appendable_obj_meta; + test_gen_object_meta( + valid_fragments, n_fragments, + n_remained_fragments, expected_start, expected_end, + expected_file_length, appendable_obj_meta); + + { + int64_t start = 0; + int64_t end = 3; + int64_t expected_start[] = {0}; + int64_t expected_end[] = {3}; + int64_t n_expected_fragments = sizeof(expected_start) / sizeof(int64_t); + test_get_needed_fragments(start, end, + n_expected_fragments, expected_start, expected_end, appendable_obj_meta); + } + { + int64_t start = 1; + int64_t end = 4; + int64_t expected_start[] = {1}; + int64_t expected_end[] = {7}; + int64_t n_expected_fragments = sizeof(expected_start) / sizeof(int64_t); + test_get_needed_fragments(start, end, + n_expected_fragments, expected_start, expected_end, appendable_obj_meta); + } + { + int64_t start = 0; + int64_t end = 15; + int64_t expected_start[] = {0, 1, 7, 8, 9, 10, 11, 12}; + int64_t expected_end[] = {3, 7, 8, 9, 10, 11, 12, 20}; + int64_t n_expected_fragments = sizeof(expected_start) / sizeof(int64_t); + test_get_needed_fragments(start, end, + n_expected_fragments, expected_start, expected_end, appendable_obj_meta); + } + { + ObArray fragments_need_to_read; + ASSERT_EQ(OB_ERR_UNEXPECTED, + appendable_obj_meta.get_needed_fragments(15, 30, fragments_need_to_read)); + } + { + int64_t start = 28; + int64_t end = 2345678; + int64_t expected_start[] = {26, 30}; + int64_t expected_end[] = {30, 1234567}; + int64_t n_expected_fragments = sizeof(expected_start) / sizeof(int64_t); + test_get_needed_fragments(start, end, + n_expected_fragments, expected_start, expected_end, appendable_obj_meta); + } + } + } +} + +int main(int argc, char **argv) +{ + system("rm -f test_storage_s3.log*"); + OB_LOGGER.set_file_name("test_storage_s3.log", true, true); + OB_LOGGER.set_log_level("DEBUG"); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/deps/oblib/unittest/lib/restore/test_storage_s3.h b/deps/oblib/unittest/lib/restore/test_storage_s3.h new file mode 100644 index 000000000..bfb4a377b --- /dev/null +++ b/deps/oblib/unittest/lib/restore/test_storage_s3.h @@ -0,0 +1,24 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef TEST_STORAGE_S3_H_ +#define TEST_STORAGE_S3_H_ + +// TODO @fangdan: configure the parameters uniformly +const bool enable_test = false; +const char *bucket = "s3://xxx"; +const char *region = "xxx"; +const char *endpoint = "xxx"; +const char *secretid = "xxx"; +const char *secretkey = "xxx"; + +#endif \ No newline at end of file diff --git a/src/logservice/archiveservice/ob_archive_file_utils.cpp b/src/logservice/archiveservice/ob_archive_file_utils.cpp index 74c0dbda7..a7addb798 100644 --- a/src/logservice/archiveservice/ob_archive_file_utils.cpp +++ b/src/logservice/archiveservice/ob_archive_file_utils.cpp @@ -135,7 +135,7 @@ int ObArchiveFileUtils::get_file_range(const ObString &prefix, ObBackupIoAdapter util; ObFileRangeOp file_range_op; - if (OB_FAIL(util.list_files(prefix, storage_info, file_range_op))) { + if (OB_FAIL(util.adaptively_list_files(prefix, storage_info, file_range_op))) { ARCHIVE_LOG(WARN, "list_files fail", K(ret), K(prefix)); } else if (0 == file_range_op.get_file_num()) { ret = OB_ENTRY_NOT_EXIST; @@ -154,7 +154,7 @@ int ObArchiveFileUtils::list_files(const ObString &prefix, int ret = OB_SUCCESS; ObBackupIoAdapter util; ObFileListOp file_list_op; - if (OB_FAIL(util.list_files(prefix, storage_info, file_list_op))) { + if (OB_FAIL(util.adaptively_list_files(prefix, storage_info, file_list_op))) { ARCHIVE_LOG(WARN, "list_files fail", K(ret), K(prefix)); } else if (OB_FAIL(file_list_op.get_file_list(array))) { ARCHIVE_LOG(WARN, "get_file_list fail", K(ret), K(prefix)); @@ -171,7 +171,7 @@ int ObArchiveFileUtils::read_file(const ObString &uri, int ret = OB_SUCCESS; ObBackupIoAdapter util; - if (OB_FAIL(util.read_single_file(uri, storage_info, buf, file_length, read_size))) { + if (OB_FAIL(util.adaptively_read_single_file(uri, storage_info, buf, file_length, read_size))) { if (OB_BACKUP_FILE_NOT_EXIST != ret) { ARCHIVE_LOG(WARN, "read_single_file fail", K(ret), K(uri)); } else { @@ -196,7 +196,7 @@ int ObArchiveFileUtils::range_read(const ObString &uri, if (OB_UNLIKELY(NULL == buf || buf_size < 0 || offset < 0)) { ret = OB_INVALID_ARGUMENT; ARCHIVE_LOG(WARN, "invalid argument", K(ret), K(buf_size), K(offset), K(uri)); - } else if (OB_FAIL(util.read_part_file(uri, storage_info, buf, buf_size, offset, read_size))) { + } else if (OB_FAIL(util.adaptively_read_part_file(uri, storage_info, buf, buf_size, offset, read_size))) { ARCHIVE_LOG(WARN, "read part file failed", K(ret), K(uri), K(buf_size), K(offset)); } return ret; @@ -240,7 +240,7 @@ int ObArchiveFileUtils::get_file_length(const ObString &uri, int ret = OB_SUCCESS; ObBackupIoAdapter util; - if (OB_FAIL(util.get_file_length(uri, storage_info, file_len))) { + if (OB_FAIL(util.adaptively_get_file_length(uri, storage_info, file_len))) { if (OB_BACKUP_FILE_NOT_EXIST != ret) { ARCHIVE_LOG(WARN, "get_file_length fail", K(ret), K(uri), KP(storage_info)); } else { diff --git a/src/logservice/archiveservice/ob_archive_io.cpp b/src/logservice/archiveservice/ob_archive_io.cpp index 6aa479554..61835386c 100644 --- a/src/logservice/archiveservice/ob_archive_io.cpp +++ b/src/logservice/archiveservice/ob_archive_io.cpp @@ -32,7 +32,8 @@ int ObArchiveIO::push_log(const ObString &uri, char *data, const int64_t data_len, const int64_t offset, - const bool is_full_file) + const bool is_full_file, + const bool is_can_seal) { int ret = OB_SUCCESS; ObBackupIoAdapter util; @@ -71,10 +72,13 @@ int ObArchiveIO::push_log(const ObString &uri, ARCHIVE_LOG(ERROR, "device_handle is NULL", K(ret), K(device_handle), K(uri)); } else if (OB_FAIL(device_handle->pwrite(fd, offset, data_len, data, write_size))) { ARCHIVE_LOG(WARN, "fail to write file", K(ret), K(uri), KP(storage_info), K(data), K(data_len)); + } else if (is_can_seal && OB_FAIL(device_handle->seal_file(fd))) { + ARCHIVE_LOG(WARN, "fail to seal file", K(ret), K(uri), KP(storage_info)); } } } + int tmp_ret = OB_SUCCESS; if (OB_SUCCESS != (tmp_ret = util.close_device_and_fd(device_handle, fd))) { ARCHIVE_LOG(WARN, "fail to close file and release device!", K(tmp_ret), K(uri), KP(storage_info)); @@ -108,7 +112,7 @@ int ObArchiveIO::check_context_match_in_normal_file_(const ObString &uri, ObBackupIoAdapter reader; ObArenaAllocator allocator; - if (OB_FAIL(reader.get_file_length(uri, storage_info, length))) { + if (OB_FAIL(reader.adaptively_get_file_length(uri, storage_info, length))) { ARCHIVE_LOG(WARN, "get file_length failed", K(uri)); } else if (OB_UNLIKELY(length <= offset)) { ret = OB_BACKUP_PWRITE_CONTENT_NOT_MATCH; @@ -119,7 +123,7 @@ int ObArchiveIO::check_context_match_in_normal_file_(const ObString &uri, } else if (OB_ISNULL(read_buffer = static_cast(allocator.alloc(data_len)))) { ret = OB_ALLOCATE_MEMORY_FAILED; ARCHIVE_LOG(WARN, "allocate memory failed", K(uri), K(data_len)); - } else if (OB_FAIL(reader.read_part_file(uri, storage_info, read_buffer, data_len, offset, read_size))) { + } else if (OB_FAIL(reader.adaptively_read_part_file(uri, storage_info, read_buffer, data_len, offset, read_size))) { ARCHIVE_LOG(WARN, "pread failed", K(uri), K(read_buffer), K(data_len), K(offset)); } else if (read_size < data_len) { ret = OB_BACKUP_PWRITE_CONTENT_NOT_MATCH; diff --git a/src/logservice/archiveservice/ob_archive_io.h b/src/logservice/archiveservice/ob_archive_io.h index aee2024c7..5e6953cda 100644 --- a/src/logservice/archiveservice/ob_archive_io.h +++ b/src/logservice/archiveservice/ob_archive_io.h @@ -33,7 +33,8 @@ public: char *data, const int64_t data_len, const int64_t offset, - const bool is_full_file); + const bool is_full_file, + const bool is_can_seal); int mkdir(const ObString &uri, const share::ObBackupStorageInfo *storage_info); diff --git a/src/logservice/archiveservice/ob_archive_sender.cpp b/src/logservice/archiveservice/ob_archive_sender.cpp index e7d8c7c8c..ae595f0f0 100644 --- a/src/logservice/archiveservice/ob_archive_sender.cpp +++ b/src/logservice/archiveservice/ob_archive_sender.cpp @@ -583,6 +583,7 @@ int ObArchiveSender::archive_log_(const ObBackupDest &backup_dest, char *filled_data = NULL; int64_t filled_data_len = 0; const bool is_full_file = (task.get_end_lsn() - task.get_start_lsn()) == MAX_ARCHIVE_FILE_SIZE; + const bool is_can_seal = 0 == task.get_end_lsn().val_ % MAX_ARCHIVE_FILE_SIZE; const int64_t start_ts = common::ObTimeUtility::current_time(); // 1. decide archive file if (OB_FAIL(decide_archive_file_(task, arg.cur_file_id_, arg.cur_file_offset_, @@ -612,8 +613,8 @@ int ObArchiveSender::archive_log_(const ObBackupDest &backup_dest, ARCHIVE_LOG(WARN, "fill file header if needed failed", K(ret)); } // 6. push log - else if (OB_FAIL(push_log_(id, path.get_obstr(), backup_dest.get_storage_info(), is_full_file, new_file ? - file_offset : file_offset + ARCHIVE_FILE_HEADER_SIZE, + else if (OB_FAIL(push_log_(id, path.get_obstr(), backup_dest.get_storage_info(), is_full_file, + is_can_seal, new_file ? file_offset : file_offset + ARCHIVE_FILE_HEADER_SIZE, new_file ? filled_data : origin_data, new_file ? filled_data_len : origin_data_len))) { ARCHIVE_LOG(WARN, "push log failed", K(ret), K(task)); // 7. 更新日志流归档任务archive file info @@ -723,6 +724,7 @@ int ObArchiveSender::push_log_(const ObLSID &id, const ObString &uri, const share::ObBackupStorageInfo *storage_info, const bool is_full_file, + const bool is_can_seal, const int64_t offset, char *data, const int64_t data_len) @@ -730,7 +732,7 @@ int ObArchiveSender::push_log_(const ObLSID &id, int ret = OB_SUCCESS; ObArchiveIO archive_io; - if (OB_FAIL(archive_io.push_log(uri, storage_info, data, data_len, offset, is_full_file))) { + if (OB_FAIL(archive_io.push_log(uri, storage_info, data, data_len, offset, is_full_file, is_can_seal))) { ARCHIVE_LOG(WARN, "push log failed", K(ret)); } else { ARCHIVE_LOG(INFO, "push log succ", K(id)); diff --git a/src/logservice/archiveservice/ob_archive_sender.h b/src/logservice/archiveservice/ob_archive_sender.h index 92599fa5e..55faceb9f 100644 --- a/src/logservice/archiveservice/ob_archive_sender.h +++ b/src/logservice/archiveservice/ob_archive_sender.h @@ -160,6 +160,7 @@ private: const ObString &uri, const share::ObBackupStorageInfo *storage_info, const bool is_full_file, + const bool is_can_seal, const int64_t offset, char *data, const int64_t data_len); diff --git a/src/logservice/ob_log_external_storage_io_task.cpp b/src/logservice/ob_log_external_storage_io_task.cpp index 5e643e3fe..dbdd0f704 100644 --- a/src/logservice/ob_log_external_storage_io_task.cpp +++ b/src/logservice/ob_log_external_storage_io_task.cpp @@ -240,7 +240,7 @@ int convert_to_storage_access_type(const OPEN_FLAG &open_flag, { int ret = OB_SUCCESS; if (OPEN_FLAG::READ_FLAG == open_flag) { - storage_access_type = ObStorageAccessType::OB_STORAGE_ACCESS_READER; + storage_access_type = ObStorageAccessType::OB_STORAGE_ACCESS_ADAPTIVE_READER; } else { ret = OB_NOT_SUPPORTED; CLOG_LOG(WARN, "not supported flag", K(open_flag)); @@ -300,7 +300,7 @@ int ObLogExternalStorageIOTaskHandleAdapter::exist(const ObString &uri, ObIODevice *io_device = NULL; if (OB_FAIL(get_and_init_io_device(uri, storage_info, io_device))) { CLOG_LOG(WARN, "get_io_device failed", K(uri), KP(io_device)); - } else if (OB_FAIL(io_device->exist(uri.ptr(), exist))) { + } else if (OB_FAIL(io_device->adaptive_exist(uri.ptr(), exist))) { CLOG_LOG(WARN, "exist failed", K(uri), KP(io_device), K(exist)); } else { CLOG_LOG(TRACE, "exist success", K(uri), KP(io_device), K(exist)); @@ -319,7 +319,7 @@ int ObLogExternalStorageIOTaskHandleAdapter::get_file_size(const ObString &uri, ObIODevice *io_device = NULL; if (OB_FAIL(get_and_init_io_device(uri, storage_info, io_device))) { CLOG_LOG(WARN, "get_io_device failed", K(uri), KP(io_device)); - } else if (OB_FAIL(io_device->stat(uri.ptr(), file_stat))) { + } else if (OB_FAIL(io_device->adaptive_stat(uri.ptr(), file_stat))) { CLOG_LOG(WARN, "stat io deveice failed", K(uri)); } else { file_size = file_stat.size_; diff --git a/src/share/backup/ob_archive_store.cpp b/src/share/backup/ob_archive_store.cpp index 548f0672e..d47049be5 100644 --- a/src/share/backup/ob_archive_store.cpp +++ b/src/share/backup/ob_archive_store.cpp @@ -1128,7 +1128,7 @@ int ObArchiveStore::is_archive_log_file_exist(const int64_t dest_id, const int64 LOG_WARN("ObArchiveStore not init", K(ret)); } else if (OB_FAIL(ObArchivePathUtil::get_ls_archive_file_path(dest, dest_id, round_id, piece_id, ls_id, file_id, full_path))) { LOG_WARN("failed to get archive log file path", K(ret), K(dest), K(dest_id), K(round_id), K(piece_id), K(ls_id), K(file_id)); - } else if (OB_FAIL(util.is_exist(full_path.get_ptr(), storage_info, is_exist))) { + } else if (OB_FAIL(util.adaptively_is_exist(full_path.get_ptr(), storage_info, is_exist))) { LOG_WARN("failed to check archive log file exist.", K(ret), K(full_path), K(storage_info)); } @@ -1439,7 +1439,7 @@ int ObArchiveStore::get_file_list_in_piece(const int64_t dest_id, const int64_t LOG_WARN("get piece ls dir path failed", K(ret), K(dest), K(dest_id), K(round_id), K(piece_id), K(ls_id)); } else if (OB_FAIL(op.init(this, &filelist))) { LOG_WARN("ObLSFileListOp init failed", K(ret)); - } else if (OB_FAIL(util.list_files(piece_path.get_ptr(), storage_info, op))) { + } else if (OB_FAIL(util.adaptively_list_files(piece_path.get_ptr(), storage_info, op))) { LOG_WARN("list files failed", K(ret), K(piece_path), K(dest)); } return ret; diff --git a/src/share/backup/ob_backup_connectivity.cpp b/src/share/backup/ob_backup_connectivity.cpp index 0e462c78d..4019537a0 100644 --- a/src/share/backup/ob_backup_connectivity.cpp +++ b/src/share/backup/ob_backup_connectivity.cpp @@ -326,7 +326,7 @@ int ObBackupCheckFile::compare_check_file_name_( ObDirPrefixEntryNameFilter prefix_op(d_entrys); if (OB_FAIL(prefix_op.init(check_file_prefix, static_cast(strlen(check_file_prefix))))) { LOG_WARN("failed to init dir prefix", K(ret), K(check_file_prefix), K_(tenant_id)); - } else if (OB_FAIL(util.list_files(path.get_obstr(), backup_dest.get_storage_info(), prefix_op))) { + } else if (OB_FAIL(util.adaptively_list_files(path.get_obstr(), backup_dest.get_storage_info(), prefix_op))) { LOG_WARN("failed to list files", K(ret), K_(tenant_id)); } else if (OB_FAIL(ObBackupStorageInfoOperator::get_check_file_name( *sql_proxy_, tenant_id_, backup_dest, check_file_name))) { @@ -346,7 +346,7 @@ int ObBackupCheckFile::compare_check_file_name_( LOG_WARN("failed to set check file path", K(ret), K(path), K_(tmp_entry.name)); } else { common::ObString uri(del_file_path); - if(OB_FAIL(util.del_file(uri, backup_dest.get_storage_info()))) { + if(OB_FAIL(util.adaptively_del_file(uri, backup_dest.get_storage_info()))) { LOG_WARN("failed to delete check file", K(ret), K_(tenant_id)); } } @@ -451,7 +451,7 @@ int ObBackupCheckFile::delete_permission_check_file(const ObBackupDest &backup_d ObDirPrefixEntryNameFilter prefix_op(d_entrys); if (OB_FAIL(prefix_op.init(check_file_prefix, static_cast(strlen(check_file_prefix))))) { LOG_WARN("failed to init dir prefix", K(ret), K(check_file_prefix), K_(tenant_id)); - } else if (OB_FAIL(util.list_files(path.get_obstr(), backup_dest.get_storage_info(), prefix_op))) { + } else if (OB_FAIL(util.adaptively_list_files(path.get_obstr(), backup_dest.get_storage_info(), prefix_op))) { LOG_WARN("failed to list files", K(ret), K_(tenant_id)); } else { char del_file_path[OB_MAX_BACKUP_PATH_LENGTH]; @@ -466,7 +466,7 @@ int ObBackupCheckFile::delete_permission_check_file(const ObBackupDest &backup_d LOG_WARN("failed to set delete file path", K(ret), K(path), K_(tmp_entry.name)); } else { common::ObString uri(del_file_path); - if(OB_FAIL(util.del_file(uri, backup_dest.get_storage_info()))) { + if(OB_FAIL(util.adaptively_del_file(uri, backup_dest.get_storage_info()))) { LOG_WARN("failed to delete permission check file", K(ret), K_(tenant_id)); } } @@ -538,7 +538,7 @@ int ObBackupCheckFile::check_appender_permission_(const ObBackupDest &backup_des LOG_WARN("fail to set data", K(ret), K(path.get_ptr())); } else if (OB_FAIL(device_handle->write(fd, data, strlen(data), write_size))) { LOG_WARN("fail to write file", K(ret), K(path.get_ptr()), K(data)); - } else if (OB_FAIL(util.del_file(path.get_obstr(), backup_dest.get_storage_info()))) { + } else if (OB_FAIL(util.adaptively_del_file(path.get_obstr(), backup_dest.get_storage_info()))) { LOG_WARN("failed to del file", K(ret)); } @@ -596,7 +596,7 @@ int ObBackupCheckFile::check_io_permission(const ObBackupDest &backup_dest) } LOG_WARN("failed to write single file", K(ret), K_(tenant_id), K(backup_dest)); } else if (FALSE_IT(write_ok = true) - || OB_FAIL(util.get_file_length(path.get_obstr(), backup_dest.get_storage_info(), file_len))) { + || OB_FAIL(util.adaptively_get_file_length(path.get_obstr(), backup_dest.get_storage_info(), file_len))) { if (is_permission_error_(ret)) { ROOTSERVICE_EVENT_ADD("connectivity_check", "permission check", "tenant_id", tenant_id_, "error_code", ret, "comment", "get file length"); @@ -606,7 +606,7 @@ int ObBackupCheckFile::check_io_permission(const ObBackupDest &backup_dest) } else if (OB_ISNULL(buf = reinterpret_cast(allocator.alloc(file_len)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to alloc buf", K(ret), K(file_len)); - } else if (OB_FAIL(util.read_single_file(path.get_obstr(), backup_dest.get_storage_info(), buf, file_len, read_size))) { + } else if (OB_FAIL(util.adaptively_read_single_file(path.get_obstr(), backup_dest.get_storage_info(), buf, file_len, read_size))) { if (is_permission_error_(ret)) { ROOTSERVICE_EVENT_ADD("connectivity_check", "permission check", "tenant_id", tenant_id_, "error_code", ret, "comment", "read single file"); @@ -614,7 +614,7 @@ int ObBackupCheckFile::check_io_permission(const ObBackupDest &backup_dest) } LOG_WARN("failed to read single file", K(ret)); } - if (write_ok && (OB_SUCCESS != (tmp_ret = util.del_file(path.get_obstr(), backup_dest.get_storage_info())))) { + if (write_ok && (OB_SUCCESS != (tmp_ret = util.adaptively_del_file(path.get_obstr(), backup_dest.get_storage_info())))) { if (is_permission_error_(tmp_ret)) { ROOTSERVICE_EVENT_ADD("connectivity_check", "permission check", "tenant_id", tenant_id_, "error_code", tmp_ret, "comment", "delete file"); @@ -672,7 +672,7 @@ int ObBackupDestCheck::check_check_file_exist_( bool need_retry = true; is_exist = false; while (retry_times--) { - if (OB_FAIL(util.is_exist(path.get_obstr(), backup_dest.get_storage_info(), is_exist))) { + if (OB_FAIL(util.adaptively_is_exist(path.get_obstr(), backup_dest.get_storage_info(), is_exist))) { LOG_WARN("failed to check is_exist", K(ret), K(path), K(backup_dest), K(retry_times)); ob_usleep(1 * 1000 * 1000L); // 1s continue; diff --git a/src/share/backup/ob_backup_io_adapter.cpp b/src/share/backup/ob_backup_io_adapter.cpp index ecef150d9..576de2068 100644 --- a/src/share/backup/ob_backup_io_adapter.cpp +++ b/src/share/backup/ob_backup_io_adapter.cpp @@ -113,7 +113,7 @@ int ObBackupIoAdapter::get_and_init_device(ObIODevice*& dev_handle, int ObBackupIoAdapter::is_exist(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, bool &exist) { int ret = OB_SUCCESS; - ObIODevice* device_handle = NULL; + ObIODevice *device_handle = NULL; if (OB_FAIL(get_and_init_device(device_handle, storage_info, uri))) { OB_LOG(WARN, "fail to get device!", K(ret), K(uri), KP(storage_info)); } else if (OB_FAIL(device_handle->exist(uri.ptr(), exist))) { @@ -123,10 +123,23 @@ int ObBackupIoAdapter::is_exist(const common::ObString &uri, const share::ObBack return ret; } +int ObBackupIoAdapter::adaptively_is_exist(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, bool &exist) +{ + int ret = OB_SUCCESS; + ObIODevice *device_handle = NULL; + if (OB_FAIL(get_and_init_device(device_handle, storage_info, uri))) { + OB_LOG(WARN, "fail to get device!", K(ret), K(uri), KP(storage_info)); + } else if (OB_FAIL(device_handle->adaptive_exist(uri.ptr(), exist))) { + OB_LOG(WARN, "fail to check exist!", K(ret), K(uri), KP(storage_info)); + } + release_device(device_handle); + return ret; +} + int ObBackupIoAdapter::get_file_length(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, int64_t &file_length) { int ret = OB_SUCCESS; - ObIODevice* device_handle = NULL; + ObIODevice *device_handle = NULL; ObIODFileStat statbuf; file_length = -1; if (OB_FAIL(get_and_init_device(device_handle, storage_info, uri))) { @@ -140,10 +153,27 @@ int ObBackupIoAdapter::get_file_length(const common::ObString &uri, const share: return ret; } +int ObBackupIoAdapter::adaptively_get_file_length(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, int64_t &file_length) +{ + int ret = OB_SUCCESS; + ObIODevice *device_handle = NULL; + ObIODFileStat statbuf; + file_length = -1; + if (OB_FAIL(get_and_init_device(device_handle, storage_info, uri))) { + OB_LOG(WARN, "fail to get device!", K(ret), K(uri), KP(storage_info)); + } else if (OB_FAIL(device_handle->adaptive_stat(uri.ptr(), statbuf))) { + OB_LOG(WARN, "fail to get file length!", K(ret), K(uri), KP(storage_info)); + } else { + file_length = statbuf.size_; + } + release_device(device_handle); + return ret; +} + int ObBackupIoAdapter::del_file(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info) { int ret = OB_SUCCESS; - ObIODevice* device_handle = NULL; + ObIODevice *device_handle = NULL; if (OB_FAIL(get_and_init_device(device_handle, storage_info, uri))) { OB_LOG(WARN, "fail to get device!", K(ret), K(uri), KP(storage_info)); } else if (OB_FAIL(device_handle->unlink(uri.ptr()))) { @@ -153,10 +183,36 @@ int ObBackupIoAdapter::del_file(const common::ObString &uri, const share::ObBack return ret; } +int ObBackupIoAdapter::adaptively_del_file(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info) +{ + int ret = OB_SUCCESS; + ObIODevice *device_handle = NULL; + if (OB_FAIL(get_and_init_device(device_handle, storage_info, uri))) { + OB_LOG(WARN, "fail to get device!", K(ret), K(uri), KP(storage_info)); + } else if (OB_FAIL(device_handle->adaptive_unlink(uri.ptr()))) { + OB_LOG(WARN, "fail to del file!", K(ret), K(uri), KP(storage_info)); + } + release_device(device_handle); + return ret; +} + +int ObBackupIoAdapter::del_unmerged_parts(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info) +{ + int ret = OB_SUCCESS; + ObIODevice *device_handle = NULL; + if (OB_FAIL(get_and_init_device(device_handle, storage_info, uri))) { + OB_LOG(WARN, "fail to get device!", K(ret), K(uri), KP(storage_info)); + } else if (OB_FAIL(device_handle->del_unmerged_parts(uri.ptr()))) { + OB_LOG(WARN, "fail to del file!", K(ret), K(uri), KP(storage_info)); + } + release_device(device_handle); + return ret; +} + int ObBackupIoAdapter::mkdir(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info) { int ret = OB_SUCCESS; - ObIODevice* device_handle = NULL; + ObIODevice *device_handle = NULL; if (OB_FAIL(get_and_init_device(device_handle, storage_info, uri))) { OB_LOG(WARN, "fail to get device!", K(ret), K(uri), KP(storage_info)); } else if (OB_FAIL(device_handle->mkdir(uri.ptr(), 0))) { @@ -171,7 +227,7 @@ int ObBackupIoAdapter::mk_parent_dir(const common::ObString &uri, const share::O { int ret = OB_SUCCESS; char path[OB_MAX_URI_LENGTH]; - ObIODevice* device_handle = NULL; + ObIODevice *device_handle = NULL; if (uri.empty()) { ret = OB_INVALID_ARGUMENT; @@ -211,7 +267,7 @@ int ObBackupIoAdapter::write_single_file(const common::ObString &uri, const shar int ret = OB_SUCCESS; int ret_tmp = OB_SUCCESS; ObIOFd fd; - ObIODevice* device_handle = NULL; + ObIODevice *device_handle = NULL; const int64_t start_ts = ObTimeUtility::current_time(); int64_t file_length = -1; int64_t write_size = -1; @@ -254,7 +310,7 @@ int ObBackupIoAdapter::read_single_file(const common::ObString &uri, const share int ret = OB_SUCCESS; int ret_tmp = OB_SUCCESS; ObIOFd fd; - ObIODevice* device_handle = NULL; + ObIODevice *device_handle = NULL; const int64_t start_ts = ObTimeUtility::current_time(); int64_t file_length = -1; @@ -263,7 +319,38 @@ int ObBackupIoAdapter::read_single_file(const common::ObString &uri, const share OB_LOG(WARN, "fail to get device and open file !", K(uri), K(ret)); } else if (OB_FAIL(device_handle->pread(fd, 0, buf_size, buf, read_size))) { OB_LOG(WARN, "failed to do read single file", K(ret), K(uri)); - } else if (OB_FAIL(get_file_size(device_handle, fd, file_length))) { + } else if (OB_FAIL(get_file_length(uri, storage_info, file_length))) { + OB_LOG(WARN, "failed to get file size", K(ret), K(uri)); + } else if (file_length != read_size) { + ret = OB_BUF_NOT_ENOUGH; + OB_LOG(WARN, "not whole file read, maybe buf not enough", + K(ret), K(read_size), K(file_length), K(uri)); + } + + if (OB_SUCCESS != (ret_tmp = close_device_and_fd(device_handle, fd))) { + ret = (OB_SUCCESS == ret) ? ret_tmp : ret; + STORAGE_LOG(WARN, "failed to close device and fd", K(ret), K(ret_tmp)); + } + + return ret; +} + +int ObBackupIoAdapter::adaptively_read_single_file(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, char *buf, + const int64_t buf_size, int64_t &read_size) +{ + int ret = OB_SUCCESS; + int ret_tmp = OB_SUCCESS; + ObIOFd fd; + ObIODevice *device_handle = NULL; + const int64_t start_ts = ObTimeUtility::current_time(); + int64_t file_length = -1; + + if (OB_FAIL(open_with_access_type(device_handle, fd, storage_info, + uri, OB_STORAGE_ACCESS_ADAPTIVE_READER))) { + OB_LOG(WARN, "fail to get device and open file !", K(uri), K(ret)); + } else if (OB_FAIL(device_handle->pread(fd, 0, buf_size, buf, read_size))) { + OB_LOG(WARN, "failed to do read single file", K(ret), K(uri)); + } else if (OB_FAIL(adaptively_get_file_length(uri, storage_info, file_length))) { OB_LOG(WARN, "failed to get file size", K(ret), K(uri)); } else if (file_length != read_size) { ret = OB_BUF_NOT_ENOUGH; @@ -295,11 +382,27 @@ int ObBackupIoAdapter::read_single_text_file(const common::ObString &uri, const return ret; } +int ObBackupIoAdapter::adaptively_read_single_text_file(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, + char *buf, const int64_t buf_size) +{ + int ret = OB_SUCCESS; + int64_t read_size = -1; + if (OB_FAIL(ObBackupIoAdapter::adaptively_read_single_file(uri, storage_info, buf, buf_size, read_size))) { + OB_LOG(WARN, "failed to read_single_object", K(ret), K(uri)); + } else if (read_size < 0 || read_size >= buf_size) { + ret = OB_BUF_NOT_ENOUGH; + OB_LOG(WARN, "buf not enough", K(ret), K(read_size), K(buf_size)); + } else { + buf[read_size] = '\0'; + } + return ret; +} + int ObBackupIoAdapter::list_files(const common::ObString &dir_path, const share::ObBackupStorageInfo *storage_info, common::ObBaseDirEntryOperator &op) { int ret = OB_SUCCESS; - ObIODevice* device_handle = NULL; + ObIODevice *device_handle = NULL; if (OB_FAIL(get_and_init_device(device_handle, storage_info, dir_path))) { OB_LOG(WARN, "fail to get device!", K(ret)); } else if (OB_FAIL(device_handle->scan_dir(dir_path.ptr(), op))) { @@ -309,6 +412,20 @@ int ObBackupIoAdapter::list_files(const common::ObString &dir_path, const share: return ret; } +int ObBackupIoAdapter::adaptively_list_files(const common::ObString &dir_path, const share::ObBackupStorageInfo *storage_info, + common::ObBaseDirEntryOperator &op) +{ + int ret = OB_SUCCESS; + ObIODevice *device_handle = NULL; + if (OB_FAIL(get_and_init_device(device_handle, storage_info, dir_path))) { + OB_LOG(WARN, "fail to get device!", K(ret)); + } else if (OB_FAIL(device_handle->adaptive_scan_dir(dir_path.ptr(), op))) { + OB_LOG(WARN, "fail to scan dir!", K(ret), K(dir_path), KP(storage_info)); + } + release_device(device_handle); + return ret; +} + int ObBackupIoAdapter::list_directories(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, common::ObBaseDirEntryOperator &op) { @@ -323,7 +440,7 @@ int ObBackupIoAdapter::list_directories(const common::ObString &uri, const share int ObBackupIoAdapter::is_tagging(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, bool &is_tagging) { int ret = OB_SUCCESS; - ObIODevice* device_handle = NULL; + ObIODevice *device_handle = NULL; if (OB_FAIL(get_and_init_device(device_handle, storage_info, uri))) { OB_LOG(WARN, "fail to get device!", K(ret), K(uri), KP(storage_info)); } else if (OB_FAIL(device_handle->is_tagging(uri.ptr(), is_tagging))) { @@ -339,7 +456,7 @@ int ObBackupIoAdapter::read_part_file(const common::ObString &uri, const share:: int ret = OB_SUCCESS; int ret_tmp = OB_SUCCESS; ObIOFd fd; - ObIODevice* device_handle = NULL; + ObIODevice *device_handle = NULL; if (OB_FAIL(open_with_access_type(device_handle, fd, storage_info, uri, OB_STORAGE_ACCESS_READER))) { @@ -355,7 +472,29 @@ int ObBackupIoAdapter::read_part_file(const common::ObString &uri, const share:: return ret; } -int ObBackupIoAdapter::get_file_size(ObIODevice* device_handle, const ObIOFd &fd, int64_t &file_length) +int ObBackupIoAdapter::adaptively_read_part_file(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, + char *buf, const int64_t buf_size, const int64_t offset, int64_t &read_size) +{ + int ret = OB_SUCCESS; + int ret_tmp = OB_SUCCESS; + ObIOFd fd; + ObIODevice*device_handle = NULL; + + if (OB_FAIL(open_with_access_type(device_handle, fd, storage_info, + uri, OB_STORAGE_ACCESS_ADAPTIVE_READER))) { + OB_LOG(WARN, "fail to get device and open file !", K(uri), K(ret), KP(storage_info)); + } else if (OB_FAIL(device_handle->pread(fd, offset, buf_size, buf, read_size))) { + OB_LOG(WARN, "fail to read file !", K(uri), K(ret), KP(storage_info)); + } + + if (OB_SUCCESS != (ret_tmp = close_device_and_fd(device_handle, fd))) { + ret = (OB_SUCCESS == ret) ? ret_tmp : ret; + STORAGE_LOG(WARN, "failed to close device and fd", K(ret), K(ret_tmp), KP(storage_info), K(uri)); + } + return ret; +} + +int ObBackupIoAdapter::get_file_size(ObIODevice *device_handle, const ObIOFd &fd, int64_t &file_length) { int ret = OB_SUCCESS; int flag = -1; @@ -364,15 +503,20 @@ int ObBackupIoAdapter::get_file_size(ObIODevice* device_handle, const ObIOFd &fd file_length = -1; ObFdSimulator::get_fd_flag(fd, flag); ObObjectDevice* obj_device_handle = (ObObjectDevice*)device_handle; - if (flag != OB_STORAGE_ACCESS_READER && flag != OB_STORAGE_ACCESS_APPENDER) { + if (OB_STORAGE_ACCESS_READER != flag + && OB_STORAGE_ACCESS_ADAPTIVE_READER != flag + && OB_STORAGE_ACCESS_APPENDER != flag ) { ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "Invaild access type, object device only support reader and appender get file size!", K(flag)); } else if (OB_FAIL(obj_device_handle->get_fd_mng().fd_to_ctx(fd, ctx))) { OB_LOG(WARN, "fail to get ctx accroding fd!", K(ret)); } else { - if (flag == OB_STORAGE_ACCESS_READER) { + if (OB_STORAGE_ACCESS_READER == flag) { ObStorageReader *reader = static_cast(ctx); file_length = reader->get_length(); + } else if (OB_STORAGE_ACCESS_ADAPTIVE_READER == flag) { + ObStorageAdaptiveReader *adaptive_reader = static_cast(ctx); + file_length = adaptive_reader->get_length(); } else { ObStorageAppender *appender = static_cast(ctx); file_length = appender->get_length(); @@ -384,7 +528,7 @@ int ObBackupIoAdapter::get_file_size(ObIODevice* device_handle, const ObIOFd &fd int ObBackupIoAdapter::del_dir(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info) { int ret = OB_SUCCESS; - ObIODevice* device_handle = NULL; + ObIODevice *device_handle = NULL; if (OB_FAIL(get_and_init_device(device_handle, storage_info, uri))) { OB_LOG(WARN, "fail to get device!", K(ret), K(uri), KP(storage_info)); } else if (OB_FAIL(device_handle->rmdir(uri.ptr()))) { @@ -397,7 +541,7 @@ int ObBackupIoAdapter::del_dir(const common::ObString &uri, const share::ObBacku class ObDelTmpFileOp : public ObBaseDirEntryOperator { public: - ObDelTmpFileOp(int64_t now_ts, char* dir_path, ObIODevice* device_handle) : + ObDelTmpFileOp(int64_t now_ts, char* dir_path, ObIODevice *device_handle) : now_ts_(now_ts), dir_path_(dir_path), device_handle_(device_handle) {} ~ObDelTmpFileOp() {} @@ -405,7 +549,7 @@ public: private: int64_t now_ts_; char* dir_path_; - ObIODevice* device_handle_; + ObIODevice *device_handle_; }; int get_tmp_file_format_timestamp(const char *file_name, @@ -506,6 +650,8 @@ int get_real_file_path(const common::ObString &uri, char *buf, const int64_t buf prefix = OB_OSS_PREFIX; } else if (OB_STORAGE_COS == device_type) { prefix = OB_COS_PREFIX; + } else if (OB_STORAGE_S3 == device_type) { + prefix = OB_S3_PREFIX; } else if (OB_STORAGE_FILE == device_type) { prefix = OB_FILE_PREFIX; } else { @@ -538,7 +684,7 @@ int get_real_file_path(const common::ObString &uri, char *buf, const int64_t buf int ObBackupIoAdapter::delete_tmp_files(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info) { int ret = OB_SUCCESS; - ObIODevice* device_handle = NULL; + ObIODevice *device_handle = NULL; if (OB_FAIL(get_and_init_device(device_handle, storage_info, uri))) { OB_LOG(WARN, "fail to get device!", K(ret), K(uri), KP(storage_info)); } else if (OB_STORAGE_FILE == device_handle->device_type_) { @@ -578,7 +724,7 @@ int ObBackupIoAdapter::is_empty_directory(const common::ObString &uri, bool &is_empty_directory) { int ret = OB_SUCCESS; - ObIODevice* device_handle = NULL; + ObIODevice *device_handle = NULL; ObCheckDirEmptOp ept_dir_op; is_empty_directory = true; if (OB_FAIL(get_and_init_device(device_handle, storage_info, uri))) { diff --git a/src/share/backup/ob_backup_io_adapter.h b/src/share/backup/ob_backup_io_adapter.h index 83464cfb7..6bd3bc908 100644 --- a/src/share/backup/ob_backup_io_adapter.h +++ b/src/share/backup/ob_backup_io_adapter.h @@ -29,22 +29,38 @@ public: explicit ObBackupIoAdapter() {} virtual ~ObBackupIoAdapter() {} int is_exist(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, bool &exist); + //TODO (@shifangdan.sfd): refine repeated logics between normal interfaces and adaptive ones + int adaptively_is_exist(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, bool &exist); int get_file_length(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, int64_t &file_length); + int adaptively_get_file_length(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, int64_t &file_length); int del_file(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info); + int adaptively_del_file(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info); + int del_unmerged_parts(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info); int mkdir(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info); int mk_parent_dir(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info); int write_single_file(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, const char *buf, const int64_t size); int read_single_file(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, char *buf,const int64_t buf_size, int64_t &read_size); + int adaptively_read_single_file(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, char *buf,const int64_t buf_size, + int64_t &read_size); int read_single_text_file(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, char *buf, const int64_t buf_size); + int adaptively_read_single_text_file(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, char *buf, const int64_t buf_size); int list_files( const common::ObString &dir_path, const share::ObBackupStorageInfo *storage_info, common::ObBaseDirEntryOperator &op); + int adaptively_list_files( + const common::ObString &dir_path, + const share::ObBackupStorageInfo *storage_info, + common::ObBaseDirEntryOperator &op); int read_part_file( const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, char *buf, const int64_t buf_size, const int64_t offset, int64_t &read_size); + int adaptively_read_part_file( + const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, + char *buf, const int64_t buf_size, const int64_t offset, + int64_t &read_size); int del_dir(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info); /*backup logical related func*/ int get_file_size(ObIODevice* device_handle, const ObIOFd &fd, int64_t &file_length); diff --git a/src/share/backup/ob_backup_struct.cpp b/src/share/backup/ob_backup_struct.cpp index 8c225b6cd..642c6c9e4 100644 --- a/src/share/backup/ob_backup_struct.cpp +++ b/src/share/backup/ob_backup_struct.cpp @@ -1171,13 +1171,13 @@ int ObBackupStorageInfo::get_access_key_(char *key_buf, const int64_t key_buf_le return ret; } -int ObBackupStorageInfo::parse_storage_info_(const char *storage_info, bool &has_appid) +int ObBackupStorageInfo::parse_storage_info_(const char *storage_info, bool &has_needed_extension) { int ret = OB_SUCCESS; if (OB_ISNULL(storage_info) || strlen(storage_info) >= OB_MAX_BACKUP_STORAGE_INFO_LENGTH) { ret = OB_INVALID_BACKUP_DEST; LOG_WARN("storage info is invalid", K(ret), K(storage_info), K(strlen(storage_info))); - } else if (OB_FAIL(ObObjectStorageInfo::parse_storage_info_(storage_info, has_appid))) { + } else if (OB_FAIL(ObObjectStorageInfo::parse_storage_info_(storage_info, has_needed_extension))) { LOG_WARN("failed to parse storage info", K(ret), K(storage_info)); } else { char tmp[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 }; diff --git a/src/share/backup/ob_backup_struct.h b/src/share/backup/ob_backup_struct.h index bf50941ec..bca2b9489 100644 --- a/src/share/backup/ob_backup_struct.h +++ b/src/share/backup/ob_backup_struct.h @@ -889,7 +889,7 @@ public: private: #ifdef OB_BUILD_TDE_SECURITY virtual int get_access_key_(char *key_buf, const int64_t key_buf_len) const override; - virtual int parse_storage_info_(const char *storage_info, bool &has_appid) override; + virtual int parse_storage_info_(const char *storage_info, bool &has_needed_extension) override; int encrypt_access_key_(char *encrypt_key, const int64_t length) const; int decrypt_access_key_(const char *buf); #endif diff --git a/src/share/ob_device_manager.cpp b/src/share/ob_device_manager.cpp index c7f69cd33..3f4a9532f 100644 --- a/src/share/ob_device_manager.cpp +++ b/src/share/ob_device_manager.cpp @@ -50,6 +50,8 @@ int ObDeviceManager::init_devices_env() OB_LOG(WARN, "fail to init oss storage", K(ret)); } else if (OB_FAIL(init_cos_env())) { OB_LOG(WARN, "fail to init cos storage", K(ret)); + } else if (OB_FAIL(init_s3_env())) { + OB_LOG(WARN, "fail to init s3 storage", K(ret)); } } @@ -84,6 +86,7 @@ void ObDeviceManager::destroy() allocator_.reset(); fin_oss_env(); fin_cos_env(); + fin_s3_env(); is_init_ = false; device_count_ = 0; OB_LOG_RET(WARN, ret_dev, "release the init resource", K(ret_dev), K(ret_handle)); @@ -120,6 +123,10 @@ int parse_storage_info(common::ObString storage_type_prefix, ObIODevice*& device device_type = OB_STORAGE_COS; mem = allocator.alloc(sizeof(ObObjectDevice)); if (NULL != mem) {new(mem)ObObjectDevice;} + } else if (storage_type_prefix.prefix_match(OB_S3_PREFIX)) { + device_type = OB_STORAGE_S3; + mem = allocator.alloc(sizeof(ObObjectDevice)); + if (NULL != mem) {new(mem)ObObjectDevice;} } else { ret = OB_INVALID_BACKUP_DEST; OB_LOG(WARN, "invaild device name info!", K(storage_type_prefix)); diff --git a/src/share/ob_errno.cpp b/src/share/ob_errno.cpp index 5c2bd39bc..b8b7d475d 100755 --- a/src/share/ob_errno.cpp +++ b/src/share/ob_errno.cpp @@ -22845,6 +22845,18 @@ static const _error _error_OB_DIRECT_LOAD_COMMIT_ERROR = { .oracle_str_error = "ORA-00600: internal error code, arguments: -9104, fail to commit direct load", .oracle_str_user_error = "ORA-00600: internal error code, arguments: -9104, fail to commit direct load" }; +static const _error _error_OB_S3_ERROR = { + .error_name = "OB_S3_ERROR", + .error_cause = "Internal Error", + .error_solution = "Contact OceanBase Support", + .mysql_errno = -1, + .sqlstate = "HY000", + .str_error = "S3 error", + .str_user_error = "S3 error", + .oracle_errno = 600, + .oracle_str_error = "ORA-00600: internal error code, arguments: -9105, S3 error", + .oracle_str_user_error = "ORA-00600: internal error code, arguments: -9105, S3 error" +}; static const _error _error_OB_ERR_RESIZE_FILE_TO_SMALLER = { .error_name = "OB_ERR_RESIZE_FILE_TO_SMALLER", .error_cause = "Internal Error", @@ -28246,6 +28258,7 @@ struct ObStrErrorInit _errors[-OB_FILE_OR_DIRECTORY_PERMISSION_DENIED] = &_error_OB_FILE_OR_DIRECTORY_PERMISSION_DENIED; _errors[-OB_TOO_MANY_OPEN_FILES] = &_error_OB_TOO_MANY_OPEN_FILES; _errors[-OB_DIRECT_LOAD_COMMIT_ERROR] = &_error_OB_DIRECT_LOAD_COMMIT_ERROR; + _errors[-OB_S3_ERROR] = &_error_OB_S3_ERROR; _errors[-OB_ERR_RESIZE_FILE_TO_SMALLER] = &_error_OB_ERR_RESIZE_FILE_TO_SMALLER; _errors[-OB_MARK_BLOCK_INFO_TIMEOUT] = &_error_OB_MARK_BLOCK_INFO_TIMEOUT; _errors[-OB_NOT_READY_TO_EXTEND_FILE] = &_error_OB_NOT_READY_TO_EXTEND_FILE; @@ -28571,7 +28584,7 @@ namespace oceanbase { namespace common { -int g_all_ob_errnos[2192] = {0, -4000, -4001, -4002, -4003, -4004, -4005, -4006, -4007, -4008, -4009, -4010, -4011, -4012, -4013, -4014, -4015, -4016, -4017, -4018, -4019, -4020, -4021, -4022, -4023, -4024, -4025, -4026, -4027, -4028, -4029, -4030, -4031, -4032, -4033, -4034, -4035, -4036, -4037, -4038, -4039, -4041, -4042, -4043, -4044, -4045, -4046, -4047, -4048, -4049, -4050, -4051, -4052, -4053, -4054, -4055, -4057, -4058, -4060, -4061, -4062, -4063, -4064, -4065, -4066, -4067, -4068, -4070, -4071, -4072, -4073, -4074, -4075, -4076, -4077, -4078, -4080, -4081, -4084, -4085, -4090, -4097, -4098, -4099, -4100, -4101, -4102, -4103, -4104, -4105, -4106, -4107, -4108, -4109, -4110, -4111, -4112, -4113, -4114, -4115, -4116, -4117, -4118, -4119, -4120, -4121, -4122, -4123, -4124, -4125, -4126, -4127, -4128, -4133, -4138, -4139, -4142, -4143, -4144, -4146, -4147, -4149, -4150, -4151, -4152, -4153, -4154, -4155, -4156, -4157, -4158, -4159, -4160, -4161, -4162, -4163, -4164, -4165, -4166, -4167, -4168, -4169, -4170, -4171, -4172, -4173, -4174, -4175, -4176, -4177, -4178, -4179, -4180, -4181, -4182, -4183, -4184, -4185, -4186, -4187, -4188, -4189, -4190, -4191, -4192, -4200, -4201, -4204, -4205, -4206, -4207, -4208, -4209, -4210, -4211, -4212, -4213, -4214, -4215, -4216, -4217, -4218, -4219, -4220, -4221, -4222, -4223, -4224, -4225, -4226, -4227, -4228, -4229, -4230, -4231, -4232, -4233, -4234, -4235, -4236, -4237, -4238, -4239, -4240, -4241, -4242, -4243, -4244, -4245, -4246, -4247, -4248, -4249, -4250, -4251, -4252, -4253, -4254, -4255, -4256, -4257, -4258, -4260, -4261, -4262, -4263, -4264, -4265, -4266, -4267, -4268, -4269, -4270, -4271, -4273, -4274, -4275, -4276, -4277, -4278, -4279, -4280, -4281, -4282, -4283, -4284, -4285, -4286, -4287, -4288, -4289, -4290, -4291, -4292, -4293, -4294, -4295, -4296, -4297, -4298, -4299, -4300, -4301, -4302, -4303, -4304, -4305, -4306, -4307, -4308, -4309, -4310, -4311, -4312, -4313, -4314, -4315, -4316, -4317, -4318, -4319, -4320, -4321, -4322, -4323, -4324, -4325, -4326, -4327, -4328, -4329, -4330, -4331, -4332, -4333, -4334, -4335, -4336, -4337, -4338, -4339, -4340, -4341, -4342, -4343, -4344, -4345, -4346, -4347, -4348, -4349, -4350, -4351, -4352, -4353, -4354, -4355, -4356, -4357, -4358, -4359, -4360, -4361, -4362, -4363, -4364, -4365, -4366, -4367, -4368, -4369, -4370, -4371, -4372, -4373, -4374, -4375, -4376, -4377, -4378, -4379, -4380, -4381, -4382, -4383, -4385, -4386, -4387, -4388, -4389, -4390, -4391, -4392, -4393, -4394, -4395, -4396, -4397, -4398, -4399, -4400, -4401, -4402, -4403, -4505, -4507, -4510, -4512, -4515, -4517, -4518, -4519, -4523, -4524, -4525, -4526, -4527, -4528, -4529, -4530, -4531, -4532, -4533, -4537, -4538, -4539, -4540, -4541, -4542, -4543, -4544, -4545, -4546, -4547, -4548, -4549, -4550, -4551, -4552, -4553, -4554, -4600, -4601, -4602, -4603, -4604, -4605, -4606, -4607, -4608, -4609, -4610, -4611, -4613, -4614, -4615, -4620, -4621, -4622, -4623, -4624, -4625, -4626, -4628, -4629, -4630, -4631, -4632, -4633, -4634, -4636, -4637, -4638, -4639, -4640, -4641, -4642, -4643, -4644, -4645, -4646, -4647, -4648, -4649, -4650, -4651, -4652, -4653, -4654, -4655, -4656, -4657, -4658, -4659, -4660, -4661, -4662, -4663, -4664, -4665, -4666, -4667, -4668, -4669, -4670, -4671, -4672, -4673, -4674, -4675, -4676, -4677, -4678, -4679, -4680, -4681, -4682, -4683, -4684, -4685, -4686, -4687, -4688, -4689, -4690, -4691, -4692, -4693, -4694, -4695, -4696, -4697, -4698, -4699, -4700, -4701, -4702, -4703, -4704, -4705, -4706, -4707, -4708, -4709, -4710, -4711, -4712, -4713, -4714, -4715, -4716, -4717, -4718, -4719, -4720, -4721, -4722, -4723, -4724, -4725, -4726, -4727, -4728, -4729, -4730, -4731, -4732, -4733, -4734, -4735, -4736, -4737, -4738, -4739, -4740, -4741, -4742, -4743, -4744, -4745, -4746, -4747, -4748, -4749, -4750, -4751, -4752, -4753, -4754, -4755, -4756, -4757, -4758, -4759, -4760, -4761, -4762, -4763, -4764, -4765, -4766, -4767, -4768, -4769, -4770, -5000, -5001, -5002, -5003, -5006, -5007, -5008, -5010, -5011, -5012, -5014, -5015, -5016, -5017, -5018, -5019, -5020, -5022, -5023, -5024, -5025, -5026, -5027, -5028, -5029, -5030, -5031, -5032, -5034, -5035, -5036, -5037, -5038, -5039, -5040, -5041, -5042, -5043, -5044, -5046, -5047, -5050, -5051, -5052, -5053, -5054, -5055, -5056, -5057, -5058, -5059, -5061, -5063, -5064, -5065, -5066, -5067, -5068, -5069, -5070, -5071, -5072, -5073, -5074, -5080, -5081, -5083, -5084, -5085, -5086, -5087, -5088, -5089, -5090, -5091, -5092, -5093, -5094, -5095, -5096, -5097, -5098, -5099, -5100, -5101, -5102, -5103, -5104, -5105, -5106, -5107, -5108, -5109, -5110, -5111, -5112, -5113, -5114, -5115, -5116, -5117, -5118, -5119, -5120, -5121, -5122, -5123, -5124, -5125, -5130, -5131, -5133, -5134, -5135, -5136, -5137, -5138, -5139, -5140, -5142, -5143, -5144, -5145, -5146, -5147, -5148, -5149, -5150, -5151, -5153, -5154, -5155, -5156, -5157, -5158, -5159, -5160, -5161, -5162, -5163, -5164, -5165, -5166, -5167, -5168, -5169, -5170, -5171, -5172, -5173, -5174, -5175, -5176, -5177, -5178, -5179, -5180, -5181, -5182, -5183, -5184, -5185, -5187, -5188, -5189, -5190, -5191, -5192, -5193, -5194, -5195, -5196, -5197, -5198, -5199, -5200, -5201, -5202, -5203, -5204, -5205, -5206, -5207, -5208, -5209, -5210, -5211, -5212, -5213, -5214, -5215, -5216, -5217, -5218, -5219, -5220, -5221, -5222, -5223, -5224, -5225, -5226, -5227, -5228, -5229, -5230, -5231, -5233, -5234, -5235, -5236, -5237, -5238, -5239, -5240, -5241, -5242, -5243, -5244, -5245, -5246, -5247, -5248, -5249, -5250, -5251, -5252, -5253, -5254, -5255, -5256, -5257, -5258, -5259, -5260, -5261, -5262, -5263, -5264, -5265, -5266, -5267, -5268, -5269, -5270, -5271, -5272, -5273, -5274, -5275, -5276, -5277, -5278, -5279, -5280, -5281, -5282, -5283, -5284, -5285, -5286, -5287, -5288, -5289, -5290, -5291, -5292, -5293, -5294, -5295, -5296, -5297, -5298, -5299, -5300, -5301, -5302, -5303, -5304, -5305, -5306, -5307, -5308, -5309, -5310, -5311, -5312, -5313, -5314, -5315, -5316, -5317, -5318, -5319, -5320, -5321, -5322, -5323, -5324, -5325, -5326, -5327, -5328, -5329, -5330, -5331, -5332, -5333, -5334, -5335, -5336, -5337, -5338, -5339, -5340, -5341, -5342, -5343, -5344, -5345, -5346, -5347, -5348, -5349, -5350, -5351, -5352, -5353, -5354, -5355, -5356, -5357, -5358, -5359, -5360, -5361, -5362, -5363, -5364, -5365, -5366, -5367, -5368, -5369, -5370, -5371, -5372, -5373, -5374, -5375, -5376, -5377, -5378, -5379, -5380, -5381, -5382, -5383, -5384, -5385, -5386, -5387, -5388, -5389, -5400, -5401, -5402, -5403, -5404, -5405, -5406, -5407, -5408, -5409, -5410, -5411, -5412, -5413, -5414, -5415, -5416, -5417, -5418, -5419, -5420, -5421, -5422, -5423, -5424, -5425, -5426, -5427, -5428, -5429, -5430, -5431, -5432, -5433, -5434, -5435, -5436, -5437, -5438, -5439, -5440, -5441, -5442, -5443, -5444, -5445, -5446, -5447, -5448, -5449, -5450, -5451, -5452, -5453, -5454, -5455, -5456, -5457, -5458, -5459, -5460, -5461, -5462, -5463, -5464, -5465, -5466, -5467, -5468, -5469, -5470, -5471, -5472, -5473, -5474, -5475, -5476, -5477, -5478, -5479, -5480, -5481, -5482, -5483, -5484, -5485, -5486, -5487, -5488, -5489, -5490, -5491, -5492, -5493, -5494, -5495, -5496, -5497, -5498, -5499, -5500, -5501, -5502, -5503, -5504, -5505, -5506, -5507, -5508, -5509, -5510, -5511, -5512, -5513, -5514, -5515, -5516, -5517, -5540, -5541, -5542, -5543, -5544, -5545, -5546, -5547, -5548, -5549, -5550, -5551, -5552, -5553, -5554, -5555, -5556, -5557, -5558, -5559, -5560, -5561, -5562, -5563, -5564, -5565, -5566, -5567, -5568, -5569, -5570, -5571, -5572, -5573, -5574, -5575, -5576, -5577, -5578, -5579, -5580, -5581, -5582, -5583, -5584, -5585, -5586, -5587, -5588, -5589, -5590, -5591, -5592, -5593, -5594, -5595, -5596, -5597, -5598, -5599, -5600, -5601, -5602, -5603, -5604, -5605, -5607, -5608, -5609, -5610, -5611, -5612, -5613, -5614, -5615, -5616, -5617, -5618, -5619, -5620, -5621, -5622, -5623, -5624, -5625, -5626, -5627, -5628, -5629, -5630, -5631, -5632, -5633, -5634, -5635, -5636, -5637, -5638, -5639, -5640, -5641, -5642, -5643, -5644, -5645, -5646, -5647, -5648, -5649, -5650, -5651, -5652, -5653, -5654, -5655, -5656, -5657, -5658, -5659, -5660, -5661, -5662, -5663, -5664, -5665, -5666, -5667, -5668, -5671, -5672, -5673, -5674, -5675, -5676, -5677, -5678, -5679, -5680, -5681, -5682, -5683, -5684, -5685, -5686, -5687, -5688, -5689, -5690, -5691, -5692, -5693, -5694, -5695, -5696, -5697, -5698, -5699, -5700, -5701, -5702, -5703, -5704, -5705, -5706, -5707, -5708, -5709, -5710, -5711, -5712, -5713, -5714, -5715, -5716, -5717, -5718, -5719, -5720, -5721, -5722, -5723, -5724, -5725, -5726, -5727, -5728, -5729, -5730, -5731, -5732, -5733, -5734, -5735, -5736, -5737, -5738, -5739, -5740, -5741, -5742, -5743, -5744, -5745, -5746, -5747, -5748, -5749, -5750, -5751, -5752, -5753, -5754, -5755, -5756, -5757, -5758, -5759, -5760, -5761, -5762, -5763, -5764, -5765, -5766, -5768, -5769, -5770, -5771, -5772, -5773, -5774, -5777, -5778, -5779, -5780, -5781, -5785, -5786, -5787, -5788, -5789, -5790, -5791, -5792, -5793, -5794, -5795, -5796, -5797, -5798, -5799, -5800, -5801, -5802, -5803, -5804, -5805, -5806, -5807, -5808, -5809, -5810, -5811, -5812, -5813, -5814, -5815, -5816, -5817, -5818, -5819, -5820, -5821, -5822, -5823, -5824, -5825, -5826, -5827, -5828, -5829, -5830, -5831, -5832, -5833, -5834, -5835, -5836, -5837, -5838, -5839, -5840, -5841, -5842, -5843, -5844, -5845, -5846, -5847, -5848, -5849, -5850, -5851, -5852, -5853, -5854, -5855, -5856, -5857, -5858, -5859, -5860, -5861, -5862, -5863, -5864, -5865, -5866, -5867, -5868, -5869, -5870, -5871, -5872, -5873, -5874, -5875, -5876, -5877, -5878, -5879, -5880, -5881, -5882, -5883, -5884, -5885, -5886, -5887, -5888, -5889, -5890, -5891, -5892, -5893, -5894, -5895, -5896, -5897, -5898, -5899, -5900, -5901, -5902, -5903, -5904, -5905, -5906, -5907, -5908, -5909, -5910, -5911, -5912, -5913, -5914, -5915, -5916, -5917, -5918, -5919, -5920, -5921, -5922, -5923, -5924, -5925, -5926, -5927, -5928, -5929, -5930, -5931, -5932, -5933, -5934, -5935, -5936, -5937, -5938, -5939, -5940, -5941, -5942, -5943, -5944, -5945, -5946, -5947, -5948, -5949, -5950, -5951, -5952, -5953, -5954, -5955, -5956, -5957, -5958, -5959, -5960, -5961, -5962, -5963, -5964, -5965, -5966, -5967, -5968, -5969, -5970, -5971, -5972, -5973, -5974, -5975, -5976, -5977, -5978, -5979, -5980, -5981, -5982, -5983, -5984, -5985, -5986, -5987, -5988, -5989, -5990, -5991, -5992, -5993, -5994, -5995, -5996, -5997, -5998, -5999, -6000, -6001, -6002, -6003, -6004, -6005, -6006, -6201, -6202, -6203, -6204, -6205, -6206, -6207, -6208, -6209, -6210, -6211, -6212, -6213, -6214, -6215, -6219, -6220, -6221, -6222, -6223, -6224, -6225, -6226, -6227, -6228, -6229, -6230, -6231, -6232, -6233, -6234, -6235, -6236, -6237, -6238, -6239, -6240, -6241, -6242, -6243, -6244, -6245, -6246, -6247, -6248, -6249, -6250, -6251, -6252, -6253, -6254, -6255, -6256, -6257, -6258, -6259, -6260, -6261, -6262, -6263, -6264, -6265, -6266, -6267, -6268, -6269, -6270, -6271, -6272, -6273, -6274, -6275, -6276, -6277, -6278, -6279, -6280, -6281, -6282, -6283, -6284, -6285, -6301, -6302, -6303, -6304, -6305, -6306, -6307, -6308, -6309, -6310, -6311, -6312, -6313, -6314, -6315, -6316, -6317, -6318, -6319, -6320, -6321, -6322, -6323, -6324, -6325, -7000, -7001, -7002, -7003, -7004, -7005, -7006, -7007, -7010, -7011, -7012, -7013, -7014, -7015, -7021, -7022, -7024, -7025, -7026, -7027, -7029, -7030, -7031, -7032, -7033, -7034, -7035, -7036, -7037, -7038, -7039, -7040, -7041, -7100, -7101, -7102, -7103, -7104, -7105, -7106, -7107, -7108, -7109, -7110, -7111, -7112, -7113, -7114, -7115, -7116, -7117, -7118, -7119, -7120, -7121, -7122, -7123, -7201, -7202, -7203, -7204, -7205, -7206, -7207, -7208, -7209, -7210, -7211, -7212, -7213, -7214, -7215, -7216, -7217, -7218, -7219, -7220, -7221, -7222, -7223, -7224, -7225, -7226, -7227, -7228, -7229, -7230, -7231, -7232, -7233, -7234, -7235, -7236, -7237, -7238, -7239, -7240, -7241, -7242, -7243, -7244, -7246, -7247, -7248, -7249, -7250, -7251, -7252, -7253, -7254, -7255, -7256, -7257, -7258, -7259, -7260, -7261, -7262, -7263, -7264, -7265, -7266, -7267, -7268, -7269, -7270, -7271, -7272, -7273, -7274, -7275, -7276, -7277, -7278, -7279, -7280, -7281, -7282, -7283, -7284, -7285, -7286, -7287, -7288, -7289, -7290, -7291, -7292, -7293, -7294, -7295, -7296, -7297, -7402, -7403, -7404, -7405, -7406, -7407, -7408, -7409, -7410, -7411, -7412, -7413, -7414, -7415, -7416, -7417, -7418, -7419, -7420, -7421, -7422, -7423, -7424, -7425, -7426, -7427, -7428, -7429, -7430, -7431, -7432, -7433, -8001, -8002, -8003, -8004, -8005, -9001, -9002, -9003, -9004, -9005, -9006, -9007, -9008, -9009, -9010, -9011, -9012, -9013, -9014, -9015, -9016, -9017, -9018, -9019, -9020, -9022, -9023, -9024, -9025, -9026, -9027, -9028, -9029, -9030, -9031, -9032, -9033, -9034, -9035, -9036, -9037, -9038, -9039, -9040, -9041, -9042, -9043, -9044, -9045, -9046, -9047, -9048, -9049, -9050, -9051, -9052, -9053, -9054, -9057, -9058, -9059, -9060, -9061, -9062, -9063, -9064, -9065, -9066, -9069, -9070, -9071, -9072, -9073, -9074, -9075, -9076, -9077, -9078, -9079, -9080, -9081, -9082, -9083, -9084, -9085, -9086, -9087, -9088, -9089, -9090, -9091, -9092, -9093, -9094, -9095, -9096, -9097, -9098, -9099, -9100, -9101, -9102, -9103, -9104, -9200, -9201, -9202, -9501, -9502, -9503, -9504, -9505, -9506, -9507, -9508, -9509, -9510, -9512, -9513, -9514, -9515, -9516, -9518, -9519, -9520, -9521, -9522, -9523, -9524, -9525, -9526, -9527, -9528, -9529, -9530, -9531, -9532, -9533, -9534, -9535, -9536, -9537, -9538, -9539, -9540, -9541, -9542, -9543, -9544, -9545, -9546, -9547, -9548, -9549, -9550, -9551, -9552, -9553, -9554, -9555, -9556, -9557, -9558, -9559, -9560, -9561, -9562, -9563, -9564, -9565, -9566, -9567, -9568, -9569, -9570, -9571, -9572, -9573, -9574, -9575, -9576, -9577, -9578, -9579, -9580, -9581, -9582, -9583, -9584, -9585, -9586, -9587, -9588, -9589, -9590, -9591, -9592, -9593, -9594, -9595, -9596, -9597, -9598, -9599, -9600, -9601, -9602, -9603, -9604, -9605, -9606, -9607, -9608, -9609, -9610, -9611, -9612, -9613, -9614, -9615, -9616, -9617, -9618, -9619, -9620, -9621, -9622, -9623, -9624, -9625, -9626, -9627, -9628, -9629, -9630, -9631, -9632, -9633, -9634, -9635, -9636, -9637, -9638, -9639, -9640, -9641, -9642, -9643, -9644, -9645, -9646, -9647, -9648, -9649, -9650, -9651, -9652, -9653, -9654, -9655, -9656, -9657, -9658, -9659, -9660, -9661, -9662, -9663, -9664, -9665, -9666, -9667, -9668, -9669, -9670, -9671, -9672, -9673, -9674, -9675, -9676, -9677, -9678, -9679, -9680, -9681, -9682, -9683, -9684, -9685, -9686, -9687, -9688, -9689, -9690, -9691, -9692, -9693, -9694, -9695, -9696, -9697, -9698, -9699, -9700, -9701, -9702, -9703, -9704, -9705, -9706, -9707, -9708, -9709, -9710, -9711, -9712, -9713, -9714, -9715, -9716, -9717, -9718, -9719, -9720, -9721, -9722, -9723, -9724, -9725, -9726, -9727, -9728, -9729, -9730, -9731, -9732, -9733, -9734, -9735, -9736, -9737, -9738, -9739, -9740, -9741, -9742, -9743, -9744, -9745, -9746, -9747, -9748, -9749, -9750, -9751, -9752, -9753, -9754, -9755, -9756, -9757, -9758, -9759, -9760, -9761, -9762, -9763, -9764, -10500, -10501, -10502, -10503, -10504, -10505, -10506, -10507, -10508, -11000, -11001, -11002, -11003, -11005, -11006, -11007, -11008, -11009, -11010, -20000, -21000, -22998, -30926, -32491, -38104, -38105}; +int g_all_ob_errnos[2193] = {0, -4000, -4001, -4002, -4003, -4004, -4005, -4006, -4007, -4008, -4009, -4010, -4011, -4012, -4013, -4014, -4015, -4016, -4017, -4018, -4019, -4020, -4021, -4022, -4023, -4024, -4025, -4026, -4027, -4028, -4029, -4030, -4031, -4032, -4033, -4034, -4035, -4036, -4037, -4038, -4039, -4041, -4042, -4043, -4044, -4045, -4046, -4047, -4048, -4049, -4050, -4051, -4052, -4053, -4054, -4055, -4057, -4058, -4060, -4061, -4062, -4063, -4064, -4065, -4066, -4067, -4068, -4070, -4071, -4072, -4073, -4074, -4075, -4076, -4077, -4078, -4080, -4081, -4084, -4085, -4090, -4097, -4098, -4099, -4100, -4101, -4102, -4103, -4104, -4105, -4106, -4107, -4108, -4109, -4110, -4111, -4112, -4113, -4114, -4115, -4116, -4117, -4118, -4119, -4120, -4121, -4122, -4123, -4124, -4125, -4126, -4127, -4128, -4133, -4138, -4139, -4142, -4143, -4144, -4146, -4147, -4149, -4150, -4151, -4152, -4153, -4154, -4155, -4156, -4157, -4158, -4159, -4160, -4161, -4162, -4163, -4164, -4165, -4166, -4167, -4168, -4169, -4170, -4171, -4172, -4173, -4174, -4175, -4176, -4177, -4178, -4179, -4180, -4181, -4182, -4183, -4184, -4185, -4186, -4187, -4188, -4189, -4190, -4191, -4192, -4200, -4201, -4204, -4205, -4206, -4207, -4208, -4209, -4210, -4211, -4212, -4213, -4214, -4215, -4216, -4217, -4218, -4219, -4220, -4221, -4222, -4223, -4224, -4225, -4226, -4227, -4228, -4229, -4230, -4231, -4232, -4233, -4234, -4235, -4236, -4237, -4238, -4239, -4240, -4241, -4242, -4243, -4244, -4245, -4246, -4247, -4248, -4249, -4250, -4251, -4252, -4253, -4254, -4255, -4256, -4257, -4258, -4260, -4261, -4262, -4263, -4264, -4265, -4266, -4267, -4268, -4269, -4270, -4271, -4273, -4274, -4275, -4276, -4277, -4278, -4279, -4280, -4281, -4282, -4283, -4284, -4285, -4286, -4287, -4288, -4289, -4290, -4291, -4292, -4293, -4294, -4295, -4296, -4297, -4298, -4299, -4300, -4301, -4302, -4303, -4304, -4305, -4306, -4307, -4308, -4309, -4310, -4311, -4312, -4313, -4314, -4315, -4316, -4317, -4318, -4319, -4320, -4321, -4322, -4323, -4324, -4325, -4326, -4327, -4328, -4329, -4330, -4331, -4332, -4333, -4334, -4335, -4336, -4337, -4338, -4339, -4340, -4341, -4342, -4343, -4344, -4345, -4346, -4347, -4348, -4349, -4350, -4351, -4352, -4353, -4354, -4355, -4356, -4357, -4358, -4359, -4360, -4361, -4362, -4363, -4364, -4365, -4366, -4367, -4368, -4369, -4370, -4371, -4372, -4373, -4374, -4375, -4376, -4377, -4378, -4379, -4380, -4381, -4382, -4383, -4385, -4386, -4387, -4388, -4389, -4390, -4391, -4392, -4393, -4394, -4395, -4396, -4397, -4398, -4399, -4400, -4401, -4402, -4403, -4505, -4507, -4510, -4512, -4515, -4517, -4518, -4519, -4523, -4524, -4525, -4526, -4527, -4528, -4529, -4530, -4531, -4532, -4533, -4537, -4538, -4539, -4540, -4541, -4542, -4543, -4544, -4545, -4546, -4547, -4548, -4549, -4550, -4551, -4552, -4553, -4554, -4600, -4601, -4602, -4603, -4604, -4605, -4606, -4607, -4608, -4609, -4610, -4611, -4613, -4614, -4615, -4620, -4621, -4622, -4623, -4624, -4625, -4626, -4628, -4629, -4630, -4631, -4632, -4633, -4634, -4636, -4637, -4638, -4639, -4640, -4641, -4642, -4643, -4644, -4645, -4646, -4647, -4648, -4649, -4650, -4651, -4652, -4653, -4654, -4655, -4656, -4657, -4658, -4659, -4660, -4661, -4662, -4663, -4664, -4665, -4666, -4667, -4668, -4669, -4670, -4671, -4672, -4673, -4674, -4675, -4676, -4677, -4678, -4679, -4680, -4681, -4682, -4683, -4684, -4685, -4686, -4687, -4688, -4689, -4690, -4691, -4692, -4693, -4694, -4695, -4696, -4697, -4698, -4699, -4700, -4701, -4702, -4703, -4704, -4705, -4706, -4707, -4708, -4709, -4710, -4711, -4712, -4713, -4714, -4715, -4716, -4717, -4718, -4719, -4720, -4721, -4722, -4723, -4724, -4725, -4726, -4727, -4728, -4729, -4730, -4731, -4732, -4733, -4734, -4735, -4736, -4737, -4738, -4739, -4740, -4741, -4742, -4743, -4744, -4745, -4746, -4747, -4748, -4749, -4750, -4751, -4752, -4753, -4754, -4755, -4756, -4757, -4758, -4759, -4760, -4761, -4762, -4763, -4764, -4765, -4766, -4767, -4768, -4769, -4770, -5000, -5001, -5002, -5003, -5006, -5007, -5008, -5010, -5011, -5012, -5014, -5015, -5016, -5017, -5018, -5019, -5020, -5022, -5023, -5024, -5025, -5026, -5027, -5028, -5029, -5030, -5031, -5032, -5034, -5035, -5036, -5037, -5038, -5039, -5040, -5041, -5042, -5043, -5044, -5046, -5047, -5050, -5051, -5052, -5053, -5054, -5055, -5056, -5057, -5058, -5059, -5061, -5063, -5064, -5065, -5066, -5067, -5068, -5069, -5070, -5071, -5072, -5073, -5074, -5080, -5081, -5083, -5084, -5085, -5086, -5087, -5088, -5089, -5090, -5091, -5092, -5093, -5094, -5095, -5096, -5097, -5098, -5099, -5100, -5101, -5102, -5103, -5104, -5105, -5106, -5107, -5108, -5109, -5110, -5111, -5112, -5113, -5114, -5115, -5116, -5117, -5118, -5119, -5120, -5121, -5122, -5123, -5124, -5125, -5130, -5131, -5133, -5134, -5135, -5136, -5137, -5138, -5139, -5140, -5142, -5143, -5144, -5145, -5146, -5147, -5148, -5149, -5150, -5151, -5153, -5154, -5155, -5156, -5157, -5158, -5159, -5160, -5161, -5162, -5163, -5164, -5165, -5166, -5167, -5168, -5169, -5170, -5171, -5172, -5173, -5174, -5175, -5176, -5177, -5178, -5179, -5180, -5181, -5182, -5183, -5184, -5185, -5187, -5188, -5189, -5190, -5191, -5192, -5193, -5194, -5195, -5196, -5197, -5198, -5199, -5200, -5201, -5202, -5203, -5204, -5205, -5206, -5207, -5208, -5209, -5210, -5211, -5212, -5213, -5214, -5215, -5216, -5217, -5218, -5219, -5220, -5221, -5222, -5223, -5224, -5225, -5226, -5227, -5228, -5229, -5230, -5231, -5233, -5234, -5235, -5236, -5237, -5238, -5239, -5240, -5241, -5242, -5243, -5244, -5245, -5246, -5247, -5248, -5249, -5250, -5251, -5252, -5253, -5254, -5255, -5256, -5257, -5258, -5259, -5260, -5261, -5262, -5263, -5264, -5265, -5266, -5267, -5268, -5269, -5270, -5271, -5272, -5273, -5274, -5275, -5276, -5277, -5278, -5279, -5280, -5281, -5282, -5283, -5284, -5285, -5286, -5287, -5288, -5289, -5290, -5291, -5292, -5293, -5294, -5295, -5296, -5297, -5298, -5299, -5300, -5301, -5302, -5303, -5304, -5305, -5306, -5307, -5308, -5309, -5310, -5311, -5312, -5313, -5314, -5315, -5316, -5317, -5318, -5319, -5320, -5321, -5322, -5323, -5324, -5325, -5326, -5327, -5328, -5329, -5330, -5331, -5332, -5333, -5334, -5335, -5336, -5337, -5338, -5339, -5340, -5341, -5342, -5343, -5344, -5345, -5346, -5347, -5348, -5349, -5350, -5351, -5352, -5353, -5354, -5355, -5356, -5357, -5358, -5359, -5360, -5361, -5362, -5363, -5364, -5365, -5366, -5367, -5368, -5369, -5370, -5371, -5372, -5373, -5374, -5375, -5376, -5377, -5378, -5379, -5380, -5381, -5382, -5383, -5384, -5385, -5386, -5387, -5388, -5389, -5400, -5401, -5402, -5403, -5404, -5405, -5406, -5407, -5408, -5409, -5410, -5411, -5412, -5413, -5414, -5415, -5416, -5417, -5418, -5419, -5420, -5421, -5422, -5423, -5424, -5425, -5426, -5427, -5428, -5429, -5430, -5431, -5432, -5433, -5434, -5435, -5436, -5437, -5438, -5439, -5440, -5441, -5442, -5443, -5444, -5445, -5446, -5447, -5448, -5449, -5450, -5451, -5452, -5453, -5454, -5455, -5456, -5457, -5458, -5459, -5460, -5461, -5462, -5463, -5464, -5465, -5466, -5467, -5468, -5469, -5470, -5471, -5472, -5473, -5474, -5475, -5476, -5477, -5478, -5479, -5480, -5481, -5482, -5483, -5484, -5485, -5486, -5487, -5488, -5489, -5490, -5491, -5492, -5493, -5494, -5495, -5496, -5497, -5498, -5499, -5500, -5501, -5502, -5503, -5504, -5505, -5506, -5507, -5508, -5509, -5510, -5511, -5512, -5513, -5514, -5515, -5516, -5517, -5540, -5541, -5542, -5543, -5544, -5545, -5546, -5547, -5548, -5549, -5550, -5551, -5552, -5553, -5554, -5555, -5556, -5557, -5558, -5559, -5560, -5561, -5562, -5563, -5564, -5565, -5566, -5567, -5568, -5569, -5570, -5571, -5572, -5573, -5574, -5575, -5576, -5577, -5578, -5579, -5580, -5581, -5582, -5583, -5584, -5585, -5586, -5587, -5588, -5589, -5590, -5591, -5592, -5593, -5594, -5595, -5596, -5597, -5598, -5599, -5600, -5601, -5602, -5603, -5604, -5605, -5607, -5608, -5609, -5610, -5611, -5612, -5613, -5614, -5615, -5616, -5617, -5618, -5619, -5620, -5621, -5622, -5623, -5624, -5625, -5626, -5627, -5628, -5629, -5630, -5631, -5632, -5633, -5634, -5635, -5636, -5637, -5638, -5639, -5640, -5641, -5642, -5643, -5644, -5645, -5646, -5647, -5648, -5649, -5650, -5651, -5652, -5653, -5654, -5655, -5656, -5657, -5658, -5659, -5660, -5661, -5662, -5663, -5664, -5665, -5666, -5667, -5668, -5671, -5672, -5673, -5674, -5675, -5676, -5677, -5678, -5679, -5680, -5681, -5682, -5683, -5684, -5685, -5686, -5687, -5688, -5689, -5690, -5691, -5692, -5693, -5694, -5695, -5696, -5697, -5698, -5699, -5700, -5701, -5702, -5703, -5704, -5705, -5706, -5707, -5708, -5709, -5710, -5711, -5712, -5713, -5714, -5715, -5716, -5717, -5718, -5719, -5720, -5721, -5722, -5723, -5724, -5725, -5726, -5727, -5728, -5729, -5730, -5731, -5732, -5733, -5734, -5735, -5736, -5737, -5738, -5739, -5740, -5741, -5742, -5743, -5744, -5745, -5746, -5747, -5748, -5749, -5750, -5751, -5752, -5753, -5754, -5755, -5756, -5757, -5758, -5759, -5760, -5761, -5762, -5763, -5764, -5765, -5766, -5768, -5769, -5770, -5771, -5772, -5773, -5774, -5777, -5778, -5779, -5780, -5781, -5785, -5786, -5787, -5788, -5789, -5790, -5791, -5792, -5793, -5794, -5795, -5796, -5797, -5798, -5799, -5800, -5801, -5802, -5803, -5804, -5805, -5806, -5807, -5808, -5809, -5810, -5811, -5812, -5813, -5814, -5815, -5816, -5817, -5818, -5819, -5820, -5821, -5822, -5823, -5824, -5825, -5826, -5827, -5828, -5829, -5830, -5831, -5832, -5833, -5834, -5835, -5836, -5837, -5838, -5839, -5840, -5841, -5842, -5843, -5844, -5845, -5846, -5847, -5848, -5849, -5850, -5851, -5852, -5853, -5854, -5855, -5856, -5857, -5858, -5859, -5860, -5861, -5862, -5863, -5864, -5865, -5866, -5867, -5868, -5869, -5870, -5871, -5872, -5873, -5874, -5875, -5876, -5877, -5878, -5879, -5880, -5881, -5882, -5883, -5884, -5885, -5886, -5887, -5888, -5889, -5890, -5891, -5892, -5893, -5894, -5895, -5896, -5897, -5898, -5899, -5900, -5901, -5902, -5903, -5904, -5905, -5906, -5907, -5908, -5909, -5910, -5911, -5912, -5913, -5914, -5915, -5916, -5917, -5918, -5919, -5920, -5921, -5922, -5923, -5924, -5925, -5926, -5927, -5928, -5929, -5930, -5931, -5932, -5933, -5934, -5935, -5936, -5937, -5938, -5939, -5940, -5941, -5942, -5943, -5944, -5945, -5946, -5947, -5948, -5949, -5950, -5951, -5952, -5953, -5954, -5955, -5956, -5957, -5958, -5959, -5960, -5961, -5962, -5963, -5964, -5965, -5966, -5967, -5968, -5969, -5970, -5971, -5972, -5973, -5974, -5975, -5976, -5977, -5978, -5979, -5980, -5981, -5982, -5983, -5984, -5985, -5986, -5987, -5988, -5989, -5990, -5991, -5992, -5993, -5994, -5995, -5996, -5997, -5998, -5999, -6000, -6001, -6002, -6003, -6004, -6005, -6006, -6201, -6202, -6203, -6204, -6205, -6206, -6207, -6208, -6209, -6210, -6211, -6212, -6213, -6214, -6215, -6219, -6220, -6221, -6222, -6223, -6224, -6225, -6226, -6227, -6228, -6229, -6230, -6231, -6232, -6233, -6234, -6235, -6236, -6237, -6238, -6239, -6240, -6241, -6242, -6243, -6244, -6245, -6246, -6247, -6248, -6249, -6250, -6251, -6252, -6253, -6254, -6255, -6256, -6257, -6258, -6259, -6260, -6261, -6262, -6263, -6264, -6265, -6266, -6267, -6268, -6269, -6270, -6271, -6272, -6273, -6274, -6275, -6276, -6277, -6278, -6279, -6280, -6281, -6282, -6283, -6284, -6285, -6301, -6302, -6303, -6304, -6305, -6306, -6307, -6308, -6309, -6310, -6311, -6312, -6313, -6314, -6315, -6316, -6317, -6318, -6319, -6320, -6321, -6322, -6323, -6324, -6325, -7000, -7001, -7002, -7003, -7004, -7005, -7006, -7007, -7010, -7011, -7012, -7013, -7014, -7015, -7021, -7022, -7024, -7025, -7026, -7027, -7029, -7030, -7031, -7032, -7033, -7034, -7035, -7036, -7037, -7038, -7039, -7040, -7041, -7100, -7101, -7102, -7103, -7104, -7105, -7106, -7107, -7108, -7109, -7110, -7111, -7112, -7113, -7114, -7115, -7116, -7117, -7118, -7119, -7120, -7121, -7122, -7123, -7201, -7202, -7203, -7204, -7205, -7206, -7207, -7208, -7209, -7210, -7211, -7212, -7213, -7214, -7215, -7216, -7217, -7218, -7219, -7220, -7221, -7222, -7223, -7224, -7225, -7226, -7227, -7228, -7229, -7230, -7231, -7232, -7233, -7234, -7235, -7236, -7237, -7238, -7239, -7240, -7241, -7242, -7243, -7244, -7246, -7247, -7248, -7249, -7250, -7251, -7252, -7253, -7254, -7255, -7256, -7257, -7258, -7259, -7260, -7261, -7262, -7263, -7264, -7265, -7266, -7267, -7268, -7269, -7270, -7271, -7272, -7273, -7274, -7275, -7276, -7277, -7278, -7279, -7280, -7281, -7282, -7283, -7284, -7285, -7286, -7287, -7288, -7289, -7290, -7291, -7292, -7293, -7294, -7295, -7296, -7297, -7402, -7403, -7404, -7405, -7406, -7407, -7408, -7409, -7410, -7411, -7412, -7413, -7414, -7415, -7416, -7417, -7418, -7419, -7420, -7421, -7422, -7423, -7424, -7425, -7426, -7427, -7428, -7429, -7430, -7431, -7432, -7433, -8001, -8002, -8003, -8004, -8005, -9001, -9002, -9003, -9004, -9005, -9006, -9007, -9008, -9009, -9010, -9011, -9012, -9013, -9014, -9015, -9016, -9017, -9018, -9019, -9020, -9022, -9023, -9024, -9025, -9026, -9027, -9028, -9029, -9030, -9031, -9032, -9033, -9034, -9035, -9036, -9037, -9038, -9039, -9040, -9041, -9042, -9043, -9044, -9045, -9046, -9047, -9048, -9049, -9050, -9051, -9052, -9053, -9054, -9057, -9058, -9059, -9060, -9061, -9062, -9063, -9064, -9065, -9066, -9069, -9070, -9071, -9072, -9073, -9074, -9075, -9076, -9077, -9078, -9079, -9080, -9081, -9082, -9083, -9084, -9085, -9086, -9087, -9088, -9089, -9090, -9091, -9092, -9093, -9094, -9095, -9096, -9097, -9098, -9099, -9100, -9101, -9102, -9103, -9104, -9105, -9200, -9201, -9202, -9501, -9502, -9503, -9504, -9505, -9506, -9507, -9508, -9509, -9510, -9512, -9513, -9514, -9515, -9516, -9518, -9519, -9520, -9521, -9522, -9523, -9524, -9525, -9526, -9527, -9528, -9529, -9530, -9531, -9532, -9533, -9534, -9535, -9536, -9537, -9538, -9539, -9540, -9541, -9542, -9543, -9544, -9545, -9546, -9547, -9548, -9549, -9550, -9551, -9552, -9553, -9554, -9555, -9556, -9557, -9558, -9559, -9560, -9561, -9562, -9563, -9564, -9565, -9566, -9567, -9568, -9569, -9570, -9571, -9572, -9573, -9574, -9575, -9576, -9577, -9578, -9579, -9580, -9581, -9582, -9583, -9584, -9585, -9586, -9587, -9588, -9589, -9590, -9591, -9592, -9593, -9594, -9595, -9596, -9597, -9598, -9599, -9600, -9601, -9602, -9603, -9604, -9605, -9606, -9607, -9608, -9609, -9610, -9611, -9612, -9613, -9614, -9615, -9616, -9617, -9618, -9619, -9620, -9621, -9622, -9623, -9624, -9625, -9626, -9627, -9628, -9629, -9630, -9631, -9632, -9633, -9634, -9635, -9636, -9637, -9638, -9639, -9640, -9641, -9642, -9643, -9644, -9645, -9646, -9647, -9648, -9649, -9650, -9651, -9652, -9653, -9654, -9655, -9656, -9657, -9658, -9659, -9660, -9661, -9662, -9663, -9664, -9665, -9666, -9667, -9668, -9669, -9670, -9671, -9672, -9673, -9674, -9675, -9676, -9677, -9678, -9679, -9680, -9681, -9682, -9683, -9684, -9685, -9686, -9687, -9688, -9689, -9690, -9691, -9692, -9693, -9694, -9695, -9696, -9697, -9698, -9699, -9700, -9701, -9702, -9703, -9704, -9705, -9706, -9707, -9708, -9709, -9710, -9711, -9712, -9713, -9714, -9715, -9716, -9717, -9718, -9719, -9720, -9721, -9722, -9723, -9724, -9725, -9726, -9727, -9728, -9729, -9730, -9731, -9732, -9733, -9734, -9735, -9736, -9737, -9738, -9739, -9740, -9741, -9742, -9743, -9744, -9745, -9746, -9747, -9748, -9749, -9750, -9751, -9752, -9753, -9754, -9755, -9756, -9757, -9758, -9759, -9760, -9761, -9762, -9763, -9764, -10500, -10501, -10502, -10503, -10504, -10505, -10506, -10507, -10508, -11000, -11001, -11002, -11003, -11005, -11006, -11007, -11008, -11009, -11010, -20000, -21000, -22998, -30926, -32491, -38104, -38105}; const char *ob_error_name(const int err) { const char *ret = "Unknown error"; diff --git a/src/share/ob_errno.def b/src/share/ob_errno.def index 2655560ac..1a9c2b9e2 100755 --- a/src/share/ob_errno.def +++ b/src/share/ob_errno.def @@ -2111,6 +2111,7 @@ DEFINE_ERROR(OB_FILE_OR_DIRECTORY_EXIST, -9101, -1, "HY000", "file or directory DEFINE_ERROR(OB_FILE_OR_DIRECTORY_PERMISSION_DENIED, -9102, -1, "HY000", "file or directory permission denied"); DEFINE_ERROR(OB_TOO_MANY_OPEN_FILES, -9103, -1, "HY000", "too many open files"); DEFINE_ERROR(OB_DIRECT_LOAD_COMMIT_ERROR, -9104, -1, "HY000", "fail to commit direct load"); +DEFINE_ERROR_DEP(OB_S3_ERROR, -9105, -1, "HY000", "S3 error"); //////////////////////////////////////////////////////////////// //error code for storage auto extend file diff --git a/src/share/ob_errno.h b/src/share/ob_errno.h index f69bba5ed..aa07d1ff0 100755 --- a/src/share/ob_errno.h +++ b/src/share/ob_errno.h @@ -3743,6 +3743,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_FILE_OR_DIRECTORY_PERMISSION_DENIED__USER_ERROR_MSG "file or directory permission denied" #define OB_TOO_MANY_OPEN_FILES__USER_ERROR_MSG "too many open files" #define OB_DIRECT_LOAD_COMMIT_ERROR__USER_ERROR_MSG "fail to commit direct load" +#define OB_S3_ERROR__USER_ERROR_MSG "S3 error" #define OB_ERR_RESIZE_FILE_TO_SMALLER__USER_ERROR_MSG "Extend ssblock file to smaller is not allowed" #define OB_MARK_BLOCK_INFO_TIMEOUT__USER_ERROR_MSG "Mark blocks timeout(5s) in auto extend process when alloc block fail" #define OB_NOT_READY_TO_EXTEND_FILE__USER_ERROR_MSG "Auto extend param is not ready to start extending file" @@ -5939,6 +5940,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_FILE_OR_DIRECTORY_PERMISSION_DENIED__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9102, file or directory permission denied" #define OB_TOO_MANY_OPEN_FILES__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9103, too many open files" #define OB_DIRECT_LOAD_COMMIT_ERROR__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9104, fail to commit direct load" +#define OB_S3_ERROR__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9105, S3 error" #define OB_ERR_RESIZE_FILE_TO_SMALLER__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9200, Extend ssblock file to smaller is not allowed" #define OB_MARK_BLOCK_INFO_TIMEOUT__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9201, Mark blocks timeout(5s) in auto extend process when alloc block fail" #define OB_NOT_READY_TO_EXTEND_FILE__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9202, Auto extend param is not ready to start extending file" @@ -6233,7 +6235,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_DATA_TOO_LONG_MSG_FMT_V2__ORA_USER_ERROR_MSG "ORA-12899: value too large for column %.*s (actual: %ld, maximum: %ld)" #define OB_ERR_INVALID_DATE_MSG_FMT_V2__ORA_USER_ERROR_MSG "ORA-01861: Incorrect datetime value for column '%.*s' at row %ld" -extern int g_all_ob_errnos[2192]; +extern int g_all_ob_errnos[2193]; const char *ob_error_name(const int oberr); const char* ob_error_cause(const int oberr); diff --git a/src/share/ob_local_device.cpp b/src/share/ob_local_device.cpp index 3fd5881bf..18ad3ba48 100644 --- a/src/share/ob_local_device.cpp +++ b/src/share/ob_local_device.cpp @@ -645,6 +645,39 @@ int ObLocalDevice::fstat(const ObIOFd &fd, ObIODFileStat &statbuf) return ret; } +int ObLocalDevice::del_unmerged_parts(const char *pathname) +{ + UNUSED(pathname); + return OB_NOT_SUPPORTED; +} + +int ObLocalDevice::adaptive_exist(const char *pathname, bool &is_exist) +{ + UNUSED(pathname); + UNUSED(is_exist); + return OB_NOT_SUPPORTED; +} + +int ObLocalDevice::adaptive_stat(const char *pathname, ObIODFileStat &statbuf) +{ + UNUSED(pathname); + UNUSED(statbuf); + return OB_NOT_SUPPORTED; +} + +int ObLocalDevice::adaptive_unlink(const char *pathname) +{ + UNUSED(pathname); + return OB_NOT_SUPPORTED; +} + +int ObLocalDevice::adaptive_scan_dir(const char *dir_name, ObBaseDirEntryOperator &op) +{ + UNUSED(dir_name); + UNUSED(op); + return OB_NOT_SUPPORTED; +} + //block interfaces int ObLocalDevice::mark_blocks(ObIBlockIterator &block_iter) { diff --git a/src/share/ob_local_device.h b/src/share/ob_local_device.h index 9ba67b229..2a976c531 100644 --- a/src/share/ob_local_device.h +++ b/src/share/ob_local_device.h @@ -88,6 +88,13 @@ public: virtual int stat(const char *pathname, common::ObIODFileStat &statbuf) override; virtual int fstat(const common::ObIOFd &fd, common::ObIODFileStat &statbuf) override; + //for object device, local device should not use these + int del_unmerged_parts(const char *pathname); + int adaptive_exist(const char *pathname, bool &is_exist); + int adaptive_stat(const char *pathname, ObIODFileStat &statbuf); + int adaptive_unlink(const char *pathname); + int adaptive_scan_dir(const char *dir_name, ObBaseDirEntryOperator &op); + //block interfaces virtual int mark_blocks(common::ObIBlockIterator &block_iter) override; virtual int alloc_block(const common::ObIODOpts *opts, common::ObIOFd &block_id) override; diff --git a/src/storage/backup/ob_backup_ctx.cpp b/src/storage/backup/ob_backup_ctx.cpp index be585b65d..655409edd 100644 --- a/src/storage/backup/ob_backup_ctx.cpp +++ b/src/storage/backup/ob_backup_ctx.cpp @@ -400,7 +400,7 @@ int ObBackupDataCtx::open_file_writer_(const share::ObBackupPath &backup_path) { int ret = OB_SUCCESS; common::ObBackupIoAdapter util; - const ObStorageAccessType access_type = OB_STORAGE_ACCESS_RANDOMWRITER; + const ObStorageAccessType access_type = OB_STORAGE_ACCESS_MULTIPART_WRITER; if (OB_FAIL(util.mk_parent_dir(backup_path.get_obstr(), param_.backup_dest_.get_storage_info()))) { LOG_WARN("failed to make parent dir", K(backup_path)); } else if (OB_FAIL(util.open_with_access_type( diff --git a/src/storage/backup/ob_backup_extern_info_mgr.cpp b/src/storage/backup/ob_backup_extern_info_mgr.cpp index aed42ed21..981747e4f 100644 --- a/src/storage/backup/ob_backup_extern_info_mgr.cpp +++ b/src/storage/backup/ob_backup_extern_info_mgr.cpp @@ -371,7 +371,7 @@ int ObExternTabletMetaWriter::prepare_backup_file_(const int64_t file_id) int ret = OB_SUCCESS; share::ObBackupPath backup_path; common::ObBackupIoAdapter util; - const ObStorageAccessType access_type = OB_STORAGE_ACCESS_RANDOMWRITER; + const ObStorageAccessType access_type = OB_STORAGE_ACCESS_MULTIPART_WRITER; const int64_t data_file_size = get_data_file_size(); if (OB_FAIL(ObBackupPathUtil::get_ls_data_tablet_info_path( backup_set_dest_, ls_id_, turn_id_, retry_id_, file_id, backup_path))) { diff --git a/src/storage/backup/ob_backup_index_merger.cpp b/src/storage/backup/ob_backup_index_merger.cpp index fac516006..5c39109eb 100644 --- a/src/storage/backup/ob_backup_index_merger.cpp +++ b/src/storage/backup/ob_backup_index_merger.cpp @@ -572,7 +572,7 @@ int ObIBackupIndexMerger::open_file_writer_(const share::ObBackupPath &path, con { int ret = OB_SUCCESS; common::ObBackupIoAdapter util; - const ObStorageAccessType access_type = OB_STORAGE_ACCESS_RANDOMWRITER; + const ObStorageAccessType access_type = OB_STORAGE_ACCESS_MULTIPART_WRITER; if (OB_FAIL(util.mk_parent_dir(path.get_obstr(), storage_info))) { LOG_WARN("failed to make parent dir", K(path), K(path), KP(storage_info)); } else if (OB_FAIL(util.open_with_access_type(dev_handle_, io_fd_, storage_info, path.get_obstr(), access_type))) { diff --git a/src/storage/backup/ob_backup_task.cpp b/src/storage/backup/ob_backup_task.cpp index 1c0b398da..8f3fb9618 100644 --- a/src/storage/backup/ob_backup_task.cpp +++ b/src/storage/backup/ob_backup_task.cpp @@ -5280,7 +5280,7 @@ int ObLSBackupComplementLogTask::inner_get_piece_file_list_(const share::ObLSID const share::SCN &start_scn = piece_attr.start_scn_; if (OB_FAIL(get_src_backup_piece_dir_(ls_id, piece_attr, src_piece_dir_path))) { LOG_WARN("failed to get src backup piece dir", K(ret), K(round_id), K(piece_id), K(ls_id), K(piece_attr)); - } else if (OB_FAIL(util.list_files(src_piece_dir_path.get_obstr(), archive_dest_.get_storage_info(), op))) { + } else if (OB_FAIL(util.adaptively_list_files(src_piece_dir_path.get_obstr(), archive_dest_.get_storage_info(), op))) { LOG_WARN("failed to list files", K(ret), K(src_piece_dir_path)); } else if (OB_FAIL(op.get_file_id_list(file_id_list))) { LOG_WARN("failed to get files", K(ret)); @@ -5551,7 +5551,7 @@ int ObLSBackupComplementLogTask::inner_transfer_clog_file_( } else if (OB_ISNULL(buf = static_cast(allocator.alloc(transfer_len)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to allocate memory", K(ret), K(transfer_len)); - } else if (OB_FAIL(util.read_part_file(src_path.get_obstr(), archive_dest_.get_storage_info(), buf, transfer_len, dst_len, read_len))) { + } else if (OB_FAIL(util.adaptively_read_part_file(src_path.get_obstr(), archive_dest_.get_storage_info(), buf, transfer_len, dst_len, read_len))) { LOG_WARN("failed to read part file", K(ret), K(src_path)); } else if (read_len != transfer_len) { ret = OB_ERR_UNEXPECTED; @@ -5585,7 +5585,7 @@ int ObLSBackupComplementLogTask::get_file_length_( { int ret = OB_SUCCESS; ObBackupIoAdapter util; - if (OB_FAIL(util.get_file_length(file_path, storage_info, length))) { + if (OB_FAIL(util.adaptively_get_file_length(file_path, storage_info, length))) { if (OB_BACKUP_FILE_NOT_EXIST == ret) { ret = OB_SUCCESS; length = 0; diff --git a/unittest/share/backup/CMakeLists.txt b/unittest/share/backup/CMakeLists.txt index aaec41242..8d48e876d 100644 --- a/unittest/share/backup/CMakeLists.txt +++ b/unittest/share/backup/CMakeLists.txt @@ -2,4 +2,5 @@ storage_unittest(test_backup_path) storage_unittest(test_backup_struct) storage_unittest(test_log_archive_backup_info_mgr) storage_unittest(test_backup_access_cos) +storage_unittest(test_backup_access_s3) storage_unittest(test_archive_checkpoint_mgr) diff --git a/unittest/share/backup/test_backup_access_s3.cpp b/unittest/share/backup/test_backup_access_s3.cpp new file mode 100644 index 000000000..a8ef8815b --- /dev/null +++ b/unittest/share/backup/test_backup_access_s3.cpp @@ -0,0 +1,323 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#include "lib/utility/ob_test_util.h" +#include "lib/restore/ob_storage.h" +#include "lib/restore/ob_storage_s3_base.h" +#include "share/backup/ob_backup_io_adapter.h" + +#include "lib/allocator/page_arena.h" +#include "test_backup_access_s3.h" + +using namespace oceanbase::common; + +class TestStorageS3Common { +public: +TestStorageS3Common() {} +~TestStorageS3Common() {} + +void init() +{ + ASSERT_EQ(OB_SUCCESS, + databuff_printf(account, sizeof(account), + "s3_region=%s&host=%s&access_id=%s&access_key=%s", + region, endpoint, secretid, secretkey)); + //build s3_base + const ObString s3_storage_info(account); + ASSERT_EQ(OB_SUCCESS, s3_base.set(ObStorageType::OB_STORAGE_S3, s3_storage_info.ptr())); +} +void destory() +{ +} +protected: + char account[OB_MAX_URI_LENGTH]; + const char *dir_name = "test_backup_io_adapter_access_s3_dir"; + char uri[OB_MAX_URI_LENGTH]; + char dir_uri[OB_MAX_URI_LENGTH]; + oceanbase::share::ObBackupStorageInfo s3_base; + + int object_prefix_len = 5; +}; + +class TestBackupIOAdapterAccessS3 : public ::testing::Test, public TestStorageS3Common +{ +public: + TestBackupIOAdapterAccessS3() : enable_test_(enable_test) {} + virtual ~TestBackupIOAdapterAccessS3() {} + virtual void SetUp() + { + init(); + } + virtual void TearDown() + { + destory(); + } + + static void SetUpTestCase() + { + } + + static void TearDownTestCase() + { + } + +private: + // disallow copy + DISALLOW_COPY_AND_ASSIGN(TestBackupIOAdapterAccessS3); +protected: + bool enable_test_; +}; + +TEST_F(TestBackupIOAdapterAccessS3, test_basic_rw) +{ + int ret = OB_SUCCESS; + if (enable_test_) { + ObBackupIoAdapter adapter; + const char *tmp_dir = "test_basic_rw"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_dir, ts)); + + // write + const char *write_content = "123456789ABCDEF"; + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/0", dir_uri)); + ASSERT_EQ(OB_SUCCESS, + adapter.write_single_file(uri, &s3_base, write_content, strlen(write_content))); + + // read + char read_buf[100] = {0}; + int64_t read_size = 0; + ASSERT_EQ(OB_SUCCESS, + adapter.read_single_file(uri, &s3_base, read_buf, sizeof(read_buf), read_size)); + ASSERT_STREQ(write_content, read_buf); + ASSERT_EQ(strlen(write_content), read_size); + + ASSERT_EQ(OB_SUCCESS, + adapter.read_part_file(uri, &s3_base, read_buf, sizeof(read_buf), 0, read_size)); + ASSERT_STREQ(write_content, read_buf); + ASSERT_EQ(strlen(write_content), read_size); + + int64_t offset = 5; + ASSERT_EQ(OB_SUCCESS, + adapter.read_part_file(uri, &s3_base, read_buf, sizeof(read_buf), offset, read_size)); + ASSERT_EQ('6', read_buf[0]); + ASSERT_EQ('F', read_buf[9]); + ASSERT_EQ(strlen(write_content) - offset, read_size); + + offset = 6; + ASSERT_EQ(OB_SUCCESS, + adapter.read_part_file(uri, &s3_base, read_buf, 5, offset, read_size)); + ASSERT_EQ('7', read_buf[0]); + ASSERT_EQ('B', read_buf[4]); + ASSERT_EQ(5, read_size); + + // offset = strlen(write_content); + // ASSERT_EQ(OB_COS_ERROR, + // adapter.read_part_file(uri, &s3_base, read_buf, sizeof(read_buf), offset, read_size)); + + ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &s3_base)); + } +} + +TEST_F(TestBackupIOAdapterAccessS3, test_util) +{ + int ret = OB_SUCCESS; + if (enable_test_) { + ObBackupIoAdapter adapter; + const char *tmp_dir = "test_util"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_dir, ts)); + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/0", dir_uri)); + + bool is_obj_exist = true; + ASSERT_EQ(OB_SUCCESS, adapter.is_exist(uri, &s3_base, is_obj_exist)); + ASSERT_FALSE(is_obj_exist); + + const char *write_content = "123456789ABCDEF"; + ASSERT_EQ(OB_SUCCESS, + adapter.write_single_file(uri, &s3_base, write_content, strlen(write_content))); + ASSERT_EQ(OB_SUCCESS, adapter.is_exist(uri, &s3_base, is_obj_exist)); + ASSERT_TRUE(is_obj_exist); + + int64_t file_length = -1; + ASSERT_EQ(OB_SUCCESS, adapter.get_file_length(uri, &s3_base, file_length)); + ASSERT_EQ(strlen(write_content), file_length); + + ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &s3_base)); + ASSERT_EQ(OB_SUCCESS, adapter.is_exist(uri, &s3_base, is_obj_exist)); + ASSERT_FALSE(is_obj_exist); + } +} + +TEST_F(TestBackupIOAdapterAccessS3, test_list_files) +{ + int ret = OB_SUCCESS; + if (enable_test_) { + ObBackupIoAdapter adapter; + const char *tmp_dir = "test_list_files"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_dir, ts)); + + int64_t file_num = 11; + const char *write_content = "0123456789"; + const char *format = "%s/%0*ld_%ld"; + for (int64_t file_idx = 0; file_idx < file_num; file_idx++) { + ASSERT_EQ(OB_SUCCESS, + databuff_printf(uri, sizeof(uri), format, dir_uri, object_prefix_len, file_idx, file_idx)); + ASSERT_EQ(OB_SUCCESS, + adapter.write_single_file(uri, &s3_base, write_content, strlen(write_content))); + } + + ObArenaAllocator allocator; + ObArray name_array; + ObFileListArrayOp op(name_array, allocator); + ASSERT_EQ(OB_SUCCESS, adapter.list_files(dir_uri, &s3_base, op)); + ASSERT_EQ(file_num, name_array.size()); + for (int64_t file_idx = 0; file_idx < file_num; file_idx++) { + // listed files do not contain prefix + ASSERT_EQ(OB_SUCCESS, + databuff_printf(uri, sizeof(uri), "%0*ld_%ld", object_prefix_len, file_idx, file_idx)); + ASSERT_STREQ(uri, name_array[file_idx].ptr()); + ASSERT_EQ(OB_SUCCESS, + databuff_printf(uri, sizeof(uri), format, dir_uri, object_prefix_len, file_idx, file_idx)); + ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &s3_base)); + } + } +} + +TEST_F(TestBackupIOAdapterAccessS3, test_list_directories) +{ + int ret = OB_SUCCESS; + if (enable_test_) { + ObBackupIoAdapter adapter; + const char *tmp_dir = "test_list_directories"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_dir, ts)); + + int64_t file_num = 11; + const char *write_content = "0123456789"; + const char *format = "%s/%0*ld/%ld"; + for (int64_t file_idx = 0; file_idx < file_num; file_idx++) { + ASSERT_EQ(OB_SUCCESS, + databuff_printf(uri, sizeof(uri), format, dir_uri, object_prefix_len, file_idx, file_idx)); + ASSERT_EQ(OB_SUCCESS, + adapter.write_single_file(uri, &s3_base, write_content, strlen(write_content))); + } + + ObArenaAllocator allocator; + ObArray name_array; + ObFileListArrayOp op(name_array, allocator); + ASSERT_EQ(OB_SUCCESS, adapter.list_directories(dir_uri, &s3_base, op)); + ASSERT_EQ(file_num, name_array.size()); + for (int64_t file_idx = 0; file_idx < file_num; file_idx++) { + // listed files do not contain prefix + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%0*ld", object_prefix_len, file_idx)); + ASSERT_STREQ(uri, name_array[file_idx].ptr()); + ASSERT_EQ(OB_SUCCESS, + databuff_printf(uri, sizeof(uri), format, dir_uri, object_prefix_len, file_idx, file_idx)); + ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &s3_base)); + } + } +} + +TEST_F(TestBackupIOAdapterAccessS3, test_is_tagging) +{ + int ret = OB_SUCCESS; + if (enable_test_) { + ObBackupIoAdapter adapter; + const char *tmp_util_dir = "test_util_is_tagging"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_util_dir, ts)); + + bool is_tagging = true; + char tmp_account[OB_MAX_URI_LENGTH]; + oceanbase::share::ObBackupStorageInfo tmp_s3_base; + const char *write_content = "123456789ABCDEF"; + + // wrong tag mode + ASSERT_EQ(OB_SUCCESS, + databuff_printf(tmp_account, sizeof(tmp_account), + "s3_region=%s&host=%s&access_id=%s&access_key=%s&delete_mode=tag", + region, endpoint, secretid, secretkey)); + ASSERT_EQ(OB_INVALID_ARGUMENT, tmp_s3_base.set(ObStorageType::OB_STORAGE_S3, tmp_account)); + tmp_s3_base.reset(); + + ASSERT_EQ(OB_SUCCESS, + databuff_printf(tmp_account, sizeof(tmp_account), + "s3_region=%s&host=%s&access_id=%s&access_key=%s&delete_mode=delete_delete", + region, endpoint, secretid, secretkey)); + ASSERT_EQ(OB_INVALID_ARGUMENT, tmp_s3_base.set(ObStorageType::OB_STORAGE_S3, tmp_account)); + tmp_s3_base.reset(); + + // delete mode + ASSERT_EQ(OB_SUCCESS, + databuff_printf(tmp_account, sizeof(tmp_account), + "s3_region=%s&host=%s&access_id=%s&access_key=%s&delete_mode=delete", + region, endpoint, secretid, secretkey)); + ASSERT_EQ(OB_SUCCESS, tmp_s3_base.set(ObStorageType::OB_STORAGE_S3, tmp_account)); + + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/delete_mode", dir_uri)); + ASSERT_EQ(OB_SUCCESS, + adapter.write_single_file(uri, &tmp_s3_base, write_content, strlen(write_content))); + ASSERT_EQ(OB_SUCCESS, adapter.is_tagging(uri, &tmp_s3_base, is_tagging)); + ASSERT_FALSE(is_tagging); + + ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &tmp_s3_base)); + ASSERT_EQ(OB_BACKUP_FILE_NOT_EXIST, adapter.is_tagging(uri, &tmp_s3_base, is_tagging)); + tmp_s3_base.reset(); + + // tagging mode + ASSERT_EQ(OB_SUCCESS, + databuff_printf(tmp_account, sizeof(tmp_account), + "s3_region=%s&host=%s&access_id=%s&access_key=%s&delete_mode=tagging", + region, endpoint, secretid, secretkey)); + ASSERT_EQ(OB_SUCCESS, tmp_s3_base.set(ObStorageType::OB_STORAGE_S3, tmp_account)); + + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/tagging_mode", dir_uri)); + ASSERT_EQ(OB_SUCCESS, + adapter.write_single_file(uri, &tmp_s3_base, write_content, strlen(write_content))); + + is_tagging = true; + ASSERT_EQ(OB_SUCCESS, adapter.is_tagging(uri, &tmp_s3_base, is_tagging)); + ASSERT_FALSE(is_tagging); + + ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &tmp_s3_base)); + ASSERT_EQ(OB_SUCCESS, adapter.is_tagging(uri, &tmp_s3_base, is_tagging)); + ASSERT_TRUE(is_tagging); + tmp_s3_base.reset(); + + // clean + ASSERT_EQ(OB_SUCCESS, + databuff_printf(tmp_account, sizeof(tmp_account), + "s3_region=%s&host=%s&access_id=%s&access_key=%s", + region, endpoint, secretid, secretkey)); + ASSERT_EQ(OB_SUCCESS, tmp_s3_base.set(ObStorageType::OB_STORAGE_S3, tmp_account)); + + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/tagging_mode", dir_uri)); + ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &tmp_s3_base)); + ASSERT_EQ(OB_BACKUP_FILE_NOT_EXIST, adapter.is_tagging(uri, &tmp_s3_base, is_tagging)); + } +} + +int main(int argc, char **argv) +{ + system("rm -f test_backup_access_s3.log*"); + OB_LOGGER.set_file_name("test_backup_access_s3.log", true, true); + OB_LOGGER.set_log_level("DEBUG"); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/unittest/share/backup/test_backup_access_s3.h b/unittest/share/backup/test_backup_access_s3.h new file mode 100644 index 000000000..d5aae6172 --- /dev/null +++ b/unittest/share/backup/test_backup_access_s3.h @@ -0,0 +1,23 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef UNITTEST_BACKUP_ACCESS_S3_H_ +#define UNITTEST_BACKUP_ACCESS_S3_H_ + +const bool enable_test = false; +const char *bucket = "s3://xxx"; +const char *region = "xxx"; +const char *endpoint = "xxx"; +const char *secretid = "xxx"; +const char *secretkey = "xxx"; + +#endif \ No newline at end of file