From 4144cf1751fa6d1a56b18e4e72d3e89f67f2daa3 Mon Sep 17 00:00:00 2001 From: WenJinyu <596324105@qq.com> Date: Mon, 21 Aug 2023 07:10:29 +0000 Subject: [PATCH] [FEAT MERGE]Added object storage COS as the backup destination --- deps/init/oceanbase.el7.aarch64.deps | 1 + deps/init/oceanbase.el7.x86_64.deps | 1 + deps/init/oceanbase.el8.aarch64.deps | 1 + deps/init/oceanbase.el8.x86_64.deps | 1 + deps/oblib/src/lib/restore/CMakeLists.txt | 11 +- deps/oblib/src/lib/restore/cos/CMakeLists.txt | 35 + .../src/lib/restore/cos/ob_cos_wrapper.cpp | 1297 +++++++++++++++++ .../src/lib/restore/cos/ob_cos_wrapper.h | 325 +++++ .../lib/restore/cos/ob_cos_wrapper_handle.cpp | 235 +++ .../lib/restore/cos/ob_cos_wrapper_handle.h | 77 + deps/oblib/src/lib/restore/cos/ob_singleton.h | 50 + deps/oblib/src/lib/restore/ob_i_storage.h | 8 +- .../src/lib/restore/ob_object_device.cpp | 89 +- deps/oblib/src/lib/restore/ob_object_device.h | 6 +- deps/oblib/src/lib/restore/ob_storage.cpp | 103 +- deps/oblib/src/lib/restore/ob_storage.h | 20 +- .../src/lib/restore/ob_storage_cos_base.cpp | 934 ++++++++++++ .../src/lib/restore/ob_storage_cos_base.h | 154 ++ .../oblib/src/lib/restore/ob_storage_file.cpp | 22 +- deps/oblib/src/lib/restore/ob_storage_file.h | 22 +- .../oblib/src/lib/restore/ob_storage_info.cpp | 297 ++++ deps/oblib/src/lib/restore/ob_storage_info.h | 80 + .../src/lib/restore/ob_storage_oss_base.cpp | 150 +- .../src/lib/restore/ob_storage_oss_base.h | 18 +- .../src/lib/statistic_event/ob_stat_event.h | 11 +- deps/oblib/unittest/lib/CMakeLists.txt | 2 + .../unittest/lib/restore/test_storage_cos.cpp | 681 +++++++++ .../unittest/lib/restore/test_storage_cos.h | 23 + .../lib/restore/test_storage_info.cpp | 114 ++ .../archiveservice/ob_archive_sender.cpp | 3 +- .../ob_log_external_storage_io_task.h | 2 +- .../ob_remote_data_generator.cpp | 5 +- src/share/backup/ob_backup_io_adapter.cpp | 36 +- src/share/backup/ob_backup_io_adapter.h | 3 - src/share/backup/ob_backup_path.cpp | 6 +- src/share/backup/ob_backup_struct.cpp | 377 +---- src/share/backup/ob_backup_struct.h | 52 +- src/share/ob_device_manager.cpp | 3 + src/sql/resolver/ddl/ob_ddl_resolver.cpp | 2 +- .../ob_admin_dump_backup_data_executor.cpp | 2 +- .../ob_admin_dump_backup_data_executor.h | 2 +- unittest/share/backup/CMakeLists.txt | 3 +- .../share/backup/test_backup_access_cos.cpp | 326 +++++ .../share/backup/test_backup_access_cos.h | 23 + unittest/share/backup/test_backup_struct.cpp | 74 +- 45 files changed, 5000 insertions(+), 687 deletions(-) create mode 100644 deps/oblib/src/lib/restore/cos/CMakeLists.txt create mode 100644 deps/oblib/src/lib/restore/cos/ob_cos_wrapper.cpp create mode 100644 deps/oblib/src/lib/restore/cos/ob_cos_wrapper.h create mode 100644 deps/oblib/src/lib/restore/cos/ob_cos_wrapper_handle.cpp create mode 100644 deps/oblib/src/lib/restore/cos/ob_cos_wrapper_handle.h create mode 100644 deps/oblib/src/lib/restore/cos/ob_singleton.h create mode 100644 deps/oblib/src/lib/restore/ob_storage_cos_base.cpp create mode 100644 deps/oblib/src/lib/restore/ob_storage_cos_base.h create mode 100644 deps/oblib/src/lib/restore/ob_storage_info.cpp create mode 100644 deps/oblib/src/lib/restore/ob_storage_info.h create mode 100644 deps/oblib/unittest/lib/restore/test_storage_cos.cpp create mode 100644 deps/oblib/unittest/lib/restore/test_storage_cos.h create mode 100644 deps/oblib/unittest/lib/restore/test_storage_info.cpp create mode 100644 unittest/share/backup/test_backup_access_cos.cpp create mode 100644 unittest/share/backup/test_backup_access_cos.h diff --git a/deps/init/oceanbase.el7.aarch64.deps b/deps/init/oceanbase.el7.aarch64.deps index 45915c58b..47bd44701 100644 --- a/deps/init/oceanbase.el7.aarch64.deps +++ b/deps/init/oceanbase.el7.aarch64.deps @@ -27,6 +27,7 @@ devdeps-ncurses-static-6.2-72022100815.el7.aarch64.rpm devdeps-boost-1.74.0-22022110914.el7.aarch64.rpm devdeps-s2geometry-0.9.0-42022111116.el7.aarch64.rpm devdeps-icu-69.1-72022112416.el7.aarch64.rpm +devdeps-cos-c-sdk-5.0.16-52023070517.el7.aarch64.rpm [tools] obdevtools-binutils-2.30-12022100413.el7.aarch64.rpm diff --git a/deps/init/oceanbase.el7.x86_64.deps b/deps/init/oceanbase.el7.x86_64.deps index 6783e81cb..2334b3818 100644 --- a/deps/init/oceanbase.el7.x86_64.deps +++ b/deps/init/oceanbase.el7.x86_64.deps @@ -29,6 +29,7 @@ devdeps-ncurses-static-6.2-72022100815.el7.x86_64.rpm devdeps-boost-1.74.0-22022110914.el7.x86_64.rpm devdeps-s2geometry-0.9.0-42022111116.el7.x86_64.rpm devdeps-icu-69.1-72022112416.el7.x86_64.rpm +devdeps-cos-c-sdk-5.0.16-52023070517.el7.x86_64.rpm [tools] obdevtools-binutils-2.30-12022100413.el7.x86_64.rpm diff --git a/deps/init/oceanbase.el8.aarch64.deps b/deps/init/oceanbase.el8.aarch64.deps index d1c19f299..e7c9a659e 100644 --- a/deps/init/oceanbase.el8.aarch64.deps +++ b/deps/init/oceanbase.el8.aarch64.deps @@ -27,6 +27,7 @@ devdeps-ncurses-static-6.2-72022100815.el8.aarch64.rpm devdeps-boost-1.74.0-22022110914.el8.aarch64.rpm devdeps-s2geometry-0.9.0-42022111116.el8.aarch64.rpm devdeps-icu-69.1-72022112416.el8.aarch64.rpm +devdeps-cos-c-sdk-5.0.16-52023070517.el8.aarch64.rpm [tools] obdevtools-binutils-2.30-12022100413.el8.aarch64.rpm diff --git a/deps/init/oceanbase.el8.x86_64.deps b/deps/init/oceanbase.el8.x86_64.deps index 747ceaf92..cfea1e8f7 100644 --- a/deps/init/oceanbase.el8.x86_64.deps +++ b/deps/init/oceanbase.el8.x86_64.deps @@ -28,6 +28,7 @@ devdeps-ncurses-static-6.2-72022100815.el8.x86_64.rpm devdeps-boost-1.74.0-22022110914.el8.x86_64.rpm devdeps-s2geometry-0.9.0-42022111116.el8.x86_64.rpm devdeps-icu-69.1-72022112416.el8.x86_64.rpm +devdeps-cos-c-sdk-5.0.16-52023070517.el8.x86_64.rpm [tools] obdevtools-binutils-2.30-12022100413.el8.x86_64.rpm diff --git a/deps/oblib/src/lib/restore/CMakeLists.txt b/deps/oblib/src/lib/restore/CMakeLists.txt index a08ae07f0..595cdf56d 100644 --- a/deps/oblib/src/lib/restore/CMakeLists.txt +++ b/deps/oblib/src/lib/restore/CMakeLists.txt @@ -6,8 +6,15 @@ oblib_add_library(restore OBJECT ob_storage.h ob_storage_oss_base.cpp ob_storage_oss_base.h + ob_storage_cos_base.cpp + ob_storage_cos_base.h ob_storage_path.cpp ob_storage_path.h + ob_storage_info.cpp + ob_storage_info.h + cos/ob_cos_wrapper_handle.cpp + cos/ob_cos_wrapper_handle.h + cos/ob_singleton.h ob_object_device.cpp ob_object_device.h) @@ -28,4 +35,6 @@ target_link_libraries(oss ${DEP_DIR}/lib/libmxml.a) -target_link_libraries(restore PUBLIC oss oblib_base) +target_link_libraries(restore PUBLIC oss cos_sdk oblib_base) + +add_subdirectory(cos) diff --git a/deps/oblib/src/lib/restore/cos/CMakeLists.txt b/deps/oblib/src/lib/restore/cos/CMakeLists.txt new file mode 100644 index 000000000..76ae235e0 --- /dev/null +++ b/deps/oblib/src/lib/restore/cos/CMakeLists.txt @@ -0,0 +1,35 @@ +add_library(cos_wrapper OBJECT + ob_singleton.h + ob_cos_wrapper.h + ob_cos_wrapper.cpp) + +target_include_directories(cos_wrapper + PUBLIC + ${DEP_DIR}/include/apr-1 + ${DEP_DIR}/include + ${DEP_DIR}/include/cos_c_sdk + ${DEP_DIR}/include/mxml +) +target_link_libraries(cos_wrapper PUBLIC oblib_base_base) + +# we need to get all .o files from cos-c-sdk rpm and mixed them into one .o file +add_custom_command( + OUTPUT cos_sdk_lib.o + COMMAND rm -rf ${CMAKE_CURRENT_BINARY_DIR}/cos_a2o && mkdir ${CMAKE_CURRENT_BINARY_DIR}/cos_a2o && cp ${DEP_DIR}/lib/libcos_c_sdk_static.a ${CMAKE_CURRENT_BINARY_DIR}/cos_a2o && cd ${CMAKE_CURRENT_BINARY_DIR}/cos_a2o && ar x libcos_c_sdk_static.a && ld -r cos_*.o -o cos_sdk_lib.o + DEPENDS ${DEP_DIR}/lib/libcos_c_sdk_static.a +) + +# we need to join cos-c-sdk .o file with cos_wrapper .o file +# cuz we need to set 'localize-hidden', which can help cos_wrapper to get hidden symbol of cos-c-sdk. +add_custom_command( + OUTPUT cos_sdk_objs.o + COMMAND ld -r $ -o cos_wrapper.o + COMMAND ld -r cos_wrapper.o ${CMAKE_CURRENT_BINARY_DIR}/cos_a2o/cos_sdk_lib.o -o cos_sdk_objs.tmp.o + COMMAND objcopy --localize-hidden cos_sdk_objs.tmp.o cos_sdk_objs.o + DEPENDS cos_wrapper cos_sdk_lib.o + BYPRODUCTS cos_sdk_objs.o + COMMAND_EXPAND_LISTS +) + +oblib_add_extra_objects(${CMAKE_CURRENT_BINARY_DIR}/cos_sdk_objs.o) +oblib_add_library(cos_sdk cos_sdk_objs.o) diff --git a/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.cpp b/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.cpp new file mode 100644 index 000000000..c795040f9 --- /dev/null +++ b/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.cpp @@ -0,0 +1,1297 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#include +#include +#include + +#include "cos_api.h" +#include "cos_log.h" +#include "cos_utility.h" +#include "cos_string.h" +#include "cos_status.h" +#include "cos_auth.h" +#include "cos_sys_util.h" +#include "apr_errno.h" + +#include "ob_cos_wrapper.h" + +namespace oceanbase +{ +namespace common +{ +namespace qcloud_cos +{ +using namespace oceanbase::common; +constexpr int OB_SUCCESS = 0; +constexpr int OB_INVALID_ARGUMENT = -4002; +constexpr int OB_INIT_TWICE = -4005; +constexpr int OB_ALLOCATE_MEMORY_FAILED = -4013; +constexpr int OB_SIZE_OVERFLOW = -4019; +constexpr int OB_BACKUP_FILE_NOT_EXIST = -9011; +constexpr int OB_COS_ERROR = -9060; +constexpr int OB_IO_LIMIT = -9061; +constexpr int OB_BACKUP_PERMISSION_DENIED = -9071; +constexpr int OB_BACKUP_PWRITE_OFFSET_NOT_MATCH = -9083; + +const int COS_OBJECT_NOT_EXIST = 404; +const int COS_PERMISSION_DENIED = 403; +const int COS_APPEND_POSITION_ERROR = 409; +const int COS_SERVICE_UNAVAILABLE = 503; + +//datetime formate : Tue, 09 Apr 2019 06:24:00 GMT +//time unit is second +static int64_t strtotime(const char *date_time) +{ + int64_t time = 0; + struct tm tm_time; + if (NULL == strptime(date_time, "%a, %d %b %Y %H:%M:%S %Z", &tm_time)) { + //skip set ret, for compat data formate + cos_warn_log("[COS]failed to transform time, time=%s\n", date_time); + } else { + time = mktime(&tm_time); + } + + return time; +} + +static void convert_io_error(cos_status_t *cos_ret, int &ob_errcode) +{ + if (NULL == cos_ret) { + ob_errcode = OB_COS_ERROR; + } else if (!cos_status_is_ok(cos_ret)) { + switch (cos_ret->code) { + case COS_PERMISSION_DENIED: { + ob_errcode = OB_BACKUP_PERMISSION_DENIED; + break; + } + case COS_OBJECT_NOT_EXIST: { + ob_errcode = OB_BACKUP_FILE_NOT_EXIST; + break; + } + case COS_APPEND_POSITION_ERROR: { + ob_errcode = OB_BACKUP_PWRITE_OFFSET_NOT_MATCH; + break; + } + default: { + ob_errcode = OB_COS_ERROR; + } + } + } +} + + +int ObCosAccount::set_field(const char *value, char *field, uint32_t length) +{ + int ret = OB_SUCCESS; + + if (NULL == value || NULL == field) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]invalid args, value=%p, field=%p, ret=%d\n", value, field, ret); + } else { + const uint32_t value_len = strlen(value); + if (value_len >= length) { + ret = OB_SIZE_OVERFLOW; + cos_warn_log("[COS]value is too long, value_len=%u, length=%u, ret=%d\n", value_len, length, ret); + } else { + memcpy(field, value, value_len); + field[value_len] = '\0'; + } + } + + return ret; +} + +int ObCosAccount::parse_from(const char *storage_info, uint32_t size) +{ + int ret = OB_SUCCESS; + if (NULL == storage_info || MAX_COS_DOMAIN_LENGTH <= size) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]cos parse account failed, storage_info=%s, size=%d, ret=%d\n", storage_info, size, ret); + } else { + // host=xxxx&access_id=xxx&access_key=xxx&appid=xxx + char tmp[MAX_COS_DOMAIN_LENGTH]; + char *token = NULL; + char *saved_ptr = NULL; + const char *HOST = "host="; + const char *ACCESS_ID = "access_id="; + const char *ACCESS_KEY = "access_key="; + const char *APPID = "appid="; + const char *DELETE_MODE = "delete_mode="; + + uint8_t bitmap = 0; + + memcpy(tmp, storage_info, size); + tmp[size] = '\0'; + token = tmp; + for (char *str = token; ret == OB_SUCCESS; str = NULL) { + token = ::strtok_r(str, "&", &saved_ptr); + if (NULL == token) { + break; + } else if (0 == strncmp(HOST, token, strlen(HOST))) { + if (OB_SUCCESS != (ret = set_field(token + strlen(HOST), endpoint_, sizeof(endpoint_)))) { + cos_warn_log("[COS]failed to set endpoint=%s, ret=%d\n", token, ret); + } else { + bitmap |= 1; + } + } else if (0 == strncmp(ACCESS_ID, token, strlen(ACCESS_ID))) { + if (OB_SUCCESS != (ret = set_field(token + strlen(ACCESS_ID), access_id_, sizeof(access_id_)))) { + cos_warn_log("[COS]failed to set access_id=%s, ret=%d\n", token, ret); + } else { + bitmap |= (1 << 1); + } + } else if (0 == strncmp(ACCESS_KEY, token, strlen(ACCESS_KEY))) { + if (OB_SUCCESS != (ret = set_field(token + strlen(ACCESS_KEY), access_key_, sizeof(access_key_)))) { + cos_warn_log("[COS]failed to set access_key=%s, ret=%d\n", token, ret); + } else { + bitmap |= (1 << 2); + } + } else if (0 == strncmp(APPID, token, strlen(APPID))) { + if (OB_SUCCESS != (ret = set_field(token + strlen(APPID), appid_, sizeof(appid_)))) { + cos_warn_log("[COS]failed to set appid=%s, ret=%d\n", token, ret); + } else { + bitmap |= (1 << 3); + } + } else if (0 == strncmp(DELETE_MODE, token, strlen(DELETE_MODE))) { + if (OB_SUCCESS != (ret = set_field(token + strlen(DELETE_MODE), delete_mode_, sizeof(delete_mode_)))) { + cos_warn_log("[COS]failed to set delete_mode=%s, ret=%d", token, ret); + } + } else { + cos_warn_log("[COS]unkown token:%s\n", token); + } + } + + if (OB_SUCCESS == ret && bitmap != 0x0F) { + ret = OB_COS_ERROR; + cos_warn_log("[COS]failed to parse cos account storage_info=%s, bitmap=%x, ret=%d\n", storage_info, bitmap, ret); + } + } + + return ret; +} + +int ObCosEnv::init() +{ + int ret = OB_SUCCESS; + int cos_ret = COSE_OK; + + if (is_inited_) { + ret = OB_INIT_TWICE; + cos_warn_log("[COS]cannot init cos env more than once, ret=%d\n", ret); + } else if(COSE_OK != (cos_ret = cos_http_io_initialize(NULL, 0))) { + ret = OB_COS_ERROR; + cos_warn_log("[COS]fail to init cos env, cos_ret=%d, ret=%d\n", cos_ret, ret); + } else { + is_inited_ = true; + } + return ret; +} + +void ObCosEnv::destroy() +{ + if (is_inited_) { + cos_http_io_deinitialize(); + is_inited_ = false; + } +} + + +// define cos context +struct CosContext +{ + cos_pool_t *mem_pool; + cos_request_options_t *options; + OB_COS_customMem custom_mem; + + CosContext(OB_COS_customMem &mem) + : mem_pool(NULL), options(NULL) + { + custom_mem.customAlloc = mem.customAlloc; + custom_mem.customFree = mem.customFree; + custom_mem.opaque = mem.opaque; + } +}; + +static void log_status(cos_status_t *s) +{ + if (NULL != s) { + cos_warn_log("status->code: %d", s->code); + if (s->error_code) { + cos_warn_log("status->error_code: %s", s->error_code); + } + if (s->error_msg) { + cos_warn_log("status->error_msg: %s", s->error_msg); + } + if (s->req_id) { + cos_warn_log("status->req_id: %s", s->req_id); + } + } +} + +int ObCosWrapper::create_cos_handle( + OB_COS_customMem &custom_mem, + const struct ObCosAccount &account, + ObCosWrapper::Handle **h) +{ + int ret = OB_SUCCESS; + int apr_ret = APR_SUCCESS; + + *h = NULL; + CosContext *ctx = static_cast(custom_mem.customAlloc(custom_mem.opaque, sizeof(CosContext))); + + if (NULL == ctx) { + ret = OB_ALLOCATE_MEMORY_FAILED; + cos_warn_log("[COS]fail to allocate cos context memory, ret=%d\n", ret); + } else { + ctx = new(ctx) CosContext(custom_mem); + if (APR_SUCCESS != (apr_ret = cos_pool_create(&ctx->mem_pool, NULL)) || NULL == ctx->mem_pool) { + custom_mem.customFree(custom_mem.opaque, ctx); + ret = OB_ALLOCATE_MEMORY_FAILED; + cos_warn_log("[COS]fail to create cos memory pool, apr_ret=%d, ret=%d\n", apr_ret, ret); + } else if (NULL == (ctx->options = cos_request_options_create(ctx->mem_pool))) { + cos_pool_destroy(ctx->mem_pool); + custom_mem.customFree(custom_mem.opaque, ctx); + ret = OB_ALLOCATE_MEMORY_FAILED; + cos_warn_log("[COS]fail to create cos request option, ret=%d\n", ret); + } else if (NULL == (ctx->options->config = cos_config_create(ctx->mem_pool))) { + cos_pool_destroy(ctx->mem_pool); + custom_mem.customFree(custom_mem.opaque, ctx); + ret = OB_ALLOCATE_MEMORY_FAILED; + cos_warn_log("[COS]fail to create cos option config, ret=%d\n", ret); + } else if (NULL == (ctx->options->ctl = cos_http_controller_create(ctx->options->pool, 0))) { + cos_pool_destroy(ctx->mem_pool); + custom_mem.customFree(custom_mem.opaque, ctx); + ret = OB_ALLOCATE_MEMORY_FAILED; + cos_warn_log("[COS]fail to create cos http controller, ret=%d\n", ret); + } else { + *h = reinterpret_cast(ctx); + + cos_str_set(&ctx->options->config->endpoint, account.endpoint_); + cos_str_set(&ctx->options->config->access_key_id, account.access_id_); + cos_str_set(&ctx->options->config->access_key_secret, account.access_key_); + cos_str_set(&ctx->options->config->appid, account.appid_); + ctx->options->config->is_cname = 0; + // connection timeout, default 60s + ctx->options->ctl->options->connect_timeout = 60; + // DNS timeout, default 60s + ctx->options->ctl->options->dns_cache_timeout = 120; + ctx->options->ctl->options->speed_time = 60; + ctx->options->ctl->options->speed_limit = 16000; + } + } + + return ret; +} + + +void ObCosWrapper::destroy_cos_handle( + Handle *h) +{ + if (NULL != h) { + CosContext *ctx = reinterpret_cast(h); + cos_pool_destroy(ctx->mem_pool); + ctx->custom_mem.customFree(ctx->custom_mem.opaque, ctx); + } +} + + +int ObCosWrapper::put( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name, + const char *buf, + const int64_t buf_size) +{ + int ret = OB_SUCCESS; + + CosContext *ctx = reinterpret_cast(h); + + if (NULL == h) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]cos handle is null, ret=%d\n", ret); + } else if (bucket_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]bucket name is null, ret=%d\n", ret); + } else if (object_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]object name is null, ret=%d\n", ret); + } else if (0 > buf_size) { + // To support empty object, size = 0 is valid. + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]buf size not valid, buf_size=%ld, ret=%d\n", buf_size, ret); + } else if (NULL == buf && 0 < buf_size) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]buf is null, buf=%p, buf_size=%ld, ret=%d\n", buf, buf_size, ret); + } else { + cos_string_t bucket; + cos_string_t object; + cos_str_set(&bucket, bucket_name.data_); + cos_str_set(&object, object_name.data_); + cos_table_t *resp_headers = NULL; + cos_status_t *cos_ret = NULL; + cos_list_t buffer; + cos_buf_t *content = NULL; + cos_list_init(&buffer); + + if (NULL == (content = cos_buf_pack(ctx->mem_pool, buf, static_cast(buf_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + cos_warn_log("[COS]fail to pack buf, ret=%d\n", ret); + } else { + cos_list_add_tail(&content->node, &buffer); + if(NULL == (cos_ret = cos_put_object_from_buffer(ctx->options, &bucket, &object, &buffer, NULL, &resp_headers)) + || !cos_status_is_ok(cos_ret)) { + convert_io_error(cos_ret, ret); + cos_warn_log("[COS]failed to put one object to cos, ret=%d\n", ret); + log_status(cos_ret); + } + } + } + + return ret; +} + +int ObCosWrapper::append( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name, + const char *buf, + const int64_t buf_size, + const int64_t offset) +{ + int ret = OB_SUCCESS; + + CosContext *ctx = reinterpret_cast(h); + + if (NULL == h) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]cos handle is null, ret=%d\n", ret); + } else if (bucket_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]bucket name is null, ret=%d\n", ret); + } else if (object_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]object name is null, ret=%d\n", ret); + } else if (0 > buf_size) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]buf size not valid, buf_size=%ld, ret=%d\n", buf_size, ret); + } else if (NULL == buf && 0 < buf_size) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]buf is null, buf=%p, buf_size=%ld, ret=%d\n", buf, buf_size, ret); + } else if (0 > offset) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]position not valid, offset=%ld, ret=%d\n", offset, ret); + } else { + cos_string_t bucket; + cos_string_t object; + cos_str_set(&bucket, bucket_name.data_); + cos_str_set(&object, object_name.data_); + cos_table_t *resp_headers = NULL; + cos_status_t *cos_ret = NULL; + cos_list_t buffer; + cos_buf_t *content = NULL; + cos_list_init(&buffer); + + if (NULL == (content = cos_buf_pack(ctx->mem_pool, buf, static_cast(buf_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + cos_warn_log("[COS]fail to pack buf, ret=%d\n", ret); + } else { + cos_list_add_tail(&content->node, &buffer); + if(NULL == (cos_ret = cos_append_object_from_buffer(ctx->options, &bucket, &object, + offset, &buffer, NULL, &resp_headers)) || !cos_status_is_ok(cos_ret)) { + convert_io_error(cos_ret, ret); + cos_warn_log("[COS]failed to append object from buffer to cos, ret=%d\n", ret); + log_status(cos_ret); + } + } + } + + return ret; +} + + +int ObCosWrapper::head_object_meta( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name, + bool &is_exist, + CosObjectMeta &meta) +{ + int ret = OB_SUCCESS; + + CosContext *ctx = reinterpret_cast(h); + is_exist = true; + + if (NULL == h) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]cos handle is null, ret=%d\n", ret); + } else if (bucket_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]bucket name is null, ret=%d\n", ret); + } else if (object_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]object name is null, ret=%d\n", ret); + } else { + cos_string_t bucket; + cos_string_t object; + cos_str_set(&bucket, bucket_name.data_); + cos_str_set(&object, object_name.data_); + cos_table_t *resp_headers = NULL; + cos_table_t *headers = NULL; + cos_status_t *cos_ret = NULL; + char *file_length_ptr = NULL; + char *last_modified_ptr = NULL; + + const char COS_CONTENT_LENGTH[] = "Content-Length"; + const char COS_LAST_MODIFIED[] = "Last-Modified"; + const char COS_OBJECT_TYPE[] = "x-cos-object-type"; + + if (NULL == (headers = cos_table_make(ctx->mem_pool, 0))) { + + } else if (NULL == (cos_ret = cos_head_object(ctx->options, &bucket, &object, headers, &resp_headers)) + || !cos_status_is_ok(cos_ret)) { + if (NULL != cos_ret && COS_OBJECT_NOT_EXIST == cos_ret->code) { + is_exist = false; + } else { + convert_io_error(cos_ret, ret); + cos_warn_log("[COS]failed to get file meta, ret=%d.\n", ret); + log_status(cos_ret); + } + } + + if (OB_SUCCESS == ret && is_exist) { + char *object_type = (char*)(apr_table_get(resp_headers, COS_OBJECT_TYPE)); + if (NULL != object_type) { + if (0 == strncmp(COS_OBJECT_TYPE_APPENDABLE, object_type, strlen(COS_OBJECT_TYPE_APPENDABLE))) { + meta.type_ = CosObjectMeta::CosObjectType::COS_OBJ_APPENDABLE; + } else if (0 == strncmp(COS_OBJECT_TYPE_NORMAL, object_type, strlen(COS_OBJECT_TYPE_NORMAL))) { + meta.type_ = CosObjectMeta::CosObjectType::COS_OBJ_NORMAL; + } else { + ret = OB_COS_ERROR; + cos_warn_log("[COS]unknown cos object type, ret=%d.\n", ret); + } + } + + is_exist = true; + // get object length + if (OB_SUCCESS != ret) { + } else if (NULL != (file_length_ptr = (char*)apr_table_get(resp_headers, COS_CONTENT_LENGTH))) { + // enhance verification + // not a valid file length, start with a non-digit character + if (0 == isdigit(*file_length_ptr)) { + ret = OB_COS_ERROR; + cos_warn_log("[COS]not a valid file length, something wrong unexpected, ret=%d, file_length_ptr=%s.\n", ret, file_length_ptr); + } else { + char *end; + meta.file_length_ = strtoll(file_length_ptr, &end, 10); + // not a valid file length, end with a non-digit character + if (0 != *end) { + ret = OB_COS_ERROR; + cos_warn_log("[COS]not a valid file length, something wrong unexpected, ret=%d, file_length_ptr=%s.\n", ret, file_length_ptr); + } + } + } + + // get object last modified time + if (OB_SUCCESS != ret) { + } else if (NULL != (last_modified_ptr = (char*)apr_table_get(resp_headers, COS_LAST_MODIFIED))) { + meta.last_modified_ts_ = strtotime(last_modified_ptr); + } else { + ret = OB_COS_ERROR; + cos_warn_log("[COS]failed to get last modified from apr table, something wrong unexpected, ret=%d.\n", ret); + } + } + } + + return ret; +} + +int ObCosWrapper::del( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name) +{ + int ret = OB_SUCCESS; + + CosContext *ctx = reinterpret_cast(h); + + if (NULL == h) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]cos handle is null, ret=%d\n", ret); + } else if (bucket_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]bucket name is null, ret=%d\n", ret); + } else if (object_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]object name is null, ret=%d\n", ret); + } else { + cos_string_t bucket; + cos_string_t object; + cos_str_set(&bucket, bucket_name.data_); + cos_str_set(&object, object_name.data_); + cos_table_t *resp_headers = NULL; + cos_status_t *cos_ret = NULL; + + if (NULL == (cos_ret = cos_delete_object(ctx->options, &bucket, &object, &resp_headers)) + || !cos_status_is_ok(cos_ret)) { + convert_io_error(cos_ret, ret); + cos_warn_log("[COS]failed to delete object, ret=%d\n", ret); + log_status(cos_ret); + } + } + + return ret; +} + +int ObCosWrapper::tag( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name) +{ + int ret = OB_SUCCESS; + + CosContext *ctx = reinterpret_cast(h); + + if (NULL == h) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]cos handle is null, ret=%d\n", ret); + } else if (bucket_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]bucket name is null, ret=%d\n", ret); + } else if (object_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]object name is null, ret=%d\n", ret); + } else { + cos_string_t bucket; + cos_string_t object; + cos_str_set(&bucket, bucket_name.data_); + cos_str_set(&object, object_name.data_); + cos_table_t *resp_headers = NULL; + cos_status_t *cos_ret = NULL; + cos_tagging_params_t *params = NULL; + cos_tagging_tag_t *tag = NULL; + + if (NULL == (params = cos_create_tagging_params(ctx->mem_pool))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + cos_warn_log("[COS]fail to create tagging params, ret=%d\n", ret); + } else if (NULL == (tag = cos_create_tagging_tag(ctx->mem_pool))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + cos_warn_log("[COS]fail to create tagging tag, ret=%d\n", ret); + } else { + cos_str_set(&tag->key, "delete_mode"); + cos_str_set(&tag->value, "tagging"); + cos_list_add_tail(&tag->node, ¶ms->node); + if (NULL == (cos_ret = cos_put_object_tagging(ctx->options, &bucket, &object, + NULL, NULL, params, &resp_headers)) + || !cos_status_is_ok(cos_ret)) { + convert_io_error(cos_ret, ret); + cos_warn_log("[COS]failed to tag object, ret=%d\n", ret); + log_status(cos_ret); + } + } + } + + return ret; +} + +int ObCosWrapper::del_objects_in_dir( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &dir_name, + int64_t &deleted_cnt) +{ + // There is no ability to delete those objects that have a common prefix + // directly which sdk supports. Otherwise, we need list these objects then + // delete them. + int ret = OB_SUCCESS; + + const char *seperator = "/"; + // dir_name must be end with '/\0' + const int32_t min_dir_name_str_len = 2; + + CosContext *ctx = reinterpret_cast(h); + deleted_cnt = 0; + if (NULL == h) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]cos handle is null, ret=%d\n", ret); + } else if (bucket_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]bucket name is null, ret=%d\n", ret); + } else if (NULL == dir_name.data_ || min_dir_name_str_len >= dir_name.size_) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]dir_name size too short, size=%d, ret=%d\n", dir_name.size_, ret); + } else if (dir_name.data_[dir_name.size_ - 2] != *seperator) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]dir_name format not right, dir=%s, ret=%d\n", dir_name.data_, ret); + } else { + cos_status_t *cos_ret = NULL; + cos_list_object_params_t *params = NULL; + + if (NULL == (params = cos_create_list_object_params(ctx->mem_pool))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + cos_warn_log("[COS]fail to create list object params, ret=%d\n", ret); + } else { + cos_str_set(¶ms->prefix, dir_name.data_); + cos_str_set(¶ms->marker, "/"); + params->max_ret = 1000; + cos_string_t bucket; + cos_str_set(&bucket, bucket_name.data_); + cos_list_object_content_t *content = NULL; + + cos_list_t to_delete_object_list; + cos_list_t deleted_object_list; + cos_object_key_t *to_delete_object = NULL; + cos_table_t *resp_headers = NULL; + + cos_list_init(&to_delete_object_list); + cos_list_init(&deleted_object_list); + + do { + // List objects from cos, limit 1000. + cos_ret = cos_list_object(ctx->options, &bucket, params, NULL); + if (NULL == cos_ret || !cos_status_is_ok(cos_ret)) { + if (NULL != cos_ret && COS_SERVICE_UNAVAILABLE == cos_ret->code) { + // Request is limited by cos, returne "SlowDown, Reduce your request rate". + ret = OB_IO_LIMIT; + } else { + ret = OB_COS_ERROR; + } + cos_warn_log("[COS]fail to list object, ret=%d\n", ret); + log_status(cos_ret); + } else { + // Traverse the returned objects + cos_list_for_each_entry(cos_list_object_content_t, content, ¶ms->object_list, node) { + // Check if the prefix of returned object key match the dir_name + size_t dir_name_str_len = strlen(dir_name.data_); + if (0 != memcmp(content->key.data, dir_name.data_, dir_name_str_len)) { + ret = OB_COS_ERROR; + cos_warn_log("[COS]returned object prefix not match, dir=%s, object=%s, ret=%d\n", dir_name.data_, content->key.data, ret); + } else if (NULL == (to_delete_object = cos_create_cos_object_key(ctx->mem_pool))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + cos_warn_log("[COS]create to delete object key memory failed, dir=%s, object=%s, ret=%d\n", dir_name.data_, content->key.data, ret); + } else { + // Mark object to do delete + cos_str_set(&to_delete_object->key, content->key.data); + cos_list_add_tail(&to_delete_object->node, &to_delete_object_list); + } + + if (OB_SUCCESS != ret) { + break; + } + } + + // Delete current batch of objects, limit 1000. + if (OB_SUCCESS == ret && !cos_list_empty(&to_delete_object_list)) { + if (NULL == (cos_ret = cos_delete_objects(ctx->options, &bucket, &to_delete_object_list, COS_FALSE, &resp_headers, &deleted_object_list)) + || !cos_status_is_ok(cos_ret)) { + convert_io_error(cos_ret, ret); + cos_warn_log("[COS]delete objects failed, ret=%d.\n", ret); + log_status(cos_ret); + } + + // Traverse the deleted objects + cos_list_object_content_t *deleted_object = NULL; + cos_list_for_each_entry(cos_list_object_content_t, deleted_object, &deleted_object_list, node) { + ++deleted_cnt; + } + } + + // Delete next batch of objects. + char *next_marker_str = NULL; + if (OB_SUCCESS == ret && COS_TRUE == params->truncated) { + if (NULL == (next_marker_str = apr_psprintf(ctx->mem_pool, "%.*s", params->next_marker.len, params->next_marker.data))) { + ret = OB_COS_ERROR; + cos_warn_log("[COS]get next marker is null, ret=%d\n", ret); + } else { + cos_str_set(¶ms->marker, next_marker_str); + cos_list_init(¶ms->object_list); + cos_list_init(¶ms->common_prefix_list); + + cos_list_init(&to_delete_object_list); + cos_list_init(&deleted_object_list); + } + } + } + } while (COS_TRUE == params->truncated && OB_SUCCESS == ret); + } + } + + return ret; +} + +int ObCosWrapper::update_object_modified_ts( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name) +{ + int ret = OB_SUCCESS; + + CosContext *ctx = reinterpret_cast(h); + + if (NULL == h) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]cos handle is null, ret=%d\n", ret); + } else if (bucket_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]bucket name is null, ret=%d\n", ret); + } else if (object_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]object name is null, ret=%d\n", ret); + } else { + cos_string_t bucket; + cos_string_t object; + cos_str_set(&bucket, bucket_name.data_); + cos_str_set(&object, object_name.data_); + cos_table_t *resp_headers = NULL; + cos_status_t *cos_ret = NULL; + + cos_table_t *headers = NULL; + cos_copy_object_params_t *params = NULL; + if (NULL == (headers = cos_table_make(ctx->mem_pool, 2))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + cos_warn_log("[COS]failed to allocate header memory, ret=%d\n", ret); + } else if (NULL == (params = cos_create_copy_object_params(ctx->mem_pool))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + cos_warn_log("[COS]failed to allocate copy object param memory, ret=%d\n", ret); + } else { + // In Cos, things are different. We cannot refresh the object's latest modified + // time by updating ACL just like what we do for OSS. Instead, we use "PUT Object - Copy". + // See https://cloud.tencent.com/document/product/436/10881. + + // Let copied object replace the source meta, default is copy. + apr_table_add(headers, "x-cos-metadata-directive", "Replaced"); + + // Following is not needed. To delete later. + // apr_table_add(headers, "Cache-Control", "no-cache"); + // Let the source and destination objects same. + cos_ret = cos_copy_object(ctx->options, &bucket, &object, &ctx->options->config->endpoint, &bucket, &object, headers, params, &resp_headers); + if (NULL == cos_ret || !cos_status_is_ok(cos_ret)) { + convert_io_error(cos_ret, ret); + cos_warn_log("[COS]fail to call copy object, ret=%d\n", ret); + log_status(cos_ret); + } else { + cos_warn_log("[COS]cos put copy succeeded, ret=%d.\n", ret); + } + } + } + + return ret; +} + +int ObCosWrapper::pread( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name, + int64_t offset, + char *buf, + int64_t buf_size, + int64_t &read_size) +{ + int ret = OB_SUCCESS; + + CosContext *ctx = reinterpret_cast(h); + read_size = 0; + + if (NULL == h) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]cos handle is null, ret=%d\n", ret); + } else if (bucket_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]bucket name is null, ret=%d\n", ret); + } else if (object_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]object name is null, ret=%d\n", ret); + } else if (NULL == buf || 0 >= buf_size) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]buffer is null, ret=%d\n", ret); + } else { + cos_string_t bucket; + cos_string_t object; + cos_str_set(&bucket, bucket_name.data_); + cos_str_set(&object, object_name.data_); + cos_table_t *headers = NULL; + cos_table_t *resp_headers = NULL; + cos_status_t *cos_ret = NULL; + + cos_list_t buffer; + cos_list_init(&buffer); + + const int COS_RANGE_SIZE = 256; + const char* const COS_RANGE_KEY = "Range"; + + char range_size[COS_RANGE_SIZE]; + // Cos read range of [10, 100] include the start offset 10 and the end offset 100. + // But what we except is [10, 100) which does not include the end. + // So we subtract 1 from the end. + int n = snprintf(range_size, COS_RANGE_SIZE, "bytes=%ld-%ld", offset, offset + buf_size - 1); + if (0 >= n || COS_RANGE_SIZE <= n) { + ret = OB_SIZE_OVERFLOW; + cos_warn_log("[COS]fail to format range size,n=%d, ret=%d\n", n, ret); + } else if (NULL == (headers = cos_table_make(ctx->mem_pool, 1))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + cos_warn_log("[COS]fail to make cos headers, ret=%d\n", ret); + } else { + apr_table_set(headers, COS_RANGE_KEY, range_size); + + if (NULL == (cos_ret = cos_get_object_to_buffer(ctx->options, &bucket, &object, headers, NULL, &buffer, &resp_headers)) || + !cos_status_is_ok(cos_ret)) { + convert_io_error(cos_ret, ret); + cos_warn_log("[COS]fail to get object to buffer, ret=%d\n", ret); + log_status(cos_ret); + } else { + int64_t size = 0; + int64_t buf_pos = 0; + cos_buf_t *content = NULL; + cos_list_for_each_entry(cos_buf_t, content, &buffer, node) { + size = cos_buf_size(content); + if (buf_pos + size > buf_size) { + ret = OB_COS_ERROR; + cos_warn_log("[COS]unexpected error, too much data returned, ret=%d, range_size=%s, buf_pos=%ld, size=%ld, req_id=%s.\n", ret, range_size, buf_pos, size, cos_ret->req_id); + log_status(cos_ret); + break; + } else { + // copy to buf + memcpy(buf + buf_pos, content->pos, (size_t)size); + buf_pos += size; + read_size += size; + } + } + } + } + } + + return ret; +} + +int ObCosWrapper::get_object( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name, + char *buf, + int64_t buf_size, + int64_t &read_size) +{ + int ret = OB_SUCCESS; + + CosContext *ctx = reinterpret_cast(h); + read_size = 0; + + if (NULL == h) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]cos handle is null, ret=%d\n", ret); + } else if (bucket_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]bucket name is null, ret=%d\n", ret); + } else if (object_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]object name is null, ret=%d\n", ret); + } else if (NULL == buf || 0 >= buf_size) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]buffer is null, ret=%d\n", ret); + } else { + cos_string_t bucket; + cos_string_t object; + cos_str_set(&bucket, bucket_name.data_); + cos_str_set(&object, object_name.data_); + cos_table_t *headers = NULL; + cos_table_t *resp_headers = NULL; + cos_status_t *cos_ret = NULL; + + cos_list_t buffer; + cos_list_init(&buffer); + + if (NULL == (cos_ret = cos_get_object_to_buffer(ctx->options, &bucket, &object, headers, NULL, &buffer, &resp_headers)) || + !cos_status_is_ok(cos_ret)) { + convert_io_error(cos_ret, ret); + cos_warn_log("[COS]fail to get object to buffer, ret=%d\n", ret); + log_status(cos_ret); + } else { + int64_t size = 0; + int64_t buf_pos = 0; + cos_buf_t *content = NULL; + cos_list_for_each_entry(cos_buf_t, content, &buffer, node) { + size = cos_buf_size(content); + if (buf_pos + size > buf_size) { + ret = OB_COS_ERROR; + cos_warn_log("[COS]unexpected error, too much data returned, ret=%d, buf_pos=%ld, size=%ld, buf_size=%ld, req_id=%s.\n", ret, buf_pos, size, buf_size, cos_ret->req_id); + log_status(cos_ret); + break; + } else { + // copy to buf + memcpy(buf + buf_pos, content->pos, (size_t)size); + buf_pos += size; + read_size += size; + } + } + } + } + + return ret; +} + +int ObCosWrapper::is_object_tagging( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name, + bool &is_tagging) +{ + int ret = OB_SUCCESS; + + CosContext *ctx = reinterpret_cast(h); + + if (NULL == h) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]cos handle is null, ret=%d\n", ret); + } else if (bucket_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]bucket name is null, ret=%d\n", ret); + } else if (object_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]object name is null, ret=%d\n", ret); + } else { + cos_string_t bucket; + cos_string_t object; + cos_str_set(&bucket, bucket_name.data_); + cos_str_set(&object, object_name.data_); + cos_table_t *headers = NULL; + cos_table_t *resp_headers = NULL; + cos_status_t *cos_ret = NULL; + cos_string_t version_id = cos_string(""); + cos_tagging_params_t *result = NULL; + + if (NULL == (result = cos_create_tagging_params(ctx->mem_pool))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + cos_warn_log("[COS]fail to create tagging params, ret=%d\n", ret); + } else if (NULL == (cos_ret = cos_get_object_tagging(ctx->options, &bucket, &object, + &version_id, headers, result, &resp_headers)) || !cos_status_is_ok(cos_ret)) { + convert_io_error(cos_ret, ret); + cos_warn_log("[COS]fail to get object tagging, ret=%d\n", ret); + log_status(cos_ret); + } else { + cos_tagging_tag_t *tag = NULL; + cos_list_for_each_entry(cos_tagging_tag_t, tag, &result->node, node) { + char key_str[MAX_TAGGING_STR_LEN]; + char value_str[MAX_TAGGING_STR_LEN]; + + int key_n = snprintf(key_str, MAX_TAGGING_STR_LEN, "%.*s", tag->key.len, tag->key.data); + int val_n = snprintf(value_str, MAX_TAGGING_STR_LEN, "%.*s", tag->key.len, tag->value.data); + if (0 >= key_n || MAX_TAGGING_STR_LEN <= key_n) { + ret = OB_SIZE_OVERFLOW; + cos_warn_log("[COS]fail to format tag, key_n=%d, ret=%d\n", key_n, ret); + } else if (0 >= val_n || MAX_TAGGING_STR_LEN <= val_n) { + ret = OB_SIZE_OVERFLOW; + cos_warn_log("[COS]fail to format tag, val_n=%d, ret=%d\n", val_n, ret); + } else if (0 == strcmp("delete_mode", key_str) && 0 == strcmp("tagging", value_str)) { + is_tagging = true; + } + if (OB_SUCCESS != ret) { + break; + } + } + } + } + + return ret; +} + +int ObCosWrapper::list_objects( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &dir_name, + const CosStringBuffer &next_marker, + handleObjectNameFunc handle_object_name_f, + void *arg) +{ + int ret = OB_SUCCESS; + + const char *seperator = "/"; + // dir_name must be end with '/\0' + const int32_t min_dir_name_str_len = 2; + + CosContext *ctx = reinterpret_cast(h); + + if (NULL == h) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]cos handle is null, ret=%d\n", ret); + } else if (bucket_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]bucket name is null, ret=%d\n", ret); + } else if (NULL == dir_name.data_ || min_dir_name_str_len >= dir_name.size_) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]dir_name size too short, size=%d, ret=%d\n", dir_name.size_, ret); + } else if (dir_name.data_[dir_name.size_ - 2] != *seperator) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]dir_name format not right, dir=%s, ret=%d\n", dir_name.data_, ret); + } else if (next_marker.empty()) { + // next_marker end with '\0', its size must be > 0 + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]next_marker is null, ret=%d\n", ret); + } else if (NULL == handle_object_name_f) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]handle_object_name_f is null, ret=%d\n", ret); + } else { + cos_status_t *cos_ret = NULL; + cos_list_object_params_t *params = NULL; + + if (NULL == (params = cos_create_list_object_params(ctx->mem_pool))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + cos_warn_log("[COS]fail to create list object params, ret=%d\n", ret); + } else { + cos_str_set(¶ms->prefix, dir_name.data_); + cos_str_set(¶ms->marker, next_marker.data_); + params->max_ret = 1000; + cos_string_t bucket; + cos_str_set(&bucket, bucket_name.data_); + cos_list_object_content_t *content = NULL; + + CosListObjPara para; + para.arg_ = arg; + para.type_ = CosListObjPara::CosListType::COS_LIST_CB_ARG; + + do { + cos_table_t *resp_headers = NULL; + // List objects from cos + if (NULL == (cos_ret = cos_list_object(ctx->options, &bucket, params, &resp_headers)) + || !cos_status_is_ok(cos_ret)) { + if (NULL != cos_ret && COS_SERVICE_UNAVAILABLE == cos_ret->code) { + // Request is limited by cos, returne "SlowDown, Reduce your request rate". + ret = OB_IO_LIMIT; + } else { + convert_io_error(cos_ret, ret); + } + cos_warn_log("[COS]fail to list object, ret=%d\n", ret); + log_status(cos_ret); + } else { + char *request_id = (char*)apr_table_get(resp_headers, "x-cos-request-id"); + // Traverse the returned objects + cos_list_for_each_entry(cos_list_object_content_t, content, ¶ms->object_list, node) { + // check if the prefix of returned object key match the dir_name + size_t dir_name_str_len = strlen(dir_name.data_); + if (0 != memcmp(content->key.data, dir_name.data_, dir_name_str_len)) { + ret = OB_COS_ERROR; + cos_warn_log("[COS]returned object prefix not match, dir=%s, object=%s, requestid=%s, ret=%d\n", dir_name.data_, content->key.data, request_id, ret); + } else if (NULL == content->size.data || 0 == content->size.len) { + ret = OB_COS_ERROR; + cos_warn_log("[COS]returned object size is empty, dir=%s, object=%s, requestid=%s, ret=%d\n", dir_name.data_, content->key.data, request_id, ret); + } else { + // Callback to handle the object name, it is a absolute path. + para.cur_full_path_slice_name_ = content->key.data; + para.full_path_size_ = content->key.len; + para.cur_object_size_ = cos_atoi64(content->size.data); + ret = handle_object_name_f(para); + if (OB_SUCCESS != ret){ + cos_warn_log("[COS]handle object name failed, ret=%d, object=%s, requestid=%s\n", ret, content->key.data, request_id); + } + } + + if (OB_SUCCESS != ret || !para.next_flag_) { + break; + } + } + + char *next_marker_str = NULL; + if (OB_SUCCESS == ret && para.next_flag_ && COS_TRUE == params->truncated) { + if (NULL == (next_marker_str = apr_psprintf(ctx->mem_pool, "%.*s", params->next_marker.len, params->next_marker.data))) { + ret = OB_COS_ERROR; + cos_warn_log("[COS]get next marker is null, ret=%d\n", ret); + } else { + cos_str_set(¶ms->marker, next_marker_str); + cos_list_init(¶ms->object_list); + cos_list_init(¶ms->common_prefix_list); + } + } + } + } while (COS_TRUE == params->truncated && OB_SUCCESS == ret && para.next_flag_); + } + } + + return ret; +} + +int ObCosWrapper::list_directories( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &dir_name, + const CosStringBuffer &next_marker, + const CosStringBuffer &delimiter, + handleDirectoryFunc handle_directory_name_f, + void *arg) +{ + int ret = OB_SUCCESS; + + const char *seperator = "/"; + // dir_name must be end with '/\0' + const int32_t min_dir_name_str_len = 2; + + CosContext *ctx = reinterpret_cast(h); + + if (NULL == h) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]cos handle is null, ret=%d\n", ret); + } else if (bucket_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]bucket name is null, ret=%d\n", ret); + } else if (NULL == dir_name.data_ || min_dir_name_str_len >= dir_name.size_) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]dir_name size too short, size=%d, ret=%d\n", dir_name.size_, ret); + } else if (dir_name.data_[dir_name.size_ - 2] != *seperator) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]dir_name format not right, dir=%s, ret=%d\n", dir_name.data_, ret); + } else if (next_marker.empty()) { + // next_marker end with '\0', its size must be > 0 + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]next_marker is null, ret=%d\n", ret); + } else if (NULL == handle_directory_name_f) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]handle_directory_name_f is null, ret=%d\n", ret); + } else { + cos_status_t *cos_ret = NULL; + cos_list_object_params_t *params = NULL; + + if (NULL == (params = cos_create_list_object_params(ctx->mem_pool))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + cos_warn_log("[COS]fail to create list object params, ret=%d\n", ret); + } else { + cos_str_set(¶ms->prefix, dir_name.data_); + cos_str_set(¶ms->marker, next_marker.data_); + cos_str_set(¶ms->delimiter, delimiter.data_); + params->max_ret = 1000; + cos_string_t bucket; + cos_str_set(&bucket, bucket_name.data_); + cos_list_object_common_prefix_t *common_prefix = NULL; + + // A mark which indicates whether to process the next object. + do { + cos_table_t *resp_headers = NULL; + // List objects from cos + if (NULL == (cos_ret = cos_list_object(ctx->options, &bucket, params, &resp_headers)) + || !cos_status_is_ok(cos_ret)) { + if (NULL != cos_ret && COS_SERVICE_UNAVAILABLE == cos_ret->code) { + // Request is limited by cos, returne "SlowDown, Reduce your request rate". + ret = OB_IO_LIMIT; + } else { + ret = OB_COS_ERROR; + } + cos_warn_log("[COS]fail to list object, ret=%d\n", ret); + log_status(cos_ret); + } else { + char *request_id = (char*)apr_table_get(resp_headers, "x-cos-request-id"); + // Traverse the returned objects + cos_list_for_each_entry(cos_list_object_common_prefix_t, common_prefix, ¶ms->common_prefix_list, node) { + // check if the prefix of returned object key match the dir_name + const size_t dir_name_str_len = strlen(dir_name.data_); + const size_t prefix_str_len = strlen(common_prefix->prefix.data); + if (prefix_str_len < dir_name_str_len) { + ret = OB_COS_ERROR; + cos_warn_log("[COS]prefix str len should not smaller than dir name str len. prefix str len : %lu, " + "dir name str len : %lu, ret = %d \n", prefix_str_len, dir_name_str_len, ret); + } else if (0 != memcmp(common_prefix->prefix.data, dir_name.data_, dir_name_str_len)) { + ret = OB_COS_ERROR; + cos_warn_log("[COS]returned object prefix not match, dir=%s, object=%s, requestid=%s, ret=%d\n", dir_name.data_, common_prefix->prefix.data, request_id, ret); + } else { + // Callback to handle the object name, it is a absolute path. + const int64_t object_size = common_prefix->prefix.len - dir_name_str_len; //include '/' + CosListObjPara::CosListType type = CosListObjPara::CosListType::COS_LIST_CB_ARG; + ret = handle_directory_name_f(arg, type, common_prefix->prefix.data + dir_name_str_len, object_size); + if (OB_SUCCESS != ret) { + // Something wrong happened when handle object name + cos_warn_log("[COS]handle object name failed, ret=%d, object=%s, requestid=%s\n", ret, common_prefix->prefix.data, request_id); + } + } + + if (OB_SUCCESS != ret) { + break; + } + } + + char *next_marker_str = NULL; + if (OB_SUCCESS == ret && COS_TRUE == params->truncated) { + if (NULL == (next_marker_str = apr_psprintf(ctx->mem_pool, "%.*s", params->next_marker.len, params->next_marker.data))) { + ret = OB_COS_ERROR; + cos_warn_log("[COS]get next marker is null, ret=%d\n", ret); + } else { + cos_str_set(¶ms->marker, next_marker_str); + cos_list_init(¶ms->object_list); + cos_list_init(¶ms->common_prefix_list); + } + } + } + } while (COS_TRUE == params->truncated && OB_SUCCESS == ret); + } + } + + return ret; +} + +int ObCosWrapper::is_empty_directory( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &dir_name, + bool &is_empty_dir) +{ + int ret = OB_SUCCESS; + is_empty_dir = false; + const char *seperator = "/"; + // dir_name must be end with '/\0' + const int32_t min_dir_name_str_len = 2; + + CosContext *ctx = reinterpret_cast(h); + + if (NULL == h) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]cos handle is null, ret=%d\n", ret); + } else if (bucket_name.empty()) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]bucket name is null, ret=%d\n", ret); + } else if (NULL == dir_name.data_ || min_dir_name_str_len >= dir_name.size_) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]dir_name size too short, size=%d, ret=%d\n", dir_name.size_, ret); + } else if (dir_name.data_[dir_name.size_ - 2] != *seperator) { + ret = OB_INVALID_ARGUMENT; + cos_warn_log("[COS]dir_name format not right, dir=%s, ret=%d\n", dir_name.data_, ret); + } else { + cos_status_t *cos_ret = NULL; + cos_list_object_params_t *params = NULL; + + if (NULL == (params = cos_create_list_object_params(ctx->mem_pool))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + cos_warn_log("[COS]fail to create list object params, ret=%d\n", ret); + } else { + cos_str_set(¶ms->prefix, dir_name.data_); + // it just decides if it is not empty + params->max_ret = 1; + cos_string_t bucket; + cos_str_set(&bucket, bucket_name.data_); + cos_table_t *resp_headers = NULL; + // List objects from cos + if (NULL == (cos_ret = cos_list_object(ctx->options, &bucket, params, &resp_headers)) + || !cos_status_is_ok(cos_ret)) { + if (NULL != cos_ret && COS_SERVICE_UNAVAILABLE == cos_ret->code) { + // Request is limited by cos, returne "SlowDown, Reduce your request rate". + ret = OB_IO_LIMIT; + } else { + ret = OB_COS_ERROR; + } + cos_warn_log("[COS]fail to list object, ret=%d\n", dir_name.data_, ret); + log_status(cos_ret); + } else { + is_empty_dir = static_cast(cos_list_empty(¶ms->object_list)); + } + } + } + + return ret; +} + +} // qcloud_cos +} // common +} // oceanbase diff --git a/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.h b/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.h new file mode 100644 index 000000000..2fac5d4c6 --- /dev/null +++ b/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.h @@ -0,0 +1,325 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef SRC_LIBRARY_SRC_LIB_OB_COS_WRAPPER_H_ +#define SRC_LIBRARY_SRC_LIB_OB_COS_WRAPPER_H_ + +#include +#include "ob_singleton.h" +#include + +namespace oceanbase +{ +namespace common +{ + +// Naming qcloud_cos to avoid conflicts with the function called cos. +namespace qcloud_cos +{ + +#define OB_PUBLIC_API __attribute__ ((visibility ("default"))) + +class OB_PUBLIC_API ObCosEnv : public ObSingleton +{ +public: + struct Conf + { + static const int64_t MIN_COS_SLICE_SIZE = 32000; + static const int64_t MAX_COS_SLICE_SIZE = 10000000; + int64_t slice_size = 524288; // default 512K + }; + + ObCosEnv() : ObSingleton(), is_inited_(false) {} + // global init cos env resource, must and only can be called once + int init(); + + void destroy(); + +private: + bool is_inited_; + Conf conf_; +}; + +static constexpr int MAX_TAGGING_STR_LEN = 16; + +// COS domain name structure: bucket_name-appid.endpoint +struct OB_PUBLIC_API ObCosAccount +{ + // max domain length + static constexpr int MAX_COS_DOMAIN_LENGTH = 1536; + // max endpoint length + static constexpr int MAX_COS_ENDPOINT_LENGTH = 128; + // max access id length + static constexpr int MAX_COS_ACCESS_ID_LENGTH = 128; + // max access key length + static constexpr int MAX_COS_ACCESS_KEY_LENGTH = 128; + // max appid length + static constexpr int MAX_COS_APPID_LENGTH = 128; + + // cos endpoint + char endpoint_[MAX_COS_ENDPOINT_LENGTH]; + // your access id + char access_id_[MAX_COS_ACCESS_ID_LENGTH]; + // your access key + char access_key_[MAX_COS_ACCESS_KEY_LENGTH]; + // your appid + char appid_[MAX_COS_APPID_LENGTH]; + // cos object delete mode + char delete_mode_[MAX_TAGGING_STR_LEN]; + + ObCosAccount() + { + memset(this, 0, sizeof(*this)); + } + + ~ObCosAccount() {} + + void clear() + { + memset(this, 0, sizeof(*this)); + } + + // parse endpoint, access id, access key and appid from storage_info + // You must call parse_from first before using any field. + int parse_from(const char *storage_info, uint32_t size); + +private: + // make sure 'value' end with '\0' + int set_field(const char *value, char *field, uint32_t length); +}; + + +// The string does not own the buffer. +struct OB_PUBLIC_API CosStringBuffer +{ + const char *data_; + int32_t size_; + + CosStringBuffer() : data_(NULL), size_(0) {} + + CosStringBuffer(const char *ptr, int len) : data_(ptr), size_(len) {} + + ~CosStringBuffer() {} + + bool empty() const + { + return NULL == data_ || 0 >= size_; + } + + bool prefix_match(const char *str) const + { + bool match = false; + int32_t str_len = 0; + if (NULL != str) { + str_len = static_cast(strlen(str)); + } + + if (data_ == str) { + match = (size_ >= str_len ? true : false); + } else if (size_ < str_len) { + // skip + } else if (0 == strncasecmp(data_, str, str_len)) { + match = true; + } + return match; + } +}; + + +struct OB_PUBLIC_API CosObjectMeta +{ + enum CosObjectType + { + COS_OBJ_INVALID, + COS_OBJ_NORMAL, + COS_OBJ_APPENDABLE, + }; + + int64_t file_length_; + int64_t last_modified_ts_; + int type_; + + CosObjectMeta() + { + reset(); + } + + ~CosObjectMeta() {} + + void reset() + { + memset(this, 0, sizeof(*this)); + file_length_ = -1; + type_ = CosObjectType::COS_OBJ_INVALID; + } +}; + + +/*= Custom memory allocation functions */ +typedef void* (*OB_COS_allocFunction) (void* opaque, size_t size); +typedef void (*OB_COS_freeFunction) (void* opaque, void* address); +typedef struct { OB_COS_allocFunction customAlloc; OB_COS_freeFunction customFree; void* opaque; } OB_COS_customMem; + +class OB_PUBLIC_API ObCosWrapper +{ +public: + struct Handle {}; + + // alloc_f is used to allocate the handle's memory. + // You need to call destroy_cos_handle when handle is not use + // any more. + static int create_cos_handle( + OB_COS_customMem &custom_mem, + const struct ObCosAccount &account, + Handle **h); + + // You can not use handle any more after destroy_cos_handle is called. + static void destroy_cos_handle(Handle *h); + + // Put an object to cos, the new object will overwrite the old one if object exist. + static int put( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name, + const char *buffer, + const int64_t size); + + // Append content to the specific object from the @offset position in cos. + static int append( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name, + const char *buf, + const int64_t buf_size, + const int64_t offset); + + // Get object meta + static int head_object_meta( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name, + bool &is_exist, + CosObjectMeta &meta); + + // Delete one object from cos. + static int del( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name); + + // Tag one object from cos + static int tag( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name); + + + // Delete all objects that match the same dir_name prefix. + static int del_objects_in_dir( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &dir_name, + int64_t &deleted_cnt); + + // Update object last modofied time. + static int update_object_modified_ts( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name); + + // Random read data of specific range from object. + // Buffer's memory is provided by user. + static int pread( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name, + int64_t offset, + char *buf, + int64_t buf_size, + int64_t &read_size); + + // Get whole object + // Buffer's memory is provided by user. + static int get_object( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name, + char *buf, + int64_t buf_size, + int64_t &read_size); + + static int is_object_tagging( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &object_name, + bool &is_tagging); + + struct CosListObjPara { + enum class CosListType + { + COS_LIST_INVALID, + COS_LIST_CB_ARG + }; + + CosListObjPara() + : arg_(NULL), cur_full_path_slice_name_(NULL), + full_path_size_(0), cur_object_size_(0), + next_flag_(false), type_(CosListType::COS_LIST_INVALID) + { + last_container_name_.d_name[0] = '\0'; + last_container_name_.d_type = DT_REG; + } + + void* arg_; + char* cur_full_path_slice_name_; + struct dirent last_container_name_; + int64_t full_path_size_; + int64_t cur_object_size_; + bool next_flag_; + CosListType type_; + }; + + typedef int (*handleObjectNameFunc)(CosListObjPara&); + typedef int (*handleDirectoryFunc)(void*, const CosListObjPara::CosListType, const char*, int64_t); + // List objects in the same directory, include all objects + // in the inner sub directories. + // dir_name must be end with "/\0". + static int list_objects( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &dir_name, + const CosStringBuffer &next_marker, + handleObjectNameFunc handle_object_name_f, + void *arg); + + static int list_directories( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &dir_name, + const CosStringBuffer &next_marker, + const CosStringBuffer &delimiter, + handleDirectoryFunc handle_directory_name_f, + void *arg); + + static int is_empty_directory( + Handle *h, + const CosStringBuffer &bucket_name, + const CosStringBuffer &dir_name, + bool &is_empty_dir); +}; + + +#undef OB_PUBLIC_API +} +} +} +#endif diff --git a/deps/oblib/src/lib/restore/cos/ob_cos_wrapper_handle.cpp b/deps/oblib/src/lib/restore/cos/ob_cos_wrapper_handle.cpp new file mode 100644 index 000000000..0ca3d684e --- /dev/null +++ b/deps/oblib/src/lib/restore/cos/ob_cos_wrapper_handle.cpp @@ -0,0 +1,235 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "lib/utility/ob_print_utils.h" +#include "ob_cos_wrapper_handle.h" + +namespace oceanbase +{ +namespace common +{ +using namespace oceanbase::common; + +/** + * ------------------------------ObCosMemAllocator--------------------- + */ +ObCosMemAllocator::ObCosMemAllocator() + : allocator_(ObModIds::BACKUP, OB_MALLOC_NORMAL_BLOCK_SIZE) +{ +} + +ObCosMemAllocator::~ObCosMemAllocator() +{ +} + +void* ObCosMemAllocator::alloc(size_t size) +{ + return allocator_.alloc(size); +} + +void ObCosMemAllocator::free(void *addr) +{ + allocator_.free(addr); +} + +void ObCosMemAllocator::reuse() +{ + allocator_.reuse(); +} + + +// Memory function used for cos context +static void *ob_cos_malloc(void *opaque, size_t size) +{ + void *buf = NULL; + if (NULL != opaque) { + ObCosMemAllocator *allocator = reinterpret_cast (opaque); + buf = allocator->alloc(size); + } + return buf; +} + +static void ob_cos_free(void *opaque, void *address) +{ + if (NULL != opaque) { + ObCosMemAllocator *allocator = reinterpret_cast (opaque); + allocator->free(address); + } +} + +/*--------------------------------ObCosWrapperHandle-----------------------------------*/ +ObCosWrapperHandle::ObCosWrapperHandle() + : is_inited_(false), handle_(nullptr), cos_account_(), allocator_(), + delete_mode_(ObIStorageUtil::DELETE) +{} + +int ObCosWrapperHandle::init(const ObObjectStorageInfo *storage_info) +{ + int ret = OB_SUCCESS; + char storage_info_str[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 }; + + if (IS_INIT) { + ret = OB_INIT_TWICE; + OB_LOG(WARN, "init twice", K(ret)); + } else if (OB_ISNULL(storage_info)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "storage info is null", K(ret)); + } else if (OB_FAIL(storage_info->get_storage_info_str(storage_info_str, sizeof(storage_info_str)))) { + OB_LOG(WARN, "fail to get cos storage info str", K(ret), K(storage_info)); + } else if (OB_FAIL(cos_account_.parse_from(storage_info_str, strlen(storage_info_str)))) { + OB_LOG(WARN, "fail to parse cos account from storage info str", K(ret)); + } else if (strlen(cos_account_.delete_mode_) > 0 && OB_FAIL(set_delete_mode(cos_account_.delete_mode_))) { + OB_LOG(WARN, "fail to set cos delete mode", K(cos_account_.delete_mode_), K(ret)); + } { + is_inited_ = true; + } + return ret; +} + +void ObCosWrapperHandle::reset() +{ + destroy_cos_handle(); + is_inited_ = false; + delete_mode_ = ObIStorageUtil::DELETE; +} + +int ObCosWrapperHandle::create_cos_handle() +{ + int ret = OB_SUCCESS; + qcloud_cos::OB_COS_customMem cos_mem = {ob_cos_malloc, ob_cos_free, &allocator_}; + if (OB_NOT_NULL(handle_)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "handle is not null", K(ret)); + } else if (OB_FAIL(qcloud_cos::ObCosWrapper::create_cos_handle(cos_mem, cos_account_, &handle_))) { + OB_LOG(WARN, "failed to create cos handle", K(ret)); + } else if (OB_ISNULL(handle_)) { + ret = OB_COS_ERROR; + OB_LOG(WARN, "create handle succeed, but returned handle is null", K(ret)); + } + + return ret; +} + +void ObCosWrapperHandle::destroy_cos_handle() +{ + if (nullptr != handle_) { + qcloud_cos::ObCosWrapper::destroy_cos_handle(handle_); + handle_ = nullptr; + } +} + +int ObCosWrapperHandle::build_bucket_and_object_name(const ObString &uri) +{ + int ret = OB_SUCCESS; + ObString::obstr_size_t bucket_start = 0; + ObString::obstr_size_t bucket_end = 0; + ObString::obstr_size_t object_start = 0; + char* bucket_name_buff = NULL; + char* object_name_buff = NULL; + + if (!uri.prefix_match(OB_COS_PREFIX)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "uri is invalid ", KCSTRING(OB_COS_PREFIX), K(uri), K(ret)); + } else { + bucket_start = static_cast(strlen(OB_COS_PREFIX)); + for (int64_t i = bucket_start; OB_SUCC(ret) && i < uri.length() - 2; i++) { + if ('/' == *(uri.ptr() + i) && '/' == *(uri.ptr() + i + 1)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "uri has two // ", K(uri), K(ret), K(i)); + break; + } + } + } + + if (OB_SUCC(ret)) { + if (OB_ISNULL(bucket_name_buff = static_cast(allocator_.alloc(OB_MAX_URI_LENGTH)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "failed to alloc bucket name buff", K(ret), K(uri)); + } else if (OB_ISNULL(object_name_buff = static_cast(allocator_.alloc(OB_MAX_URI_LENGTH)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "failed to alloc object name buff", K(ret), K(uri)); + } else { + //make sure this last always 0 + bucket_name_buff[0] = '\0'; + object_name_buff[0] = '\0'; + } + } + + // parse bucket name + if (OB_SUCC(ret)) { + if (bucket_start >= uri.length()) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "bucket name is NULL", K(uri), K(ret)); + } else { + for (bucket_end = bucket_start; OB_SUCC(ret) && bucket_end < uri.length(); ++bucket_end) { + if ('/' == *(uri.ptr() + bucket_end)) { + ObString::obstr_size_t bucket_length = bucket_end - bucket_start; + //must end with '\0' + if (OB_FAIL(databuff_printf(bucket_name_buff, OB_MAX_URI_LENGTH, "%.*s", bucket_length, uri.ptr() + bucket_start))) { + OB_LOG(WARN, "fail to deep copy bucket", K(uri), K(bucket_start), K(bucket_length), K(ret)); + } else { + bucket_name_.assign_ptr(bucket_name_buff, strlen(bucket_name_buff) + 1);// must include '\0' + } + break; + } + } + + if (OB_SUCC(ret) && bucket_end == uri.length()) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "bucket name is invalid", K(uri), K(ret)); + } + } + } + + // parse the object name + if (OB_SUCC(ret)) { + if (bucket_end + 1 >= uri.length()) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "object name is NULL", K(uri), K(ret)); + } else { + object_start = bucket_end + 1; + ObString::obstr_size_t object_length = uri.length() - object_start; + //must end with '\0' + if (OB_FAIL(databuff_printf(object_name_buff, OB_MAX_URI_LENGTH, "%.*s", object_length, uri.ptr() + object_start))) { + OB_LOG(WARN, "fail to deep copy object", K(uri), K(object_start), K(object_length), K(ret)); + } else { + object_name_.assign_ptr(object_name_buff, strlen(object_name_buff) + 1);//must include '\0' + } + } + } + + if (OB_SUCC(ret)) { + OB_LOG(DEBUG, "get bucket object name", K(uri), K_(bucket_name), K_(object_name)); + } + + return ret; +} + +int ObCosWrapperHandle::set_delete_mode(const char *parameter) +{ + int ret = OB_SUCCESS; + if (NULL == parameter) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid args", K(ret), KP(parameter)); + } else if (0 == strcmp(parameter, "delete")) { + delete_mode_ = ObIStorageUtil::DELETE; + } else if (0 == strcmp(parameter, "tagging")) { + delete_mode_ = ObIStorageUtil::TAGGING; + } else { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "delete mode is invalid", K(ret), K(parameter)); + } + return ret; +} + +} // common +} // oceanbase diff --git a/deps/oblib/src/lib/restore/cos/ob_cos_wrapper_handle.h b/deps/oblib/src/lib/restore/cos/ob_cos_wrapper_handle.h new file mode 100644 index 000000000..3a5671f31 --- /dev/null +++ b/deps/oblib/src/lib/restore/cos/ob_cos_wrapper_handle.h @@ -0,0 +1,77 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef SRC_LIBRARY_SRC_LIB_RESTORE_OB_COS_WRAPPER_HANDLE_H_ +#define SRC_LIBRARY_SRC_LIB_RESTORE_OB_COS_WRAPPER_HANDLE_H_ + +#include +#include +#include "lib/string/ob_string.h" +#include "lib/allocator/page_arena.h" +#include "lib/restore/ob_storage_info.h" +#include "ob_cos_wrapper.h" +#include "lib/restore/ob_i_storage.h" + +namespace oceanbase +{ +namespace common +{ + +// Allocator for creating cos handle +class ObCosMemAllocator +{ +public: + ObCosMemAllocator(); + virtual ~ObCosMemAllocator(); + void *alloc(size_t size); + void free(void *addr); + void reuse(); +private: + ObArenaAllocator allocator_; +}; + +class ObCosWrapperHandle +{ +public: + ObCosWrapperHandle(); + virtual ~ObCosWrapperHandle() {}; + + int init(const common::ObObjectStorageInfo *storage_info); + void reset(); + int create_cos_handle(); + void destroy_cos_handle(); + qcloud_cos::ObCosWrapper::Handle *get_ptr() { return handle_; } + + int build_bucket_and_object_name(const ObString &uri); + const ObString& get_bucket_name() const { return bucket_name_; } + const ObString& get_object_name() const { return object_name_; } + + bool is_valid() const { return is_inited_ && handle_ != nullptr; } + bool is_inited() const { return is_inited_; } + + int set_delete_mode(const char *parameter); + int64_t get_delete_mode() const { return delete_mode_; } + +private: + bool is_inited_; + qcloud_cos::ObCosWrapper::Handle *handle_; + qcloud_cos::ObCosAccount cos_account_; + ObCosMemAllocator allocator_; + ObString bucket_name_; + ObString object_name_; + int64_t delete_mode_; +}; + + +} // common +} // oceanbase +#endif \ No newline at end of file diff --git a/deps/oblib/src/lib/restore/cos/ob_singleton.h b/deps/oblib/src/lib/restore/cos/ob_singleton.h new file mode 100644 index 000000000..e2caa745c --- /dev/null +++ b/deps/oblib/src/lib/restore/cos/ob_singleton.h @@ -0,0 +1,50 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef SRC_LIBRARY_SRC_LIB_OB_SINGLETON_H_ +#define SRC_LIBRARY_SRC_LIB_OB_SINGLETON_H_ + +namespace oceanbase +{ +namespace common +{ +namespace qcloud_cos +{ + +// A singleton base class offering an easy way to create singleton. +template +class __attribute__ ((visibility ("default"))) ObSingleton +{ +public: + // not thread safe + static T& get_instance() + { + static T instance; + return instance; + } + + virtual ~ObSingleton() {} + +protected: + ObSingleton() {} + +private: + ObSingleton(const ObSingleton &); + ObSingleton& operator=(const ObSingleton&); +}; + + +} +} +} + +#endif \ No newline at end of file diff --git a/deps/oblib/src/lib/restore/ob_i_storage.h b/deps/oblib/src/lib/restore/ob_i_storage.h index 7ffef936d..b6f493370 100644 --- a/deps/oblib/src/lib/restore/ob_i_storage.h +++ b/deps/oblib/src/lib/restore/ob_i_storage.h @@ -17,6 +17,7 @@ #include "lib/string/ob_string.h" #include "lib/container/ob_array.h" #include "common/storage/ob_device_common.h" +#include "ob_storage_info.h" namespace oceanbase { @@ -39,7 +40,7 @@ public: TAGGING = 2, MAX }; - virtual int open(void* base_info) = 0; + virtual int open(common::ObObjectStorageInfo *storage_info) = 0; virtual void close() = 0; virtual int is_exist(const common::ObString &uri, bool &exist) = 0; virtual int get_file_length(const common::ObString &uri, int64_t &file_length) = 0; @@ -48,7 +49,6 @@ public: virtual int mkdir(const common::ObString &uri) = 0; virtual int list_files(const common::ObString &dir_path, common::ObBaseDirEntryOperator &op) = 0; virtual int del_dir(const common::ObString &uri) = 0; - virtual int check_backup_dest_lifecycle(const common::ObString &path, bool &is_set_lifecycle) = 0; virtual int list_directories(const common::ObString &dir_path, common::ObBaseDirEntryOperator &op) = 0; virtual int is_tagging(const common::ObString &uri, bool &is_tagging) = 0; }; @@ -56,7 +56,7 @@ public: class ObIStorageReader { public: - virtual int open(const common::ObString &uri, void* handle) = 0; + virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info) = 0; virtual int pread(char *buf,const int64_t buf_size, int64_t offset, int64_t &read_size) = 0; virtual int close() = 0; virtual int64_t get_length() const = 0; @@ -66,7 +66,7 @@ public: class ObIStorageWriter { public: - virtual int open(const common::ObString &uri, void* handle) = 0; + virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info) = 0; virtual int write(const char *buf,const int64_t size) = 0; virtual int pwrite(const char *buf, const int64_t size, const int64_t offset) = 0; virtual int close() = 0; diff --git a/deps/oblib/src/lib/restore/ob_object_device.cpp b/deps/oblib/src/lib/restore/ob_object_device.cpp index 6a10d0fd1..6493e2653 100644 --- a/deps/oblib/src/lib/restore/ob_object_device.cpp +++ b/deps/oblib/src/lib/restore/ob_object_device.cpp @@ -19,7 +19,8 @@ namespace common const char *OB_STORAGE_ACCESS_TYPES_STR[] = {"reader", "overwriter", "appender", "random_write"}; -ObObjectDevice::ObObjectDevice() : oss_account_(), base_info_(NULL), is_started_(false), lock_(common::ObLatchIds::OBJECT_DEVICE_LOCK) +ObObjectDevice::ObObjectDevice() + : storage_info_(), is_started_(false), lock_(common::ObLatchIds::OBJECT_DEVICE_LOCK) { auto attr = SET_USE_500("ObjectDevice"); reader_ctx_pool_.set_attr(attr); @@ -51,7 +52,7 @@ void ObObjectDevice::destroy() ObObjectDevice::~ObObjectDevice() { destroy(); - OB_LOG(INFO, "destory the device!", KCSTRING(storage_info_)); + OB_LOG(INFO, "destory the device!", KCSTRING(storage_info_str_)); } /*the app logical use call ObBackupIoAdapter::get_and_init_device*/ @@ -70,29 +71,19 @@ int ObObjectDevice::start(const ObIODOpts &opts) ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "fail to start device, args wrong !", KCSTRING(opts.opts_[0].key_), K(ret)); } else { - ObString storage_info(opts.opts_[0].value_.value_str); - if (OB_STORAGE_OSS == device_type_) { - if (OB_FAIL(oss_account_.parse_oss_arg(storage_info))) { - OB_LOG(WARN, "fail to init oss base", K(storage_info) ,K(ret)); - } - base_info_ = (void*)&oss_account_; - } else if (OB_STORAGE_FILE == device_type_) { - //do nothing - base_info_ = NULL; - } else { - ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "Invalid object device type!", K(ret), K(device_type_), KCSTRING(opts.opts_[0].value_.value_str)); + if (OB_FAIL(storage_info_.set(device_type_, opts.opts_[0].value_.value_str))) { + OB_LOG(WARN, "failed to build storage_info"); } if (OB_SUCCESS != ret) { //mem resource will be free with device destroy - } else if (OB_FAIL(util_.open(base_info_, device_type_))) { + } else if (OB_FAIL(util_.open(&storage_info_))) { OB_LOG(WARN, "fail to open the util!", K(ret), KCSTRING(opts.opts_[0].value_.value_str)); } else if (OB_FAIL(fd_mng_.init())) { OB_LOG(WARN, "fail to init fd manager!", K(ret)); } else { is_started_ = true; - if (OB_FAIL(databuff_printf(storage_info_, OB_MAX_URI_LENGTH, "%s", opts.opts_[0].value_.value_str))) { + if (OB_FAIL(databuff_printf(storage_info_str_, OB_MAX_URI_LENGTH, "%s", opts.opts_[0].value_.value_str))) { OB_LOG(WARN, "fail to copy str to storage info", K(ret)); } } @@ -161,7 +152,7 @@ int ObObjectDevice::open_for_reader(const char *pathname, void*& ctx) ret = OB_ALLOCATE_MEMORY_FAILED; OB_LOG(WARN, "fail to alloc mem for object device reader! ", K(ret)); } else { - if (OB_FAIL(reader->open(pathname, base_info_))) { + if (OB_FAIL(reader->open(pathname, &storage_info_))) { OB_LOG(WARN, "fail to open for read!", K(ret)); } else { ctx = (void*)reader; @@ -184,7 +175,7 @@ int ObObjectDevice::open_for_overwriter(const char *pathname, void*& ctx) ret = OB_ALLOCATE_MEMORY_FAILED; OB_LOG(WARN, "fail to alloc mem for object device reader! ", K(ret)); } else { - if (OB_FAIL(overwriter->open(pathname, base_info_))) { + if (OB_FAIL(overwriter->open(pathname, &storage_info_))) { OB_LOG(WARN, "fail to open for overwrite!", K(ret)); } else { ctx = (void*)overwriter; @@ -202,16 +193,10 @@ int ObObjectDevice::open_for_appender(const char *pathname, ObIODOpts *opts, voi int ret = OB_SUCCESS; ObStorageAppender *appender = NULL; ObOptValue opt_value; - const char* append_strategy = NULL; - int64_t append_version= -1; const char* open_mode = NULL; - get_opt_value(opts, "AppendStrategy", append_strategy); - get_opt_value(opts, "AppendVersion", append_version); get_opt_value(opts, "OpenMode", open_mode); - ObStorageAppender::AppenderParam param; StorageOpenMode mode = StorageOpenMode::CREATE_OPEN_LOCK; - ObAppendStrategy strategy = ObAppendStrategy::OB_APPEND_USE_OVERRITE; appender = appender_ctx_pool_.alloc(); if (OB_ISNULL(appender)) { @@ -232,29 +217,9 @@ int ObObjectDevice::open_for_appender(const char *pathname, ObIODOpts *opts, voi OB_LOG(WARN, "Invalid open mode!", KCSTRING(open_mode), K(ret)); } - if (NULL == append_strategy || 0 == STRCMP(append_strategy, "OB_APPEND_USE_OVERRITE")) { - //just keep the default value - } else if (0 == STRCMP(append_strategy, "OB_APPEND_USE_SLICE_PUT")) { - strategy = ObAppendStrategy::OB_APPEND_USE_SLICE_PUT; - } else { - ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "Invalid append strategy!", KCSTRING(append_strategy), K(ret)); - } - param.strategy_ = strategy; - param.version_param_.version_ = append_version; - if (-1 != append_version) { - param.version_param_.open_object_version_ = true; - if (param.version_param_.version_ <= 0) { - ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "Invalid strategy version!", K(param.version_param_.version_), K(ret)); - } - } else { - param.version_param_.open_object_version_ = false; - } - if (OB_SUCCESS == ret) { appender->set_open_mode(mode); - if (OB_FAIL(appender->open(pathname, base_info_, param))){ + if (OB_FAIL(appender->open(pathname, &storage_info_))){ OB_LOG(WARN, "fail to open the appender!", K(ret)); } else { ctx = appender; @@ -558,43 +523,11 @@ int ObObjectDevice::pwrite(const ObIOFd &fd, const int64_t offset, const int64_t return ret; } -int ObObjectDevice::ob_get_bucket_lifecycle(ObIODOpts &opts) -{ - int ret = OB_SUCCESS; - const char* path = NULL; - bool is_set_lifecycle = false; - if (opts.opt_cnt_ != 2) { - ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "ObGetBucketLifecycle need 3 element at least!", K(opts.opt_cnt_), K(ret)); - } else if (0 != STRCMP(opts.opts_[1].key_, "path") || NULL == opts.opts_[1].value_.value_str) { - ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "ObGetBucketLifecycle, the first para is wrong!", KCSTRING(opts.opts_[1].key_), K(ret)); - } else { - path = opts.opts_[1].value_.value_str; - if (OB_FAIL(util_.check_backup_dest_lifecycle(ObString(path), is_set_lifecycle))) { - OB_LOG(WARN, "ObGetBucketLifecycle, the second para is wrong!", K(ret)); - } else { - opts.opts_[0].set("get_bucket_lifecycle", is_set_lifecycle); - } - } - return ret; -} - //the first opt is config name & the return value int ObObjectDevice::get_config(ObIODOpts &opts) { int ret = OB_SUCCESS; - if (opts.opt_cnt_ < 1) { - ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "opts need 1 element at least!", K(opts.opt_cnt_), K(ret)); - } else if (0 == STRCMP(opts.opts_[0].key_, "get_bucket_lifecycle")) { - if (OB_FAIL(ob_get_bucket_lifecycle(opts))) { - OB_LOG(WARN, "fail to get bucket life cycle!", K(ret)); - } - } else { - ret = OB_NOT_SUPPORTED; - OB_LOG(WARN, "the para is wrong, this interface only support get_bucket_lifecycle now!", K(device_type_), K(ret)); - } + UNUSED(opts); return ret; } diff --git a/deps/oblib/src/lib/restore/ob_object_device.h b/deps/oblib/src/lib/restore/ob_object_device.h index 13b83b599..01d7c19d1 100644 --- a/deps/oblib/src/lib/restore/ob_object_device.h +++ b/deps/oblib/src/lib/restore/ob_object_device.h @@ -82,20 +82,18 @@ private: int open_for_overwriter(const char *pathname, void*& ctx); int open_for_appender(const char *pathname, ObIODOpts *opts, void*& ctx); int release_res(void* ctx, const ObIOFd &fd, ObStorageAccessType access_type); - int ob_get_bucket_lifecycle(ObIODOpts &opts); private: //maybe fd mng can be device level common::ObFdSimulator fd_mng_; - ObOssAccount oss_account_; ObStorageUtil util_; /*obj ctx pool: use to create fd ctx(reader/writer)*/ common::ObPooledAllocator reader_ctx_pool_; common::ObPooledAllocator appender_ctx_pool_; common::ObPooledAllocator overwriter_ctx_pool_; - void* base_info_; + common::ObObjectStorageInfo storage_info_; bool is_started_; - char storage_info_[OB_MAX_URI_LENGTH]; + char storage_info_str_[OB_MAX_URI_LENGTH]; common::ObSpinLock lock_; private: diff --git a/deps/oblib/src/lib/restore/ob_storage.cpp b/deps/oblib/src/lib/restore/ob_storage.cpp index 54512642b..382b5b010 100644 --- a/deps/oblib/src/lib/restore/ob_storage.cpp +++ b/deps/oblib/src/lib/restore/ob_storage.cpp @@ -56,6 +56,7 @@ int validate_uri_type(const common::ObString &uri) { int ret = OB_SUCCESS; if (!uri.prefix_match(OB_OSS_PREFIX) && + !uri.prefix_match(OB_COS_PREFIX) && !uri.prefix_match(OB_FILE_PREFIX)) { ret = OB_INVALID_BACKUP_DEST; STORAGE_LOG(ERROR, "invalid backup uri", K(ret), K(uri)); @@ -70,6 +71,8 @@ int get_storage_type_from_path(const common::ObString &uri, ObStorageType &type) if (uri.prefix_match(OB_OSS_PREFIX)) { type = OB_STORAGE_OSS; + } else if (uri.prefix_match(OB_COS_PREFIX)) { + type = OB_STORAGE_COS; } else if (uri.prefix_match(OB_FILE_PREFIX)) { type = OB_STORAGE_FILE; } else { @@ -91,7 +94,7 @@ const char *get_storage_type_str(const ObStorageType &type) bool is_io_error(const int result) { - return OB_IO_ERROR == result || OB_OSS_ERROR == result; + return OB_IO_ERROR == result || OB_OSS_ERROR == result || OB_COS_ERROR == result; } int get_storage_type_from_name(const char *type_str, ObStorageType &type) @@ -187,21 +190,30 @@ ObExternalIOCounterGuard::~ObExternalIOCounterGuard() ObStorageUtil::ObStorageUtil() : file_util_(), oss_util_(), + cos_util_(), util_(NULL), - obj_base_info_(NULL), + storage_info_(NULL), init_state(false) { } /*this fun just like the get_util*/ -int ObStorageUtil::open(void* obj_base, int device_type) +int ObStorageUtil::open(common::ObObjectStorageInfo *storage_info) { int ret = OB_SUCCESS; + ObStorageType device_type = OB_STORAGE_MAX_TYPE; + if (is_init()) { ret = OB_INIT_TWICE; - STORAGE_LOG(WARN, "double init the storage util", K(ret), K(device_type)); + STORAGE_LOG(WARN, "double init the storage util", K(ret)); + } else if (OB_ISNULL(storage_info) || OB_UNLIKELY(!storage_info->is_valid())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", K(ret), KP(storage_info)); + } else if (OB_FALSE_IT(device_type = storage_info->get_type())) { } else if (OB_STORAGE_OSS == device_type) { util_ = &oss_util_; + } else if (OB_STORAGE_COS == device_type) { + util_ = &cos_util_; } else if (OB_STORAGE_FILE == device_type) { util_ = &file_util_; } else { @@ -210,11 +222,11 @@ int ObStorageUtil::open(void* obj_base, int device_type) } if (OB_SUCC(ret) && NULL != util_) { - if (OB_FAIL(util_->open(obj_base))) { + if (OB_FAIL(util_->open(storage_info))) { STORAGE_LOG(WARN, "failed to open util", K(ret), K(device_type)); util_ = NULL; } else { - obj_base_info_ = obj_base; + storage_info_ = storage_info; init_state = true; } } @@ -226,7 +238,7 @@ void ObStorageUtil::close() if (NULL != util_) { util_->close(); util_ = NULL; - obj_base_info_ = NULL; + storage_info_ = NULL; init_state = false; } } @@ -462,37 +474,6 @@ int ObStorageUtil::del_dir(const common::ObString &uri) return ret; } -int ObStorageUtil::check_backup_dest_lifecycle( - const common::ObString &path, - bool &is_set_lifecycle) -{ - int ret = OB_NOT_SUPPORTED; - UNUSED(path); - UNUSED(is_set_lifecycle); - // Forbidden check backup dest lifecycle, because oss has bug which make observer core - /* - const int64_t start_ts = ObTimeUtility::current_time(); - ObIStorageUtil *util = NULL; - is_set_lifecycle = false; - - if (OB_FAIL(ret)) { - } else if (!is_init()) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "util is not inited", K(ret), K(path)); - } else if (path.empty()) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "delete tmp files get invalid argument", K(ret), K(path)); - } else if (ObStorageGlobalIns::get_instance().is_io_prohibited()) { - ret = OB_BACKUP_IO_PROHIBITED; - STORAGE_LOG(WARN, "current observer backup io is prohibited", K(ret), K(path)); - } else if (OB_FAIL(util_->check_backup_dest_lifecycle(path, is_set_lifecycle))) { - STORAGE_LOG(WARN, "failed to delete tmp files", K(ret), K(path)); - } - print_access_storage_log("check_bucket_lifecycle", path, start_ts, 0); - */ - return ret; -} - int ObStorageUtil::list_directories(const common::ObString &uri, common::ObBaseDirEntryOperator &op) { int ret = OB_SUCCESS; @@ -540,6 +521,7 @@ ObStorageReader::ObStorageReader() reader_(NULL), file_reader_(), oss_reader_(), + cos_reader_(), start_ts_(0) { uri_[0] = '\0'; @@ -552,7 +534,7 @@ ObStorageReader::~ObStorageReader() } } -int ObStorageReader::open(const common::ObString &uri, void* obj_base_info) +int ObStorageReader::open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -575,6 +557,8 @@ int ObStorageReader::open(const common::ObString &uri, void* obj_base_info) STORAGE_LOG(WARN, "failed to get type", K(ret), K(uri)); } else if (OB_STORAGE_OSS == type) { reader_ = &oss_reader_; + } else if (OB_STORAGE_COS == type) { + reader_ = &cos_reader_; } else if (OB_STORAGE_FILE == type) { reader_ = &file_reader_; } else { @@ -586,7 +570,7 @@ int ObStorageReader::open(const common::ObString &uri, void* obj_base_info) if (OB_ISNULL(reader_)) { ret = OB_ERR_UNEXPECTED; STORAGE_LOG(WARN, "reader_ is null", K(ret), K(uri)); - } else if (OB_FAIL(reader_->open(uri, obj_base_info))) { + } else if (OB_FAIL(reader_->open(uri, storage_info))) { STORAGE_LOG(WARN, "failed to open reader", K(ret), K(uri)); } else { file_length_ = reader_->get_length(); @@ -661,6 +645,7 @@ ObStorageWriter::ObStorageWriter() : writer_(NULL), file_writer_(), oss_writer_(), + cos_writer_(), start_ts_(0) { uri_[0] = '\0'; @@ -673,7 +658,7 @@ ObStorageWriter::~ObStorageWriter() } } -int ObStorageWriter::open(const common::ObString &uri, void* obj_base_info) +int ObStorageWriter::open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -696,6 +681,8 @@ int ObStorageWriter::open(const common::ObString &uri, void* obj_base_info) STORAGE_LOG(WARN, "failed to get type", K(ret), K(uri)); } else if (OB_STORAGE_OSS == type) { writer_ = &oss_writer_; + } else if (OB_STORAGE_COS == type) { + writer_ = &cos_writer_; } else if (OB_STORAGE_FILE == type) { writer_ = &file_writer_; } else { @@ -707,7 +694,7 @@ int ObStorageWriter::open(const common::ObString &uri, void* obj_base_info) if (OB_ISNULL(writer_)) { ret = OB_ERR_UNEXPECTED; STORAGE_LOG(WARN, "writer_ is null", K(ret), K(uri)); - } else if (OB_FAIL(writer_->open(uri, obj_base_info))) { + } else if (OB_FAIL(writer_->open(uri, storage_info))) { STORAGE_LOG(WARN, "failed to open writer", K(ret), K(uri)); } } @@ -774,9 +761,10 @@ ObStorageAppender::ObStorageAppender() : appender_(NULL), file_appender_(), oss_appender_(), + cos_appender_(), start_ts_(0), is_opened_(false), - storage_info_(NULL) + storage_info_() { uri_[0] = '\0'; } @@ -785,9 +773,10 @@ ObStorageAppender::ObStorageAppender(StorageOpenMode mode) : appender_(NULL), file_appender_(mode), oss_appender_(), + cos_appender_(), start_ts_(0), is_opened_(false), - storage_info_(NULL) + storage_info_() { uri_[0] = '\0'; } @@ -801,8 +790,7 @@ ObStorageAppender::~ObStorageAppender() int ObStorageAppender::open( const common::ObString &uri, - void* obj_base_info, - const AppenderParam ¶m) + common::ObObjectStorageInfo *storage_info) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -819,17 +807,20 @@ int ObStorageAppender::open( } else if (NULL != appender_) { ret = OB_INIT_TWICE; STORAGE_LOG(WARN, "cannot open twice", K(ret), K(uri)); + } else if (OB_UNLIKELY(uri.empty()) || OB_ISNULL(storage_info)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", K(ret), K(uri), KP(storage_info)); } else if (OB_FAIL(databuff_printf(uri_, sizeof(uri_), "%.*s", uri.length(), uri.ptr()))) { STORAGE_LOG(WARN, "failed to fill uri", K(ret), K(uri)); } else if (OB_FAIL(get_storage_type_from_path(uri, type))) { STORAGE_LOG(WARN, "failed to get type", K(ret), K(uri)); - } else if (OB_STORAGE_OSS == type) { - appender_ = &oss_appender_; - if (OB_ISNULL(storage_info_ = allocator_.alloc(sizeof(ObOssAccount)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - STORAGE_LOG(WARN, "failed to alloc memory", K(ret), K(sizeof(ObOssAccount))); - } else { - MEMCPY(storage_info_, obj_base_info, sizeof(ObOssAccount)); + } else if (OB_STORAGE_OSS == type || OB_STORAGE_COS == type) { + if (OB_FAIL(storage_info_.assign(*storage_info))) { + STORAGE_LOG(WARN, "failed to copy storage info", K(ret)); + } else if (OB_STORAGE_OSS == type) { + appender_ = &oss_appender_; + } else if (OB_STORAGE_COS == type) { + appender_ = &cos_appender_; } } else if (OB_STORAGE_FILE == type) { appender_ = &file_appender_; @@ -842,7 +833,7 @@ int ObStorageAppender::open( if (OB_ISNULL(appender_)) { ret = OB_ERR_UNEXPECTED; STORAGE_LOG(WARN, "appender_ is null", K(ret), K(uri)); - } else if (OB_FAIL(appender_->open(uri, obj_base_info))) { + } else if (OB_FAIL(appender_->open(uri, storage_info))) { STORAGE_LOG(WARN, "failed to open writer", K(ret), K(uri)); } else { is_opened_ = true; @@ -900,7 +891,7 @@ int ObStorageAppender::repeatable_pwrite_(const char *buf, const int64_t size, c if (OB_ISNULL(appender_)) { ret = OB_NOT_INIT; STORAGE_LOG(WARN, "not opened", K(ret)); - } else if (OB_FAIL(reader.open(uri_, storage_info_))) { + } else if (OB_FAIL(reader.open(uri_, &storage_info_))) { STORAGE_LOG(WARN, "failed to open reader", K(ret)); } else if (reader.get_length() <= offset) { // This situation also has concurrency issues. @@ -982,7 +973,7 @@ int ObStorageAppender::close() print_access_storage_log("storage appender_", uri_, start_ts_, appender_->get_length()); } - if (OB_ISNULL(appender_) || !is_opened_) { + if (OB_ISNULL(appender_)) { ret = OB_NOT_INIT; STORAGE_LOG(WARN, "not opened", K(ret)); } else if (OB_FAIL(appender_->close())) { diff --git a/deps/oblib/src/lib/restore/ob_storage.h b/deps/oblib/src/lib/restore/ob_storage.h index c749bf422..d35493d74 100644 --- a/deps/oblib/src/lib/restore/ob_storage.h +++ b/deps/oblib/src/lib/restore/ob_storage.h @@ -15,6 +15,7 @@ #include "ob_i_storage.h" #include "ob_storage_file.h" #include "ob_storage_oss_base.h" +#include "ob_storage_cos_base.h" namespace oceanbase { @@ -112,7 +113,7 @@ public: // When physical backup lease is timeout, retry won't stop until 300s. explicit ObStorageUtil(); virtual ~ObStorageUtil() {} - int open(void* obj_base, int device_type); + int open(common::ObObjectStorageInfo *storage_info); void close(); int is_exist(const common::ObString &uri, bool &exist); int get_file_length(const common::ObString &uri, int64_t &file_length); @@ -121,18 +122,18 @@ public: int write_single_file(const common::ObString &uri, const char *buf, const int64_t size); int list_files(const common::ObString &dir_path, common::ObBaseDirEntryOperator &op); int del_dir(const common::ObString &uri); - int check_backup_dest_lifecycle(const common::ObString &path, bool &is_set_lifecycle); int list_directories(const common::ObString &dir_path, common::ObBaseDirEntryOperator &op); int is_tagging(const common::ObString &uri, bool &is_tagging); private: -//we does not use obj_base_info_ to judge init, since for nfs&local, obj_base_info_ is null + // we does not use storage_info_ to judge init, since for nfs&local, storage_info_ is null bool is_init() {return init_state;} ObStorageFileUtil file_util_; ObStorageOssUtil oss_util_; + ObStorageCosUtil cos_util_; ObIStorageUtil* util_; - void* obj_base_info_; + common::ObObjectStorageInfo* storage_info_; bool init_state; DISALLOW_COPY_AND_ASSIGN(ObStorageUtil); }; @@ -142,7 +143,7 @@ class ObStorageReader public: ObStorageReader(); virtual ~ObStorageReader(); - int open(const common::ObString &uri, void* obj_base_info); + int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info); int pread(char *buf,const int64_t buf_size, int64_t offset, int64_t &read_size); int close(); int64_t get_length() const { return file_length_; } @@ -151,6 +152,7 @@ private: ObIStorageReader *reader_; ObStorageFileReader file_reader_; ObStorageOssReader oss_reader_; + ObStorageCosReader cos_reader_; int64_t start_ts_; char uri_[OB_MAX_URI_LENGTH]; DISALLOW_COPY_AND_ASSIGN(ObStorageReader); @@ -161,13 +163,14 @@ class ObStorageWriter public: ObStorageWriter(); virtual ~ObStorageWriter(); - int open(const common::ObString &uri, void* obj_base_info); + int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info); int write(const char *buf,const int64_t size); int close(); private: ObIStorageWriter *writer_; ObStorageFileWriter file_writer_; ObStorageOssWriter oss_writer_; + ObStorageCosWriter cos_writer_; int64_t start_ts_; char uri_[OB_MAX_URI_LENGTH]; DISALLOW_COPY_AND_ASSIGN(ObStorageWriter); @@ -186,7 +189,7 @@ public: ObStorageObjectVersionParam version_param_; }; - int open(const common::ObString &uri, void* obj_base_info, const AppenderParam ¶m); + int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info); int write(const char *buf,const int64_t size); int pwrite(const char *buf, const int64_t size, const int64_t offset); int close(); @@ -198,10 +201,11 @@ private: ObIStorageWriter *appender_; ObStorageFileAppender file_appender_; ObStorageOssAppendWriter oss_appender_; + ObStorageCosAppendWriter cos_appender_; int64_t start_ts_; bool is_opened_; char uri_[OB_MAX_URI_LENGTH]; - void *storage_info_; + common::ObObjectStorageInfo storage_info_; ObArenaAllocator allocator_; int repeatable_pwrite_(const char *buf, const int64_t size, const int64_t offset); diff --git a/deps/oblib/src/lib/restore/ob_storage_cos_base.cpp b/deps/oblib/src/lib/restore/ob_storage_cos_base.cpp new file mode 100644 index 000000000..3ba1c2d0a --- /dev/null +++ b/deps/oblib/src/lib/restore/ob_storage_cos_base.cpp @@ -0,0 +1,934 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "lib/restore/ob_storage_cos_base.h" +#include +#include +#include "common/ob_string_buf.h" +#include "apr_errno.h" +#include "ob_storage.h" +#include "lib/hash/ob_hashset.h" +#include "lib/utility/ob_tracepoint.h" + +namespace oceanbase +{ +namespace common +{ +using namespace oceanbase::common; + +/*--------------------------------GLOBAL---------------------------*/ +int init_cos_env() +{ + return qcloud_cos::ObCosEnv::get_instance().init(); +} + +void fin_cos_env() +{ + // wait doing io finish before destroy cos env. + const int64_t start_time = ObTimeUtility::current_time(); + const int64_t timeout = ObExternalIOCounter::FLYING_IO_WAIT_TIMEOUT; + int64_t flying_io_cnt = ObExternalIOCounter::get_flying_io_cnt(); + while(0 < flying_io_cnt) { + const int64_t end_time = ObTimeUtility::current_time(); + if (end_time - start_time > timeout) { + OB_LOG(INFO, "force fin_cos_env", K(flying_io_cnt)); + break; + } + usleep(100 * 1000L); // 100ms + flying_io_cnt = ObExternalIOCounter::get_flying_io_cnt(); + } + + qcloud_cos::ObCosEnv::get_instance().destroy(); +} + +struct CosListFilesCbArg +{ + common::ObIAllocator &allocator; + ObString &dir_path; + ObBaseDirEntryOperator &list_op; + + CosListFilesCbArg( + common::ObIAllocator &alloc, + ObString &dir, + ObBaseDirEntryOperator &op) + : allocator(alloc), + dir_path(dir), + list_op(op) {} + + ~CosListFilesCbArg() {} +}; + +static int execute_list_callback( + ObBaseDirEntryOperator &op, + qcloud_cos::ObCosWrapper::CosListObjPara ¶, + ObString &file_name) +{ + int ret = OB_SUCCESS; + int64_t dname_size = sizeof(para.last_container_name_.d_name); + if (dname_size >= (file_name.length() + 1)) { + if (OB_FAIL(databuff_printf(para.last_container_name_.d_name, dname_size, "%s", file_name.ptr()))) { + OB_LOG(WARN, "fail to copy file name to entry buf!", K(ret), K(dname_size)); + } else if (OB_FAIL(op.func(&(para.last_container_name_)))) { + OB_LOG(WARN, "fail to execute list callback!", K(ret), KCSTRING(para.last_container_name_.d_name), + K(para.last_container_name_.d_type), K(DT_REG), K(dname_size)); + } + } else { + ret = OB_BUF_NOT_ENOUGH; + OB_LOG(WARN, "file name len is too long!", K(ret), K(file_name.length()), K(sizeof(dirent))); + } + return ret; +} + +static int handle_object_name_cb(qcloud_cos::ObCosWrapper::CosListObjPara ¶) +{ + int ret = OB_SUCCESS; + para.next_flag_ = true; + if (OB_ISNULL(para.arg_)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "arg is empty", K(ret)); + } else if (OB_ISNULL(para.cur_full_path_slice_name_) || 0 >= para.full_path_size_) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "current object_name is empty", K(ret)); + } else if (qcloud_cos::ObCosWrapper::CosListObjPara::CosListType::COS_LIST_CB_ARG == para.type_) { + CosListFilesCbArg *ctx = static_cast(para.arg_); + + // Returned object name is the whole object path, but we donot need the prefix dir_path. + // So, we trim the dir_name from object_name path. + const int dir_name_str_len = strlen(ctx->dir_path.ptr()); + const int object_name_len = para.full_path_size_ - dir_name_str_len; + const int object_name_start_pos = dir_name_str_len; + //tmp_file_name like slice name + ObString tmp_file_name(object_name_len, object_name_len, + para.cur_full_path_slice_name_ + object_name_start_pos); + if (OB_FAIL(execute_list_callback(ctx->list_op, para, tmp_file_name))) { + OB_LOG(WARN, "fail to execute list callback!", K(ret)); + } + } else { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "not supported type", K(ret), K(para.type_)); + } + + return ret; +} + +static int handle_directory_name_cb( + void *arg, + const qcloud_cos::ObCosWrapper::CosListObjPara::CosListType type, + const char *object_name, + int64_t object_size) +{ + int ret = OB_SUCCESS; + const char delimiter = '/'; + if (OB_ISNULL(arg)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "arg is empty", K(ret), KP(arg)); + } else if (OB_ISNULL(object_name) || 0 >= object_size) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "object_name is empty", K(ret), KP(object_name), K(object_size)); + } else if (delimiter != object_name[object_size - 1]) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "object name is unexpected", K(ret), KCSTRING(object_name), K(object_size)); + } else { + const int64_t name_length = object_size - 1; + if (name_length <= 0) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "object is not exist", K(ret), KCSTRING(object_name), K(object_size)); + } else if (qcloud_cos::ObCosWrapper::CosListObjPara::CosListType::COS_LIST_CB_ARG == type) { + CosListFilesCbArg *ctx = static_cast(arg); + ObString directory_name; + dirent dir_name_entry; + dir_name_entry.d_type = DT_DIR; + ObString tmp_directory_name(name_length, name_length, object_name); + if (OB_FAIL(databuff_printf(dir_name_entry.d_name, sizeof(dir_name_entry.d_name), "%.*s", + tmp_directory_name.length(), tmp_directory_name.ptr()))) { + OB_LOG(WARN, "fail to set dir name entry", K(ret), K(tmp_directory_name)); + } else if (OB_FAIL(ctx->list_op.func(&dir_name_entry))) { + OB_LOG(WARN, "fail to call list directories operation callback", K(ret), K(tmp_directory_name)); + } + } else { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "not supported type", K(ret), K(type)); + } + } + return ret; +} + +/*--------------------------------ObStorageCosUtil---------------------------*/ + +ObStorageCosUtil::ObStorageCosUtil() + : is_opened_(false), storage_info_(NULL) +{ +} + +ObStorageCosUtil::~ObStorageCosUtil() +{ +} + +int ObStorageCosUtil::open(ObObjectStorageInfo *storage_info) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(is_opened_)) { + ret = OB_COS_ERROR; + OB_LOG(WARN, "cos util already open, cannot open again", K(ret)); + } else if (OB_ISNULL(storage_info)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "storage info is null", K(ret)); + } else { + storage_info_ = storage_info; + is_opened_ = true; + } + return ret; +} + +void ObStorageCosUtil::close() +{ + is_opened_ = false; + storage_info_ = NULL; +} + +int ObStorageCosUtil::is_exist(const ObString &uri, bool &is_exist) +{ + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + is_exist = false; + int64_t file_length = 0; + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_COS_ERROR; + OB_LOG(WARN, "cos util not opened", K(ret)); + } else if (OB_UNLIKELY(uri.empty())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid argument", K(ret), K(uri)); + } else if (OB_FAIL(get_object_meta_(uri, is_exist, file_length))) { + OB_LOG(WARN, "fail to get object meta", K(ret), K(uri)); + } + return ret; +} + +int ObStorageCosUtil::get_file_length(const ObString &uri, int64_t &file_length) +{ + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + bool is_file_exist = false; + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_COS_ERROR; + OB_LOG(WARN, "cos util not opened", K(ret)); + } else if (OB_UNLIKELY(uri.empty())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid argument", K(ret), K(uri)); + } else if (OB_FAIL(get_object_meta_(uri, is_file_exist, file_length))) { + OB_LOG(WARN, "fail to get object meta", K(ret), K(uri)); + } else if (!is_file_exist) { + ret = OB_BACKUP_FILE_NOT_EXIST; + OB_LOG(WARN, "backup file is not exist", K(ret), K(uri)); + } + return ret; +} + +// inner function, won't check params valid or not. +int ObStorageCosUtil::get_object_meta_( + const ObString &uri, + bool &is_file_exist, + int64_t &file_length) +{ + int ret = OB_SUCCESS; + + ObStorageCosBase cos_base; + qcloud_cos::CosObjectMeta obj_meta; + is_file_exist = false; + if (OB_FAIL(cos_base.open(uri, storage_info_))) { + OB_LOG(WARN, "fail to open cos base", K(ret), K(uri)); + } else if (OB_FAIL(cos_base.get_cos_file_meta(is_file_exist, obj_meta))) { + OB_LOG(WARN, "fail to get object meta", K(ret)); + } else { + file_length = obj_meta.file_length_; + } + cos_base.reset(); + + return ret; +} + +int ObStorageCosUtil::write_single_file( + const ObString &uri, + const char *buf, + const int64_t size) +{ + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + ObStorageCosWriter writer; + if (OB_FAIL(writer.open(uri, storage_info_))) { + OB_LOG(WARN, "fail to open cos writer", K(ret), K(uri), KP_(storage_info)); + } else if (OB_FAIL(writer.write(buf, size))) { + OB_LOG(WARN, "fail to write into cos", K(ret), K(size), KP(buf)); + } else if (OB_FAIL(writer.close())) { + OB_LOG(WARN, "fail to close cos writer", K(ret)); + } + return ret; +} + +int ObStorageCosUtil::mkdir(const ObString &uri) +{ + int ret = OB_SUCCESS; + OB_LOG(DEBUG, "no need to create dir in cos", K(uri)); + UNUSED(uri); + return ret; +} + +int ObStorageCosUtil::del_dir(const ObString &uri) +{ + int ret = OB_SUCCESS; + OB_LOG(DEBUG, "no need to del dir in cos", K(uri)); + UNUSED(uri); + return ret; +} + +int ObStorageCosUtil::is_tagging( + const ObString &uri, + bool &is_tagging) +{ + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + is_tagging = false; + + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_COS_ERROR; + OB_LOG(WARN, "cos util not opened", K(ret)); + } else if (OB_UNLIKELY(uri.empty())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid argument", K(ret), K(uri)); + } else { + ObStorageCosBase cos_base; + if (OB_FAIL(cos_base.open(uri, storage_info_))) { + OB_LOG(WARN, "fail to open cos base", K(ret), K(uri)); + } else if (OB_FAIL(cos_base.is_object_tagging(uri, is_tagging))) { + OB_LOG(WARN, "fail to check object tag", K(ret), K(uri)); + } + cos_base.reset(); + } + return ret; +} + +int ObStorageCosUtil::del_file(const ObString &uri) +{ + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_COS_ERROR; + OB_LOG(WARN, "cos util not opened", K(ret)); + } else if (OB_UNLIKELY(uri.empty())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid argument", K(ret), K(uri)); + } else { + ObStorageCosBase cos_base; + if (OB_FAIL(cos_base.open(uri, storage_info_))) { + OB_LOG(WARN, "fail to open cos base", K(ret), K(uri)); + } else if (OB_FAIL(cos_base.delete_object(uri))) { + OB_LOG(WARN, "fail to get object meta", K(ret)); + } else { + OB_LOG(DEBUG, "succ to delete object", K(uri)); + } + cos_base.reset(); + } + return ret; +} + +int ObStorageCosUtil::list_files( + const ObString &uri, + ObBaseDirEntryOperator &op) +{ + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + ObArenaAllocator allocator; + + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_COS_ERROR; + OB_LOG(WARN, "cos util not opened", K(ret)); + } else if (OB_UNLIKELY(uri.empty())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid argument", K(ret), K(uri)); + } else { + char dir_path_array[OB_MAX_URI_LENGTH] = {0}; + const char *ptr = NULL; // point to actual dir path + const int min_dir_path_len = 2; // cuz dir_path end with '\0' + const char *separator = "/"; + const char end_marker = '\0'; + + ObStorageCosBase cos_base; + if (OB_FAIL(cos_base.open(uri, storage_info_))) { + OB_LOG(WARN, "fail to open cos base", K(ret), K(uri)); + } else { + const ObString &dir_path_str = cos_base.get_handle().get_object_name(); + const ObString::obstr_size_t dir_path_len = dir_path_str.length(); + int32_t actual_dir_path_len = dir_path_len; + if (dir_path_len < min_dir_path_len) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "dir_path_str length is unepxected", K(ret), K(dir_path_str), K(dir_path_len)); + } else { + // Let dir path end with '/' + if (dir_path_str.ptr()[dir_path_len - 1] != end_marker) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "dir_path_str not end with '\0'"); + } else if (dir_path_str.ptr()[dir_path_len - 2] == *separator) { + ptr = dir_path_str.ptr(); + } else if (OB_FAIL(databuff_printf(dir_path_array, OB_MAX_URI_LENGTH, "%s/", dir_path_str.ptr()))) { + OB_LOG(WARN, "fail to deep copy dir path", K(ret), K(dir_path_len), K(OB_MAX_URI_LENGTH)); + } else { + ptr = dir_path_array; + actual_dir_path_len += 1; + } + } + + if (OB_SUCC(ret)) { + ObString tmp_dir; + tmp_dir.assign(const_cast(ptr), actual_dir_path_len); + + // Construct list object callback arg + CosListFilesCbArg arg(allocator, tmp_dir, op); + if (OB_FAIL(cos_base.list_objects(uri, tmp_dir, separator, arg))) { + OB_LOG(WARN, "fail to list object in cos_base", K(ret), K(uri), K(tmp_dir)); + } + } + } + cos_base.reset(); + } + return ret; +} + +int ObStorageCosUtil::list_directories( + const ObString &uri, + ObBaseDirEntryOperator &op) +{ + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + ObArenaAllocator allocator; + + if (OB_UNLIKELY(!is_opened_)) { + ret = OB_COS_ERROR; + OB_LOG(WARN, "cos util not opened", K(ret)); + } else if (OB_UNLIKELY(uri.empty())) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid argument", K(ret), K(uri)); + } else { + char dir_path_array[OB_MAX_URI_LENGTH] = {0}; + const char *ptr = NULL; // point to actual dir path + const int min_dir_path_len = 2; // cuz dir_path end with '\0' + const char *delimiter_string = "/"; + const char *next_marker_string = ""; + const char end_marker = '\0'; + + ObStorageCosBase cos_base; + if (OB_FAIL(cos_base.open(uri, storage_info_))) { + OB_LOG(WARN, "fail to open cos base", K(ret), K(uri)); + } else { + const ObString &dir_path_str = cos_base.get_handle().get_object_name(); + const ObString::obstr_size_t dir_path_len = dir_path_str.length(); + int32_t actual_dir_path_len = dir_path_len; + if (dir_path_len < min_dir_path_len) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "dir_path_str length is unepxected", K(ret), K(dir_path_str), K(dir_path_len)); + } else { + // Let dir path end with '/' + if (dir_path_str.ptr()[dir_path_len - 1] != end_marker) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "dir_path_str not end with '\0'"); + } else if (dir_path_str.ptr()[dir_path_len - 2] == *delimiter_string) { + ptr = dir_path_str.ptr(); + } else if (OB_FAIL(databuff_printf(dir_path_array, OB_MAX_URI_LENGTH, "%s/", dir_path_str.ptr()))) { + OB_LOG(WARN, "fail to deep copy dir path", K(ret), K(dir_path_len), K(OB_MAX_URI_LENGTH)); + } else { + ptr = dir_path_array; + actual_dir_path_len += 1; + } + } + + if (OB_SUCC(ret)) { + ObString tmp_dir; + tmp_dir.assign(const_cast(ptr), actual_dir_path_len); + + CosListFilesCbArg arg(allocator, tmp_dir, op); + if (OB_FAIL(cos_base.list_directories(uri, tmp_dir, next_marker_string, delimiter_string, arg))) { + OB_LOG(WARN, "fail to list directories in cos_base", K(ret), K(uri), K(tmp_dir)); + } + } + } + cos_base.reset(); + } + return ret; +} + +/*--------------------------------ObStorageCosBase---------------------------*/ + +ObStorageCosBase::ObStorageCosBase() + : is_opened_(false), handle_() +{ +} + +ObStorageCosBase::~ObStorageCosBase() +{ + reset(); +} + +void ObStorageCosBase::reset() +{ + handle_.reset(); + is_opened_ = false; +} + +int ObStorageCosBase::init_handle(const ObObjectStorageInfo &storage_info) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(handle_.is_inited())) { + ret = OB_INIT_TWICE; + OB_LOG(WARN, "handle in cos base already inited", K(ret)); + } else if (OB_FAIL(handle_.init(&storage_info))) { + OB_LOG(WARN, "fail to init cos wrapper handle", K(ret)); + } + return ret; +} + +int ObStorageCosBase::open( + const ObString &uri, + ObObjectStorageInfo *storage_info) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(uri.empty()) || OB_ISNULL(storage_info)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "uri is empty", K(ret), K(uri), KP(storage_info)); + } else if (OB_FAIL(init_handle(*storage_info))) { + OB_LOG(WARN, "failed to init cos wrapper handle", K(ret), K(uri)); + } else if (OB_FAIL(handle_.create_cos_handle())) { + OB_LOG(WARN, "failed to create cos handle", K(ret), K(uri)); + } else if (OB_FAIL(handle_.build_bucket_and_object_name(uri))) { + OB_LOG(WARN, "failed to build bucket and object name", K(ret), K(uri)); + } + return ret; +} + +int ObStorageCosBase::get_cos_file_meta( + bool &is_file_exist, + qcloud_cos::CosObjectMeta &obj_meta) +{ + int ret = OB_SUCCESS; + if (OB_LIKELY(is_valid())) { + qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer( + handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length()); + qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer( + handle_.get_object_name().ptr(), handle_.get_object_name().length()); + + is_file_exist = false; + if (OB_FAIL(qcloud_cos::ObCosWrapper::head_object_meta(handle_.get_ptr(), bucket_name, + object_name, is_file_exist, obj_meta))) { + OB_LOG(WARN, "fail to get object meta", K(ret)); + } + } else { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "cos wrapper handle not init or create", K(ret)); + } + return ret; +} + +int ObStorageCosBase::delete_object(const ObString &uri) +{ + int ret = OB_SUCCESS; + if (OB_LIKELY(is_valid())) { + qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer( + handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length()); + qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer( + handle_.get_object_name().ptr(), handle_.get_object_name().length()); + + if (ObIStorageUtil::DELETE == handle_.get_delete_mode()) { + if (OB_FAIL(qcloud_cos::ObCosWrapper::del(handle_.get_ptr(), bucket_name, object_name))) { + OB_LOG(WARN, "fail to delete object meta", K(ret), K(uri)); + } + } else if (ObIStorageUtil::TAGGING == handle_.get_delete_mode()) { + if (OB_FAIL(qcloud_cos::ObCosWrapper::tag(handle_.get_ptr(), bucket_name, object_name))) { + OB_LOG(WARN, "fail to tag object", K(ret), K(uri)); + } + } else { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "cos delete mode invalid", K(ret), K(uri)); + } + } else { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "cos wrapper handle not init or create", K(ret)); + } + return ret; +} + +int ObStorageCosBase::list_objects( + const ObString &uri, + const ObString &dir_name_str, + const char *separator, + CosListFilesCbArg &arg) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(separator)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid argument", K(ret)); + } else if (OB_LIKELY(is_valid())) { + qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer( + handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length()); + qcloud_cos::CosStringBuffer dir_name(dir_name_str.ptr(), dir_name_str.length()); + qcloud_cos::CosStringBuffer next_marker(separator, strlen(separator) + 1); + if (OB_FAIL(qcloud_cos::ObCosWrapper::list_objects(handle_.get_ptr(), bucket_name, + dir_name, next_marker, handle_object_name_cb, (void *)(&arg)))) { + OB_LOG(WARN, "fail to list objects", K(ret), K(uri)); + } + } else { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "cos wrapper handle not init or create", K(ret)); + } + return ret; +} + +int ObStorageCosBase::list_directories( + const ObString &uri, + const ObString &dir_name_str, + const char *next_marker_str, + const char *delimiter_str, + CosListFilesCbArg &arg) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(next_marker_str) || OB_ISNULL(delimiter_str)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid argument", K(ret), KP(next_marker_str), KP(delimiter_str)); + } else if (OB_LIKELY(is_valid())) { + qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer( + handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length()); + qcloud_cos::CosStringBuffer dir_name(dir_name_str.ptr(), dir_name_str.length()); + qcloud_cos::CosStringBuffer next_marker(next_marker_str, strlen(next_marker_str) + 1); + qcloud_cos::CosStringBuffer delimiter(delimiter_str, strlen(delimiter_str) + 1); + if (OB_FAIL(qcloud_cos::ObCosWrapper::list_directories(handle_.get_ptr(), bucket_name, dir_name, + next_marker, delimiter, handle_directory_name_cb, (void *)(&arg)))) { + OB_LOG(WARN, "failed to list directories", K(ret), K(uri)); + } + } else { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "cos wrapper handle not init or create", K(ret)); + } + return ret; +} + +int ObStorageCosBase::is_object_tagging( + const ObString &uri, + bool &is_tagging) +{ + int ret = OB_SUCCESS; + if (OB_LIKELY(is_valid())) { + qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer( + handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length()); + qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer( + handle_.get_object_name().ptr(), handle_.get_object_name().length()); + if (OB_FAIL(qcloud_cos::ObCosWrapper::is_object_tagging(handle_.get_ptr(), + bucket_name, object_name, is_tagging))) { + OB_LOG(WARN, "fail to check object tagging", K(ret), K(uri)); + } + } else { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "cos wrapper handle not init or create", K(ret)); + } + return ret; +} + +/*--------------------------------ObStorageCosReader---------------------------*/ + +ObStorageCosReader::ObStorageCosReader() + : ObStorageCosBase(), file_length_(-1) +{ +} + +ObStorageCosReader::~ObStorageCosReader() +{ +} + +int ObStorageCosReader::open(const ObString &uri, ObObjectStorageInfo *storage_info) +{ + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + + if (OB_UNLIKELY(is_opened_)) { + ret = OB_COS_ERROR; + OB_LOG(WARN, "cos reader already open, cannot open again", K(ret), K(uri)); + } else if (OB_FAIL(ObStorageCosBase::open(uri, storage_info))) { + OB_LOG(WARN, "fail to open in cos_base", K(ret), K(uri)); + } else { + const ObString &bucket_str = handle_.get_bucket_name(); + const ObString &object_str = handle_.get_object_name(); + qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer( + bucket_str.ptr(), bucket_str.length()); + qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer( + object_str.ptr(), object_str.length()); + bool is_file_exist = false; + qcloud_cos::CosObjectMeta obj_meta; + if (OB_FAIL(qcloud_cos::ObCosWrapper::head_object_meta(handle_.get_ptr(), bucket_name, + object_name, is_file_exist, obj_meta))) { + OB_LOG(WARN, "fail to get object meta", K(ret), K(bucket_str), K(object_str)); + } else if (!is_file_exist) { + ret = OB_BACKUP_FILE_NOT_EXIST; + OB_LOG(WARN, "backup file is not exist", K(ret), K(bucket_str), K(object_str)); + } else { + file_length_ = obj_meta.file_length_; + is_opened_ = true; + } + } + return ret; +} + +int ObStorageCosReader::pread( + char *buf, + const int64_t buf_size, + int64_t offset, + int64_t &read_size) +{ + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + + if (!is_opened_) { + ret = OB_COS_ERROR; + OB_LOG(WARN, "cos reader cannot read before it is opened", K(ret)); + } else if (NULL == buf || buf_size <= 0 || offset < 0) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid argument", K(ret), KP(buf), K(buf_size), K(offset)); + } else { + qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer( + handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length()); + qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer( + handle_.get_object_name().ptr(), handle_.get_object_name().length()); + if (OB_FAIL(qcloud_cos::ObCosWrapper::pread(handle_.get_ptr(), bucket_name, + object_name, offset, buf, buf_size, read_size))) { + OB_LOG(WARN, "fail to read object from cos", K(ret), KP(buf), K(buf_size), K(offset)); + } + } + return ret; +} + +int ObStorageCosReader::close() +{ + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + file_length_ = -1; + reset(); + return ret; +} + +/*--------------------------------ObStorageCosWriter---------------------------*/ + +ObStorageCosWriter::ObStorageCosWriter() + : ObStorageCosBase(), file_length_(-1) +{ +} + +ObStorageCosWriter::~ObStorageCosWriter() +{ + if (is_opened_) { + close(); + } +} + +int ObStorageCosWriter::open( + const ObString &uri, + ObObjectStorageInfo *storage_info) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(is_opened_)) { + ret = OB_COS_ERROR; + OB_LOG(WARN, "cos writer already open, cannot open again", K(ret), K(uri)); + } else if (OB_FAIL(ObStorageCosBase::open(uri, storage_info))) { + OB_LOG(WARN, "fail to open in cos_base", K(ret), K(uri)); + } else { + file_length_ = 0; + is_opened_ = true; + } + return ret; +} + +int ObStorageCosWriter::close() +{ + int ret = OB_SUCCESS; + file_length_ = -1; + reset(); + return ret; +} + +int ObStorageCosWriter::pwrite(const char *buf, const int64_t size, const int64_t offset) +{ + int ret = OB_NOT_SUPPORTED; + UNUSED(buf); + UNUSED(size); + UNUSED(offset); + return ret; +} + +int ObStorageCosWriter::write(const char *buf, const int64_t size) +{ + int ret = OB_SUCCESS; + if (!is_opened_) { + ret = OB_NOT_INIT; + OB_LOG(WARN, "cos writer not opened", K(ret)); + } else if (NULL == buf || size < 0) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "buf is NULL or size is invalid", K(ret), KP(buf), K(size)); + } else { + qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer( + handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length()); + qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer( + handle_.get_object_name().ptr(), handle_.get_object_name().length()); + if (OB_FAIL(qcloud_cos::ObCosWrapper::put(handle_.get_ptr(), bucket_name, + object_name, buf, size))) { + OB_LOG(WARN, "fail to write object into cos", K(ret), KP(buf), K(size)); + } else { + file_length_ += size; + } + } + return ret; +} + +/*--------------------------------ObStorageCosAppendWriter---------------------------*/ + +ObStorageCosAppendWriter::ObStorageCosAppendWriter() + : ObStorageCosBase(), + file_length_(-1) +{ +} + +ObStorageCosAppendWriter::~ObStorageCosAppendWriter() +{ +} + +int ObStorageCosAppendWriter::open( + const ObString &uri, + ObObjectStorageInfo *storage_info) +{ + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + + if (OB_UNLIKELY(is_opened_)) { + ret = OB_COS_ERROR; + OB_LOG(WARN, "cos appender already open, cannot open again", K(ret), K(uri)); + } else if (OB_FAIL(ObStorageCosBase::open(uri, storage_info))) { + OB_LOG(WARN, "fail to open in cos_base", K(ret), K(uri)); + } else { + file_length_ = 0; + is_opened_ = true; + } + return ret; +} + +int ObStorageCosAppendWriter::write( + const char *buf, + const int64_t size) +{ + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + const int64_t fake_offset = 0; + const bool is_pwrite = false; + + if(OB_UNLIKELY(!is_opened_)) { + ret = OB_COS_ERROR; + OB_LOG(WARN, "cos append writer cannot write before it is not opened", K(ret)); + } else if(NULL == buf || size < 0) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "buf is NULL or size is invalid", KP(buf), K(size), K(ret)); + } else if (OB_FAIL(do_write(buf, size, fake_offset, is_pwrite))) { + OB_LOG(WARN, "failed to do write", K(ret), KP(buf), K(size)); + } + return ret; +} + +int ObStorageCosAppendWriter::pwrite(const char *buf, const int64_t size, const int64_t offset) +{ + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + const bool is_pwrite = true; + + if(OB_UNLIKELY(!is_opened_)) { + ret = OB_COS_ERROR; + OB_LOG(WARN, "cos append writer cannot write before it is not opened", K(ret)); + } else if(NULL == buf || size < 0 || offset < 0) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "buf is NULL or size is invalid", KP(buf), K(size), K(ret)); + } else if (OB_FAIL(do_write(buf, size, offset, is_pwrite))) { + OB_LOG(WARN, "failed to do write", K(ret), KP(buf), K(size), K(offset)); + } + return ret; +} + +int ObStorageCosAppendWriter::close() +{ + int ret = OB_SUCCESS; + ObExternalIOCounterGuard io_guard; + file_length_ = -1; + reset(); + return ret; +} + +int ObStorageCosAppendWriter::do_write( + const char *buf, + const int64_t size, + const int64_t offset, + const bool is_pwrite) +{ + int ret = OB_SUCCESS; + + const int64_t start_time = ObTimeUtility::current_time(); + if(NULL == buf || size < 0) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "buf is NULL or size is invalid", K(ret), KP(buf), K(size)); + } else { + qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer( + handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length()); + qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer( + handle_.get_object_name().ptr(), handle_.get_object_name().length()); + + int64_t pos = 0; + bool is_exist = false; + qcloud_cos::CosObjectMeta obj_meta; + bool is_appendable = true; + if (OB_FAIL(qcloud_cos::ObCosWrapper::head_object_meta(handle_.get_ptr(), bucket_name, + object_name, is_exist, obj_meta))) { + OB_LOG(WARN, "fail to get object meta", K(ret)); + } else if (is_exist) { + pos = obj_meta.file_length_; + is_appendable = (obj_meta.type_ == qcloud_cos::CosObjectMeta::COS_OBJ_APPENDABLE); + } + + if (OB_FAIL(ret)) { + } else if (!is_appendable) { + ret = OB_CLOUD_OBJECT_NOT_APPENDABLE; + OB_LOG(WARN, "we can only append an appendable obj", K(ret), K(is_appendable)); + } else if (is_pwrite && pos != offset) { + ret = OB_BACKUP_PWRITE_OFFSET_NOT_MATCH; + OB_LOG(WARN, "offset is not match with real length", K(ret), K(pos), K(offset), K(obj_meta.type_)); + } else if (OB_FAIL(qcloud_cos::ObCosWrapper::append(handle_.get_ptr(), bucket_name, + object_name, buf, size, offset))) { + OB_LOG(WARN, "fail to append object in cos", K(ret), KP(buf), K(size), K(offset), K(is_pwrite)); + + // If append failed, print the current object meta, to help debugging. + int tmp_ret = OB_SUCCESS; + obj_meta.reset(); + is_exist = false; + if (OB_TMP_FAIL(qcloud_cos::ObCosWrapper::head_object_meta(handle_.get_ptr(), bucket_name, + object_name, is_exist, obj_meta))) { + OB_LOG(WARN, "fail to get object meta", K(tmp_ret)); + } else { + OB_LOG(INFO, "after append fail, we got the object meta", K(is_exist), K(obj_meta.type_), + K(obj_meta.file_length_)); + } + } else { + file_length_ += size; + } + } + return ret; +} + +} //common +} //oceanbase diff --git a/deps/oblib/src/lib/restore/ob_storage_cos_base.h b/deps/oblib/src/lib/restore/ob_storage_cos_base.h new file mode 100644 index 000000000..667985712 --- /dev/null +++ b/deps/oblib/src/lib/restore/ob_storage_cos_base.h @@ -0,0 +1,154 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_AGENTSERVER_OSS_STORAGE_COS_BASE_H_ +#define OCEANBASE_AGENTSERVER_OSS_STORAGE_COS_BASE_H_ + +#include "lib/allocator/page_arena.h" +#include "lib/container/ob_array.h" +#include "lib/allocator/page_arena.h" +#include "lib/lock/ob_spin_rwlock.h" +#include "ob_i_storage.h" +#include "cos/ob_cos_wrapper_handle.h" + +namespace oceanbase +{ +namespace common +{ +class ObStorageCosBase; +struct CosListFilesCbArg; + +// Before using cos, you need to initialize cos enviroment. +// Thread safe guaranteed by user. +int init_cos_env(); + +// You need to clean cos resource when not use cos any more. +// Thread safe guaranteed by user. +void fin_cos_env(); + +class ObStorageCosUtil: public ObIStorageUtil +{ +public: + ObStorageCosUtil(); + virtual ~ObStorageCosUtil(); + virtual int open(common::ObObjectStorageInfo *storage_info); + virtual void close(); + virtual int is_exist(const common::ObString &uri, bool &is_exist); + virtual int get_file_length(const common::ObString &uri, int64_t &file_length); + virtual int write_single_file(const common::ObString &uri, const char *buf, + const int64_t size); + + //cos no dir + virtual int mkdir(const common::ObString &uri); + virtual int del_file(const common::ObString &uri); + virtual int list_files(const common::ObString &uri, common::ObBaseDirEntryOperator &op); + virtual int del_dir(const common::ObString &uri); + virtual int list_directories(const common::ObString &uri, common::ObBaseDirEntryOperator &op); + virtual int is_tagging(const common::ObString &uri, bool &is_tagging); +private: + int get_object_meta_(const common::ObString &uri, bool &is_file_exist, int64_t &file_length); + +private: + bool is_opened_; + common::ObObjectStorageInfo *storage_info_; +}; + +class ObStorageCosBase +{ +public: + ObStorageCosBase(); + virtual ~ObStorageCosBase(); + + virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info); + void reset(); + const ObCosWrapperHandle &get_handle() { return handle_; } + + // some cos function + int get_cos_file_meta(bool &is_file_exist, common::qcloud_cos::CosObjectMeta &obj_meta); + int delete_object(const common::ObString &uri); + int list_objects(const common::ObString &uri, const common::ObString &dir_name_str, + const char *separator, common::CosListFilesCbArg &arg); + int list_directories(const common::ObString &uri, const common::ObString &dir_name_str, + const char *next_marker_str, const char *delimiter_str, common::CosListFilesCbArg &arg); + int is_object_tagging(const common::ObString &uri, bool &is_tagging); + +private: + int init_handle(const common::ObObjectStorageInfo &storage_info); + bool is_valid() const { return handle_.is_valid(); } + +protected: + bool is_opened_; + ObCosWrapperHandle handle_; + +private: + DISALLOW_COPY_AND_ASSIGN(ObStorageCosBase); +}; + +class ObStorageCosWriter : public ObStorageCosBase, public ObIStorageWriter +{ +public: + ObStorageCosWriter(); + ~ObStorageCosWriter(); + virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info) override; + int write(const char *buf, const int64_t size); + int pwrite(const char *buf, const int64_t size, const int64_t offset); + int close(); + int64_t get_length() const { return file_length_;} + bool is_opened() const { return is_opened_; } +private: + int64_t file_length_; + DISALLOW_COPY_AND_ASSIGN(ObStorageCosWriter); +}; + +class ObStorageCosReader: public ObStorageCosBase, public ObIStorageReader +{ +public: + ObStorageCosReader(); + virtual ~ObStorageCosReader(); + virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info) override; + int pread(char *buf, const int64_t buf_size, int64_t offset, int64_t &read_size); + int close(); + int64_t get_length() const { return file_length_; } + bool is_opened() const { return is_opened_; } + +private: + int64_t file_length_; + + DISALLOW_COPY_AND_ASSIGN(ObStorageCosReader); +}; + +class ObStorageCosAppendWriter : public ObStorageCosBase, public ObIStorageWriter +{ +public: + ObStorageCosAppendWriter(); + virtual ~ObStorageCosAppendWriter(); + +public: + virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info) override; + int write(const char *buf, const int64_t size); + int pwrite(const char *buf, const int64_t size, const int64_t offset); + int close(); + int64_t get_length() const { return file_length_; } + bool is_opened() const { return is_opened_; } + +private: + int do_write(const char *buf, const int64_t size, const int64_t offset, const bool is_pwrite); + +private: + int64_t file_length_; + + DISALLOW_COPY_AND_ASSIGN(ObStorageCosAppendWriter); +}; + +} //common +} //oceanbase +#endif \ No newline at end of file diff --git a/deps/oblib/src/lib/restore/ob_storage_file.cpp b/deps/oblib/src/lib/restore/ob_storage_file.cpp index c04e3c6c2..c0fd56bae 100644 --- a/deps/oblib/src/lib/restore/ob_storage_file.cpp +++ b/deps/oblib/src/lib/restore/ob_storage_file.cpp @@ -426,16 +426,6 @@ int ObStorageFileUtil::del_dir(const common::ObString &uri) return ret; } -int ObStorageFileUtil::check_backup_dest_lifecycle( - const common::ObString &path, - bool &is_set_lifecycle) -{ - int ret = OB_SUCCESS; - UNUSED(path); - is_set_lifecycle = false; - return ret; -} - int ObStorageFileUtil::list_directories( const ObString &uri, ObBaseDirEntryOperator &op) @@ -547,13 +537,13 @@ ObStorageFileReader::~ObStorageFileReader() } } -int ObStorageFileReader::open(const common::ObString &uri, void* device_handle) +int ObStorageFileReader::open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; char errno_buf[OB_MAX_ERROR_MSG_LEN] = ""; struct stat64 file_info; - UNUSED(device_handle); + UNUSED(storage_info); if (is_opened_) { ret = OB_INIT_TWICE; @@ -813,9 +803,9 @@ ObStorageFileWriter::~ObStorageFileWriter() } } -int ObStorageFileWriter::open(const common::ObString &uri, void* device_handle) +int ObStorageFileWriter::open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info) { - UNUSED(device_handle); + UNUSED(storage_info); int ret = OB_SUCCESS; const char *TMP_NAME_FORMAT = "%s.tmp.%ld"; ObStorageFileUtil util; @@ -901,9 +891,9 @@ ObStorageFileAppender::~ObStorageFileAppender() } } -int ObStorageFileAppender::open(const common::ObString &uri, void* device_hanlde) +int ObStorageFileAppender::open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info) { - UNUSED(device_hanlde); + UNUSED(storage_info); int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; ObStorageFileUtil util; diff --git a/deps/oblib/src/lib/restore/ob_storage_file.h b/deps/oblib/src/lib/restore/ob_storage_file.h index be906babb..fdc5772cd 100644 --- a/deps/oblib/src/lib/restore/ob_storage_file.h +++ b/deps/oblib/src/lib/restore/ob_storage_file.h @@ -24,15 +24,13 @@ class ObStorageFileUtil: public ObIStorageUtil public: ObStorageFileUtil(); virtual ~ObStorageFileUtil(); - virtual int open(void* base_info) + virtual int open(common::ObObjectStorageInfo *storage_info) { - UNUSED(base_info); + UNUSED(storage_info); return OB_SUCCESS; } - virtual void close() - { - } + virtual void close() {} virtual int is_exist(const common::ObString &uri, bool &exist); virtual int get_file_length(const common::ObString &uri, int64_t &file_length); @@ -41,14 +39,10 @@ public: virtual int mkdir(const common::ObString &uri); virtual int list_files(const common::ObString &uri, common::ObBaseDirEntryOperator &op); virtual int del_dir(const common::ObString &uri); - virtual int check_backup_dest_lifecycle(const common::ObString &dir_path, bool &is_set_lifecycle); virtual int list_directories(const common::ObString &uri, common::ObBaseDirEntryOperator &op); virtual int is_tagging(const common::ObString &uri, bool &is_tagging); private: - int get_tmp_file_format_timestamp( - const char *file_name, - bool &is_tmp_file, - int64_t ×tamp); + int get_tmp_file_format_timestamp(const char *file_name, bool &is_tmp_file, int64_t ×tamp); private: DISALLOW_COPY_AND_ASSIGN(ObStorageFileUtil); @@ -59,7 +53,7 @@ class ObStorageFileReader: public ObIStorageReader public: ObStorageFileReader(); virtual ~ObStorageFileReader(); - virtual int open(const common::ObString &uri, void* device_handle = NULL); + virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info = NULL); virtual int pread(char *buf,const int64_t buf_size, int64_t offset, int64_t &read_size); virtual int close(); virtual int64_t get_length() const { return file_length_; } @@ -78,7 +72,7 @@ class ObStorageFileBaseWriter: public ObIStorageWriter public: ObStorageFileBaseWriter(); virtual ~ObStorageFileBaseWriter(); - virtual int open(const common::ObString &uri, void* device_handle = NULL) = 0; + virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info = NULL) = 0; virtual int open(const int flags); virtual int write(const char *buf,const int64_t size); virtual int pwrite(const char *buf, const int64_t size, const int64_t offset); @@ -101,7 +95,7 @@ class ObStorageFileWriter: public ObStorageFileBaseWriter public: ObStorageFileWriter(); virtual ~ObStorageFileWriter(); - virtual int open(const common::ObString &uri, void* device_handle = NULL); + virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info = NULL); virtual int close() override; private: char real_path_[OB_MAX_URI_LENGTH]; @@ -116,7 +110,7 @@ public: ObStorageFileAppender(); ObStorageFileAppender(StorageOpenMode mode); virtual ~ObStorageFileAppender(); - virtual int open(const common::ObString &uri, void* device_handle = NULL); + virtual int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info = NULL); virtual int close() override; void set_open_mode(StorageOpenMode mode) {open_mode_ = mode;} private: diff --git a/deps/oblib/src/lib/restore/ob_storage_info.cpp b/deps/oblib/src/lib/restore/ob_storage_info.cpp new file mode 100644 index 000000000..adb63d99e --- /dev/null +++ b/deps/oblib/src/lib/restore/ob_storage_info.cpp @@ -0,0 +1,297 @@ +/* + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + * Authors: + * + */ +#define USING_LOG_PREFIX STORAGE + +#include "ob_storage_info.h" +#include "ob_storage.h" + +namespace oceanbase +{ + +namespace common +{ +//***********************ObObjectStorageInfo*************************** +ObObjectStorageInfo::ObObjectStorageInfo() + : device_type_(ObStorageType::OB_STORAGE_MAX_TYPE) +{ + endpoint_[0] = '\0'; + access_id_[0] = '\0'; + access_key_[0] = '\0'; + extension_[0] = '\0'; +} + +ObObjectStorageInfo::~ObObjectStorageInfo() +{ + reset(); +} + +void ObObjectStorageInfo::reset() +{ + device_type_ = ObStorageType::OB_STORAGE_MAX_TYPE; + endpoint_[0] = '\0'; + access_id_[0] = '\0'; + access_key_[0] = '\0'; + extension_[0] = '\0'; +} + +bool ObObjectStorageInfo::is_valid() const +{ + return device_type_ >= 0 && device_type_ < ObStorageType::OB_STORAGE_MAX_TYPE; +} + +int64_t ObObjectStorageInfo::hash() const +{ + int64_t hash_value = 0; + hash_value = murmurhash(&device_type_, static_cast(sizeof(device_type_)), hash_value); + hash_value = murmurhash(endpoint_, static_cast(strlen(endpoint_)), hash_value); + hash_value = murmurhash(access_id_, static_cast(strlen(access_id_)), hash_value); + hash_value = murmurhash(access_key_, static_cast(strlen(access_key_)), hash_value); + hash_value = murmurhash(extension_, static_cast(strlen(extension_)), hash_value); + return hash_value; +} + +bool ObObjectStorageInfo::operator ==(const ObObjectStorageInfo &storage_info) const +{ + return device_type_ == storage_info.device_type_ + && (0 == STRCMP(endpoint_, storage_info.endpoint_)) + && (0 == STRCMP(access_id_, storage_info.access_id_)) + && (0 == STRCMP(access_key_, storage_info.access_key_)) + && (0 == STRCMP(extension_, storage_info.extension_)); +} + +bool ObObjectStorageInfo::operator !=(const ObObjectStorageInfo &storage_info) const +{ + return !(*this == storage_info); +} + +const char *ObObjectStorageInfo::get_type_str() const +{ + return get_storage_type_str(device_type_); +} + +ObStorageType ObObjectStorageInfo::get_type() const +{ + return device_type_; +} + +// oss:host=xxxx&access_id=xxx&access_key=xxx +// cos:host=xxxx&access_id=xxx&access_key=xxxappid=xxx +int ObObjectStorageInfo::set(const common::ObStorageType device_type, const char *storage_info) +{ + bool has_appid = false; + int ret = OB_SUCCESS; + if (is_valid()) { + ret = OB_INIT_TWICE; + LOG_WARN("storage info init twice", K(ret)); + } else if (OB_ISNULL(storage_info) || strlen(storage_info) >= OB_MAX_BACKUP_STORAGE_INFO_LENGTH) { + ret = OB_INVALID_BACKUP_DEST; + LOG_WARN("storage info is invalid", K(ret), K(storage_info)); + } else if (FALSE_IT(device_type_ = device_type)) { + } else if (0 == strlen(storage_info)) { + if (OB_STORAGE_FILE != device_type_) { + ret = OB_INVALID_BACKUP_DEST; + LOG_WARN("storage info is empty", K(ret), K_(device_type)); + } + } else if (OB_FAIL(parse_storage_info_(storage_info, has_appid))) { + LOG_WARN("parse storage info failed", K(ret)); + } else if (OB_STORAGE_FILE != device_type + && (0 == strlen(endpoint_) || 0 == strlen(access_id_) || 0 == strlen(access_key_))) { + ret = OB_INVALID_BACKUP_DEST; + LOG_WARN("backup device is not nfs, endpoint/access_id/access_key do not allow to be empty", + K(ret), K_(device_type), K_(endpoint), K_(access_id)); + } else if (OB_STORAGE_COS == device_type && !has_appid) { + ret = OB_INVALID_BACKUP_DEST; + LOG_WARN("invalid cos info, appid do not allow to be empty", K(ret), K_(extension)); + } else if (OB_STORAGE_FILE == device_type + && (0 != strlen(endpoint_) || 0 != strlen(access_id_) || 0 != strlen(access_key_))) { + ret = OB_INVALID_BACKUP_DEST; + LOG_WARN("backup device is nfs, endpoint/access_id/access_key must be empty", + K(ret), K_(device_type), K_(endpoint), K_(access_id)); + } else { + } + + + if (OB_SUCC(ret)) { + LOG_INFO("succ to parse storage info", + K_(device_type), K_(endpoint), K_(access_id), K_(extension)); + } else { + reset(); + } + return ret; +} + +int ObObjectStorageInfo::set(const char *uri, const char *storage_info) +{ + int ret = OB_SUCCESS; + common::ObStorageType device_type; + if (OB_FAIL(get_storage_type_from_path(uri, device_type))) { + LOG_WARN("failed to get storage type from path", K(ret), KPC(this)); + } else if (OB_FAIL(set(device_type, storage_info))) { + LOG_WARN("failed to set storage info", K(ret), KPC(this)); + } + return ret; +} + +int ObObjectStorageInfo::parse_storage_info_(const char *storage_info, bool &has_appid) +{ + int ret = OB_SUCCESS; + + if (OB_ISNULL(storage_info) || strlen(storage_info) >= OB_MAX_BACKUP_STORAGE_INFO_LENGTH) { + ret = OB_INVALID_BACKUP_DEST; + LOG_WARN("storage info is invalid", K(ret), K(storage_info), K(strlen(storage_info))); + } else { + char tmp[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 }; + char *token = NULL; + char *saved_ptr = NULL; + int64_t info_len = strlen(storage_info); + + MEMCPY(tmp, storage_info, info_len); + tmp[info_len] = '\0'; + token = tmp; + for (char *str = token; OB_SUCC(ret); str = NULL) { + token = ::strtok_r(str, "&", &saved_ptr); + if (NULL == token) { + break; + } else if (0 == strncmp(HOST, token, strlen(HOST))) { + if (OB_FAIL(set_storage_info_field_(token, endpoint_, sizeof(endpoint_)))) { + LOG_WARN("failed to set endpoint",K(ret), K(token)); + } + } else if (0 == strncmp(ACCESS_ID, token, strlen(ACCESS_ID))) { + if (OB_FAIL(set_storage_info_field_(token, access_id_, sizeof(access_id_)))) { + LOG_WARN("failed to set access id", K(ret), K(token)); + } + } else if (0 == strncmp(ACCESS_KEY, token, strlen(ACCESS_KEY))) { + if (OB_FAIL(set_storage_info_field_(token, access_key_, sizeof(access_key_)))) { + LOG_WARN("failed to set access key", K(ret), K(token)); + } + } else if (OB_STORAGE_FILE != device_type_ && 0 == strncmp(APPID, token, strlen(APPID))) { + has_appid = true; + if (OB_FAIL(set_storage_info_field_(token, extension_, sizeof(extension_)))) { + LOG_WARN("failed to set appid", K(ret), K(token)); + } + } else if (0 == strncmp(DELETE_MODE, token, strlen(DELETE_MODE))) { + if (OB_STORAGE_FILE == device_type_) { + ret = OB_INVALID_BACKUP_DEST; + OB_LOG(WARN, "OB_STORAGE_FILE don't support delete mode yet", + K(ret), K_(device_type), K(token)); + } else if (OB_FAIL(check_delete_mode_(token + strlen(DELETE_MODE)))) { + OB_LOG(WARN, "failed to check delete mode", K(ret), K(token)); + } else if (OB_FAIL(set_storage_info_field_(token, extension_, sizeof(extension_)))) { + LOG_WARN("failed to set delete mode", K(ret), K(token)); + } + } else { + } + } + } + return ret; +} + +int ObObjectStorageInfo::check_delete_mode_(const char *delete_mode) const +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(delete_mode)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid args", K(ret), KP(delete_mode)); + } else if (0 != strcmp(delete_mode, "delete") && 0 != strcmp(delete_mode, "tagging")) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "delete mode is invalid", K(ret), K(delete_mode)); + } + return ret; +} + +int ObObjectStorageInfo::set_storage_info_field_(const char *info, char *field, const int64_t length) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(info) || OB_ISNULL(field)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", K(ret), KP(info), KP(field)); + } else { + const int64_t info_len = strlen(info); + int64_t pos = strlen(field); + if (info_len >= length) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("info is too long ", K(ret), K(info), K(length)); + } else if (pos > 0 && OB_FAIL(databuff_printf(field, length, pos, "&"))) { + // cos:host=xxxx&access_id=xxx&access_key=xxxappid=xxx&delete_mode=xxx + // extension_ may contain both appid and delete_mode + // so delimiter '&' should be included + LOG_WARN("failed to add delimiter to storage info field", K(ret), K(info), K(field), K(length)); + } else if (OB_FAIL(databuff_printf(field, length, pos, "%s", info))) { + LOG_WARN("failed to set storage info field", K(ret), K(info), K(field), K(length)); + } + } + return ret; +} + +int ObObjectStorageInfo::assign(const ObObjectStorageInfo &storage_info) +{ + int ret = OB_SUCCESS; + device_type_ = storage_info.device_type_; + MEMCPY(endpoint_, storage_info.endpoint_, sizeof(endpoint_)); + MEMCPY(access_id_, storage_info.access_id_, sizeof(access_id_)); + MEMCPY(access_key_, storage_info.access_key_, sizeof(access_key_)); + MEMCPY(extension_, storage_info.extension_, sizeof(extension_)); + + return ret; +} + +int ObObjectStorageInfo::get_storage_info_str(char *storage_info, const int64_t info_len) const +{ + int ret = OB_SUCCESS; + const int64_t key_len = MAX(OB_MAX_BACKUP_SERIALIZEKEY_LENGTH, OB_MAX_BACKUP_ACCESSKEY_LENGTH); + char key[key_len] = { 0 }; + if (!is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("storage info not init", K(ret)); + } else if (OB_ISNULL(storage_info) || (info_len <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", K(ret), KP(storage_info), K(info_len)); + } else if (OB_STORAGE_FILE != device_type_) { + if (OB_FAIL(get_access_key_(key, sizeof(key)))) { + LOG_WARN("failed to get access key", K(ret)); + } else if (OB_FAIL(databuff_printf(storage_info, info_len, "%s&%s&%s", + endpoint_, access_id_, key))) { + LOG_WARN("failed to set storage info", K(ret), K(info_len)); + } + } + + if (OB_SUCC(ret) && 0 != strlen(extension_) && info_len > strlen(storage_info)) { + // if OB_STORAGE_FILE's extension is not empty + // delimiter should be included + int64_t str_len = strlen(storage_info); + if (str_len > 0 && OB_FAIL(databuff_printf(storage_info, info_len, str_len, "&"))) { + LOG_WARN("failed to add delimiter to storage info", K(ret), K(info_len), K(str_len)); + } else if (OB_FAIL(databuff_printf(storage_info, info_len, str_len, "%s", extension_))) { + LOG_WARN("failed to add extension", K(ret), K(info_len), K(str_len), K_(extension)); + } + } + + return ret; +} + +int ObObjectStorageInfo::get_access_key_(char *key_buf, const int64_t key_buf_len) const +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(key_buf) || key_buf_len <= strlen(access_key_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", K(ret), KP(key_buf), K(key_buf_len)); + } else { + MEMCPY(key_buf, access_key_, strlen(access_key_)); + key_buf[strlen(access_key_)] = '\0'; + } + return ret; +} + +} +} diff --git a/deps/oblib/src/lib/restore/ob_storage_info.h b/deps/oblib/src/lib/restore/ob_storage_info.h new file mode 100644 index 000000000..91d00614f --- /dev/null +++ b/deps/oblib/src/lib/restore/ob_storage_info.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + * Authors: + * + */ + +#ifndef OCEANBASE_LIB_RESTORE_OB_STORAGE_INFO_H_ +#define OCEANBASE_LIB_RESTORE_OB_STORAGE_INFO_H_ + +#include "common/storage/ob_device_common.h" +#include "lib/utility/ob_print_utils.h" +#include "lib/utility/ob_unify_serialize.h" + +namespace oceanbase +{ + +namespace common +{ + +const int64_t OB_MAX_BACKUP_EXTENSION_LENGTH = 512; +const int64_t OB_MAX_BACKUP_ENDPOINT_LENGTH = 256; +const int64_t OB_MAX_BACKUP_ACCESSID_LENGTH = 256; +const int64_t OB_MAX_BACKUP_ACCESSKEY_LENGTH = 256; +const int64_t OB_MAX_BACKUP_STORAGE_INFO_LENGTH = 1536; +const int64_t OB_MAX_BACKUP_ENCRYPTKEY_LENGTH = OB_MAX_BACKUP_ACCESSKEY_LENGTH + 32; +const int64_t OB_MAX_BACKUP_SERIALIZEKEY_LENGTH = OB_MAX_BACKUP_ENCRYPTKEY_LENGTH * 2; + +const char *const ACCESS_ID = "access_id="; +const char *const ACCESS_KEY = "access_key="; +const char *const HOST = "host="; +const char *const APPID = "appid="; +const char *const DELETE_MODE = "delete_mode="; + +class ObObjectStorageInfo +{ + OB_UNIS_VERSION(1); +public: + ObObjectStorageInfo(); + virtual ~ObObjectStorageInfo(); + + int set(const common::ObStorageType device_type, const char *storage_info); + int set(const char *uri, const char *storage_info); + virtual int assign(const ObObjectStorageInfo &storage_info); + ObStorageType get_type() const; + const char *get_type_str() const; + int get_storage_info_str(char *storage_info, const int64_t info_len) const; + + bool is_valid() const; + void reset(); + int64_t hash() const; + bool operator ==(const ObObjectStorageInfo &storage_info) const; + bool operator !=(const ObObjectStorageInfo &storage_info) const; + TO_STRING_KV(K_(endpoint), K_(access_id), K_(extension), "type", get_type_str()); + +protected: + virtual int get_access_key_(char *key_buf, const int64_t key_buf_len) const; + virtual int parse_storage_info_(const char *storage_info, bool &has_appid); + int check_delete_mode_(const char *delete_mode) const; + int set_storage_info_field_(const char *info, char *field, const int64_t length); + +public: + common::ObStorageType device_type_; + char endpoint_[OB_MAX_BACKUP_ENDPOINT_LENGTH]; + char access_id_[OB_MAX_BACKUP_ACCESSID_LENGTH]; + char access_key_[OB_MAX_BACKUP_ACCESSKEY_LENGTH]; + char extension_[OB_MAX_BACKUP_EXTENSION_LENGTH]; +}; + +} +} + +#endif diff --git a/deps/oblib/src/lib/restore/ob_storage_oss_base.cpp b/deps/oblib/src/lib/restore/ob_storage_oss_base.cpp index fc553cd7e..17b6668aa 100644 --- a/deps/oblib/src/lib/restore/ob_storage_oss_base.cpp +++ b/deps/oblib/src/lib/restore/ob_storage_oss_base.cpp @@ -351,7 +351,7 @@ ObStorageOssBase::ObStorageOssBase() :aos_pool_(NULL), oss_option_(NULL), is_inited_(false), - oss_account_(NULL) + oss_account_() { memset(oss_endpoint_, 0, MAX_OSS_ENDPOINT_LENGTH); } @@ -374,16 +374,21 @@ void ObStorageOssBase::reset() } } -int ObStorageOssBase::init_with_oss_account(void* account) +int ObStorageOssBase::init_with_storage_info(common::ObObjectStorageInfo *storage_info) { int ret = OB_SUCCESS; - oss_account_ = static_cast(account); + char info_str[common::OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 }; + if(is_inited_) { ret = OB_INIT_TWICE; OB_LOG(WARN, "oss client init twice", K(ret)); - } else if (OB_ISNULL(oss_account_)) { + } else if (OB_ISNULL(storage_info)) { ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "oss account is null, fail to init oss base!", K(ret)); + } else if (OB_FAIL(storage_info->get_storage_info_str(info_str, sizeof(info_str)))) { + OB_LOG(WARN, "fail to get storage info str", K(ret), KPC(storage_info)); + } else if (OB_FAIL(oss_account_.parse_oss_arg(info_str))) { + OB_LOG(WARN, "fail to build oss account", K(ret)); } else if (OB_FAIL(init_oss_options(aos_pool_, oss_option_))) { OB_LOG(WARN, "fail to init oss options", K(aos_pool_), K(oss_option_), K(ret)); } else if (OB_ISNULL(aos_pool_) || OB_ISNULL(oss_option_)) { @@ -510,8 +515,8 @@ int ObStorageOssBase::init_oss_options(aos_pool_t *&aos_pool, oss_request_option OB_LOG(WARN, "fail to init oss endpoind", K(ret)); } else { aos_str_set(&oss_option->config->endpoint, oss_endpoint_); - aos_str_set(&oss_option->config->access_key_id, oss_account_->oss_id_); - aos_str_set(&oss_option->config->access_key_secret, oss_account_->oss_key_); + aos_str_set(&oss_option->config->access_key_id, oss_account_.oss_id_); + aos_str_set(&oss_option->config->access_key_secret, oss_account_.oss_key_); oss_option->config->is_cname = 0; oss_option->ctl = aos_http_controller_create(oss_option->pool, 0); @@ -549,16 +554,16 @@ int ObStorageOssBase::reinit_oss_option() int ObStorageOssBase::init_oss_endpoint() { int ret = OB_SUCCESS; - if (0 == strlen(oss_account_->oss_domain_)) { + if (0 == strlen(oss_account_.oss_domain_)) { ret = OB_OSS_ERROR; OB_LOG(WARN, "oss domain is empty", K(ret)); } else { - int64_t str_len = strlen(oss_account_->oss_domain_); + int64_t str_len = strlen(oss_account_.oss_domain_); if (str_len >= sizeof(oss_endpoint_)) { ret = OB_BUF_NOT_ENOUGH; - OB_LOG(WARN, "oss domain is too long", K(ret), KCSTRING(oss_account_->oss_domain_)); + OB_LOG(WARN, "oss domain is too long", K(ret), KCSTRING(oss_account_.oss_domain_)); } else { - MEMCPY(oss_endpoint_, oss_account_->oss_domain_, str_len); + MEMCPY(oss_endpoint_, oss_account_.oss_domain_, str_len); oss_endpoint_[str_len] = '\0'; } } @@ -641,9 +646,9 @@ void ObStorageOssBase::print_oss_info(aos_table_t *resp_headers, aos_status_s *a } OB_LOG_RET(WARN, OB_SUCCESS, "oss info ", K(aos_ret->code), KCSTRING(aos_ret->error_code), KCSTRING(aos_ret->error_msg), KCSTRING(aos_ret->req_id), KCSTRING(delay_time), - KCSTRING(oss_account_->oss_domain_), KCSTRING(oss_endpoint_), KCSTRING(oss_account_->oss_id_)); + KCSTRING(oss_account_.oss_domain_), KCSTRING(oss_endpoint_), KCSTRING(oss_account_.oss_id_)); } else { - OB_LOG_RET(WARN, OB_SUCCESS, "oss info ", KCSTRING(oss_account_->oss_domain_), KCSTRING(oss_endpoint_), KCSTRING(oss_account_->oss_id_)); + OB_LOG_RET(WARN, OB_SUCCESS, "oss info ", KCSTRING(oss_account_.oss_domain_), KCSTRING(oss_endpoint_), KCSTRING(oss_account_.oss_id_)); } } @@ -667,7 +672,7 @@ ObStorageOssMultiPartWriter::~ObStorageOssMultiPartWriter() { } -int ObStorageOssMultiPartWriter::open(const ObString &uri, void* oss_account) +int ObStorageOssMultiPartWriter::open(const ObString &uri, common::ObObjectStorageInfo *storage_info) { int ret = OB_SUCCESS; ObExternalIOCounterGuard io_guard; @@ -677,7 +682,7 @@ int ObStorageOssMultiPartWriter::open(const ObString &uri, void* oss_account) } else if (is_opened_) { ret = OB_OSS_ERROR; OB_LOG(WARN, "already open, cannot open again", K(ret)); - } else if (OB_FAIL(init_with_oss_account(oss_account))) { + } else if (OB_FAIL(init_with_storage_info(storage_info))) { OB_LOG(WARN, "failed to init oss", K(ret)); } else if (OB_FAIL(get_bucket_object_name(uri, bucket_, object_, allocator_))) { OB_LOG(WARN, "get bucket object error", K(uri), K(ret)); @@ -1031,7 +1036,7 @@ ObStorageOssReader::~ObStorageOssReader() { } -int ObStorageOssReader::open(const ObString &uri, void* base_info) +int ObStorageOssReader::open(const ObString &uri, common::ObObjectStorageInfo *storage_info) { int ret = OB_SUCCESS; ObExternalIOCounterGuard io_guard; @@ -1045,7 +1050,7 @@ int ObStorageOssReader::open(const ObString &uri, void* base_info) } else if(is_opened_) { ret = OB_OSS_ERROR; OB_LOG(WARN, "already open, cannot open again", K(ret)); - } else if (OB_FAIL(init_with_oss_account(base_info))) { + } else if (OB_FAIL(init_with_storage_info(storage_info))) { OB_LOG(WARN, "failed to init oss", K(ret)); } else if (OB_SUCCESS != (ret = get_bucket_object_name(uri, bucket_, object_, allocator_))) { OB_LOG(WARN, "bucket name of object name is empty", K(ret)); @@ -1192,7 +1197,7 @@ int ObStorageOssReader::close() return ret; } -ObStorageOssUtil::ObStorageOssUtil() :is_opened_(false), oss_account_(NULL) +ObStorageOssUtil::ObStorageOssUtil() :is_opened_(false), storage_info_(NULL) { } @@ -1200,18 +1205,18 @@ ObStorageOssUtil::~ObStorageOssUtil() { } -int ObStorageOssUtil::open(void* account) +int ObStorageOssUtil::open(common::ObObjectStorageInfo *storage_info) { int ret = OB_SUCCESS; if (is_opened_) { ret = OB_OSS_ERROR; OB_LOG(WARN, "already open, cannot open again", K(ret)); - } else if (OB_ISNULL(account)) { + } else if (OB_ISNULL(storage_info)) { ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "oss account is null", K(ret)); } else { is_opened_ = true; - oss_account_ = static_cast(account); + storage_info_ = storage_info; } return ret; } @@ -1222,7 +1227,7 @@ void ObStorageOssUtil::close() OB_LOG_RET(WARN, OB_ERR_UNEXPECTED, "oss util cannot close before it is opened"); } else { is_opened_ = false; - oss_account_ = NULL; + storage_info_ = NULL; } } @@ -1244,7 +1249,7 @@ int ObStorageOssUtil::is_exist(const ObString &uri, bool &exist) } else if (!is_opened_) { ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "oss util is not inited", K(ret), K(uri)); - } else if (OB_FAIL(oss_base.init_with_oss_account(oss_account_))) { + } else if (OB_FAIL(oss_base.init_with_storage_info(storage_info_))) { OB_LOG(WARN, "fail to init oss base with account", K(ret), K(uri)); } else if (OB_FAIL(get_bucket_object_name(uri, bucket_ob_string, object_ob_string, allocator))) { @@ -1277,7 +1282,7 @@ int ObStorageOssUtil::get_file_length(const common::ObString &uri, int64_t &file } else if (!is_opened_) { ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "oss util is not inited", K(ret), K(uri)); - } else if (OB_FAIL(oss_base.init_with_oss_account(oss_account_))) { + } else if (OB_FAIL(oss_base.init_with_storage_info(storage_info_))) { OB_LOG(WARN, "fail to init oss base with account", K(ret), K(uri)); } else if (OB_SUCCESS != (ret = get_bucket_object_name(uri, bucket_ob_string, object_ob_string, allocator))) { @@ -1302,7 +1307,7 @@ int ObStorageOssUtil::write_single_file( int ret = OB_SUCCESS; ObExternalIOCounterGuard io_guard; ObStorageOssWriter writer; - if (OB_FAIL(writer.open(uri, oss_account_))) { + if (OB_FAIL(writer.open(uri, storage_info_))) { OB_LOG(WARN, "fail to open the writer(overwrite)", K(ret)); } else if (OB_FAIL(writer.write(buf, size))) { OB_LOG(WARN, "fail to write the writer(overwrite)", K(ret), K(size), KP(buf)); @@ -1333,7 +1338,7 @@ int ObStorageOssUtil::is_tagging( if (uri.empty()) { ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "name is empty", K(ret)); - } else if (OB_FAIL(oss_base.init_with_oss_account(oss_account_))) { + } else if (OB_FAIL(oss_base.init_with_storage_info(storage_info_))) { OB_LOG(WARN, "fail to init oss base with account", K(ret), K(uri)); } else if (OB_FAIL(get_bucket_object_name(uri, bucket_str, object_str, allocator))) { OB_LOG(WARN, "bucket or object name is empty", K(ret), K(uri), K(bucket_str), K(object_str)); @@ -1446,15 +1451,15 @@ int ObStorageOssUtil::del_file(const common::ObString &uri) } else if (!is_opened_) { ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "oss util is not inited", K(ret), K(uri)); - } else if (OB_FAIL(oss_base.init_with_oss_account(oss_account_))) { + } else if (OB_FAIL(oss_base.init_with_storage_info(storage_info_))) { OB_LOG(WARN, "fail to init oss base with account", K(ret), K(uri)); } else if (OB_FAIL(get_bucket_object_name(uri, bucket_str, object_str, allocator))) { OB_LOG(WARN, "bucket or object name is empty", K(ret), K(uri), K(bucket_str), K(object_str)); - } else if (ObIStorageUtil::DELETE == oss_base.oss_account_->delete_mode_) { + } else if (ObIStorageUtil::DELETE == oss_base.oss_account_.delete_mode_) { if (OB_FAIL(delete_object_(uri, oss_base, bucket_str, object_str))) { OB_LOG(WARN, "failed to delete object", K(ret), K(uri)); } - } else if (ObIStorageUtil::TAGGING == oss_base.oss_account_->delete_mode_) { + } else if (ObIStorageUtil::TAGGING == oss_base.oss_account_.delete_mode_) { if (OB_FAIL(tagging_object_(uri, oss_base, bucket_str, object_str))) { OB_LOG(WARN, "failed to tagging file", K(ret), K(uri)); } @@ -1508,7 +1513,7 @@ int ObStorageOssUtil::list_files(const common::ObString &dir_path, } else if (!is_opened_) { ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "oss util is not inited", K(ret), K(dir_path)); - } else if (OB_FAIL(oss_base.init_with_oss_account(oss_account_))) { + } else if (OB_FAIL(oss_base.init_with_storage_info(storage_info_))) { OB_LOG(WARN, "fail to init oss base with account", K(ret), K(dir_path)); } else if (OB_FAIL(get_bucket_object_name(dir_path, bucket_str, object_str, tmp_allocator))) { OB_LOG(WARN, "bucket or object name is empty", K(ret), K(dir_path), K(bucket_str), K(object_str)); @@ -1634,65 +1639,6 @@ int ObStorageOssUtil::strtotime(const char *date_time, int64_t &time) return ret; } -int ObStorageOssUtil::check_backup_dest_lifecycle( - const common::ObString &path, - bool &is_set_lifecycle) -{ - int ret = OB_SUCCESS; - ObExternalIOCounterGuard io_guard; - common::ObArenaAllocator allocator; - ObString bucket_str; - ObString object_str; - aos_status_t *aos_ret = NULL; - aos_list_t lifecycle_rule_list; - aos_table_t *resp_headers = NULL; - oss_lifecycle_rule_content_t *rule_content = NULL; - aos_string_t bucket; - ObString rule_id; - ObString prefix; - ObString status; - ObString date; - ObStorageOssBase oss_base; - int days = 0; - is_set_lifecycle = false; - - if (path.empty()) { - ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "name is empty", K(ret)); - } else if (!is_opened_) { - ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "oss util is not inited", K(ret), K(path)); - } else if (OB_FAIL(oss_base.init_with_oss_account(oss_account_))) { - OB_LOG(WARN, "fail to init oss base with account", K(ret), K(path)); - } else if (OB_SUCCESS != (ret = get_bucket_object_name(path, bucket_str, - object_str, allocator))) { - OB_LOG(WARN, "bucket or object name is empty", - K(bucket_str), K(object_str), K(ret)); - } else { - aos_str_set(&bucket, bucket_str.ptr()); - aos_list_init(&lifecycle_rule_list); - if (OB_ISNULL(aos_ret = oss_get_bucket_lifecycle(oss_base.oss_option_, &bucket, &lifecycle_rule_list, &resp_headers)) || !aos_status_is_ok(aos_ret)) { - convert_io_error(aos_ret, ret); - OB_LOG(WARN, "fail to list all object", K(ret)); - oss_base.print_oss_info(resp_headers, aos_ret); - } else { - aos_list_for_each_entry(oss_lifecycle_rule_content_t, rule_content, &lifecycle_rule_list, node) { - prefix = apr_psprintf(oss_base.aos_pool_, "%.*s", rule_content->prefix.len, rule_content->prefix.data); - status = apr_psprintf(oss_base.aos_pool_,"%.*s", rule_content->status.len, rule_content->status.data); - date = apr_psprintf(oss_base.aos_pool_, "%.*s", rule_content->date.len, rule_content->date.data); - days = rule_content->days; - if (object_str.prefix_match(prefix)) { - if (days > 0 || !date.empty()) { - is_set_lifecycle = true; - break; - } - } - } - } - } - return ret; -} - int ObStorageOssUtil::list_directories( const common::ObString &dir_path, common::ObBaseDirEntryOperator &op) @@ -1719,7 +1665,7 @@ int ObStorageOssUtil::list_directories( } else if (!is_opened_) { ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "oss util is not inited", K(ret), K(dir_path)); - } else if (OB_FAIL(oss_base.init_with_oss_account(oss_account_))) { + } else if (OB_FAIL(oss_base.init_with_storage_info(storage_info_))) { OB_LOG(WARN, "fail to init oss base with account", K(ret), K(dir_path)); } else if (OB_FAIL(get_bucket_object_name(dir_path, bucket_str, object_str, tmp_allocator))) { OB_LOG(WARN, "bucket or object name is empty", K(ret), K(dir_path), K(bucket_str), K(object_str)); @@ -1845,7 +1791,9 @@ ObStorageOssAppendWriter::~ObStorageOssAppendWriter() { } -int ObStorageOssAppendWriter::open(const common::ObString &uri, void* oss_account) +int ObStorageOssAppendWriter::open( + const common::ObString &uri, + common::ObObjectStorageInfo *storage_info) { int ret = OB_SUCCESS; ObExternalIOCounterGuard io_guard; @@ -1856,7 +1804,7 @@ int ObStorageOssAppendWriter::open(const common::ObString &uri, void* oss_accoun } else if (is_opened_) { ret = OB_OSS_ERROR; OB_LOG(WARN, "already open, cannot open again", K(ret), K(uri)); - } else if (OB_FAIL(init_with_oss_account(oss_account))) { + } else if (OB_FAIL(init_with_storage_info(storage_info))) { OB_LOG(WARN, "failed to init oss", K(ret), K(uri)); } else if (OB_FAIL(get_bucket_object_name(uri, bucket_, object_, allocator_))) { OB_LOG(WARN, "get bucket object error", K(uri), K(ret)); @@ -1971,7 +1919,7 @@ int ObStorageOssAppendWriter::do_write(const char *buf, const int64_t size, cons OB_LOG(ERROR, "oss type is null", K(ret)); } else if (0 != strncmp(OSS_OBJECT_TYPE_APPENDABLE, object_type, strlen(OSS_OBJECT_TYPE_APPENDABLE))) { ret = OB_CLOUD_OBJECT_NOT_APPENDABLE; - OB_LOG(WARN, "oss object not match", K(ret), KCSTRING(object_type)); + OB_LOG(WARN, "oss object must be appendable", K(ret), KCSTRING(object_type)); } else { char *next_append_position = (char*)(apr_table_get(resp_headers, OSS_NEXT_APPEND_POSITION)); position = aos_atoi64(next_append_position); @@ -2005,6 +1953,18 @@ int ObStorageOssAppendWriter::do_write(const char *buf, const int64_t size, cons print_oss_info(resp_headers, aos_ret); convert_io_error(aos_ret, ret); OB_LOG(WARN, "fail to append", K(content), K(ret)); + + // If append failed, print the current object meta, to help debugging. + aos_table_t *headers3 = NULL; + if(OB_NOT_NULL(headers3 = aos_table_make(aos_pool_, 0))) { + if(OB_NOT_NULL(aos_ret = oss_head_object(oss_option_, &bucket, &object, headers3, &resp_headers))) { + if ((0 != aos_status_is_ok(aos_ret))) { + char *append_pos_str = (char*)(apr_table_get(resp_headers, OSS_NEXT_APPEND_POSITION)); + int64_t cur_pos = aos_atoi64(append_pos_str); + OB_LOG(WARN, "after append fail, we got the object meta", K(cur_pos)); + } + } + } } } } @@ -2044,7 +2004,9 @@ ObStorageOssWriter::~ObStorageOssWriter() } } -int ObStorageOssWriter::open(const common::ObString &uri, void* oss_base) +int ObStorageOssWriter::open( + const common::ObString &uri, + common::ObObjectStorageInfo *storage_info) { int ret = OB_SUCCESS; if (uri.empty()) { @@ -2053,7 +2015,7 @@ int ObStorageOssWriter::open(const common::ObString &uri, void* oss_base) } else if (is_opened_) { ret = OB_OSS_ERROR; OB_LOG(WARN, "already open, cannot open again", K(ret)); - } else if (OB_FAIL(init_with_oss_account(oss_base))) { + } else if (OB_FAIL(init_with_storage_info(storage_info))) { OB_LOG(WARN, "failed to init oss", K(ret), K(uri)); } else if (OB_FAIL(get_bucket_object_name(uri, bucket_, object_, allocator_))) { OB_LOG(WARN, "get bucket object error", K(uri), K(ret)); diff --git a/deps/oblib/src/lib/restore/ob_storage_oss_base.h b/deps/oblib/src/lib/restore/ob_storage_oss_base.h index f62fc204c..2d5b7d10a 100644 --- a/deps/oblib/src/lib/restore/ob_storage_oss_base.h +++ b/deps/oblib/src/lib/restore/ob_storage_oss_base.h @@ -124,14 +124,14 @@ public: bool &is_file_exist, char *&remote_md5, int64_t &file_length); void print_oss_info(aos_table_t *resp_headers, aos_status_s *aos_ret); - int init_with_oss_account(void* account); + int init_with_storage_info(common::ObObjectStorageInfo *storage_info); int init_oss_endpoint(); aos_pool_t *aos_pool_; oss_request_options_t *oss_option_; char oss_endpoint_[MAX_OSS_ENDPOINT_LENGTH]; bool is_inited_; - ObOssAccount* oss_account_; + ObOssAccount oss_account_; DISALLOW_COPY_AND_ASSIGN(ObStorageOssBase); }; @@ -141,7 +141,7 @@ class ObStorageOssWriter : public ObStorageOssBase, public ObIStorageWriter public: ObStorageOssWriter(); ~ObStorageOssWriter(); - int open(const common::ObString &uri, void* oss_base); + int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info); int write(const char *buf,const int64_t size); int pwrite(const char *buf, const int64_t size, const int64_t offset); int close(); @@ -162,7 +162,7 @@ class ObStorageOssMultiPartWriter: public ObStorageOssBase, public ObIStorageWri public: ObStorageOssMultiPartWriter(); virtual ~ObStorageOssMultiPartWriter(); - int open(const common::ObString &uri, void* oss_base); + int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info); int write(const char *buf,const int64_t size); int pwrite(const char *buf, const int64_t size, const int64_t offset); int close(); @@ -196,7 +196,7 @@ class ObStorageOssReader: public ObStorageOssBase, public ObIStorageReader public: ObStorageOssReader(); virtual ~ObStorageOssReader(); - int open(const common::ObString &uri, void* oss_base); + int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info); int pread(char *buf,const int64_t buf_size, int64_t offset, int64_t &read_size); int close(); int64_t get_length() const { return file_length_; } @@ -217,7 +217,7 @@ class ObStorageOssUtil: public ObIStorageUtil public: ObStorageOssUtil(); virtual ~ObStorageOssUtil(); - virtual int open(void* account); + virtual int open(common::ObObjectStorageInfo *storage_info); virtual void close(); virtual int is_exist(const common::ObString &uri, bool &exist); virtual int get_file_length(const common::ObString &uri, int64_t &file_length); @@ -229,8 +229,6 @@ public: virtual int del_file(const common::ObString &uri); virtual int list_files(const common::ObString &dir_path, common::ObBaseDirEntryOperator &op); virtual int del_dir(const common::ObString &uri); - virtual int check_backup_dest_lifecycle(const common::ObString &path, - bool &is_set_lifecycle); virtual int list_directories(const common::ObString &uri, common::ObBaseDirEntryOperator &op); virtual int is_tagging(const common::ObString &uri, bool &is_tagging); private: @@ -246,7 +244,7 @@ private: const common::ObString &bucket_str, const common::ObString &object_str); bool is_opened_; - ObOssAccount* oss_account_; + common::ObObjectStorageInfo *storage_info_; }; class ObStorageOssAppendWriter : public ObStorageOssBase, public ObIStorageWriter @@ -256,7 +254,7 @@ public: virtual ~ObStorageOssAppendWriter(); public: - int open(const common::ObString &uri, void* oss_base); + int open(const common::ObString &uri, common::ObObjectStorageInfo *storage_info); int write(const char *buf, const int64_t size); int pwrite(const char *buf, const int64_t size, const int64_t offset); int close(); diff --git a/deps/oblib/src/lib/statistic_event/ob_stat_event.h b/deps/oblib/src/lib/statistic_event/ob_stat_event.h index 9c6c78389..0762e5cb5 100644 --- a/deps/oblib/src/lib/statistic_event/ob_stat_event.h +++ b/deps/oblib/src/lib/statistic_event/ob_stat_event.h @@ -293,16 +293,23 @@ STAT_EVENT_ADD_DEF(COS_IO_READ_DELAY, "cos io read delay", ObStatClassIds::STORA STAT_EVENT_ADD_DEF(COS_IO_WRITE_DELAY, "cos io write delay", ObStatClassIds::STORAGE, "cos io write delay", 69015, true, true) STAT_EVENT_ADD_DEF(COS_IO_LS_DELAY, "cos io list delay", ObStatClassIds::STORAGE, "cos io list delay", 69016, true, true) STAT_EVENT_ADD_DEF(BACKUP_DELETE_DELAY, "backup delete delay", ObStatClassIds::STORAGE, "backup delete delay", 69017, true, true) - STAT_EVENT_ADD_DEF(BACKUP_IO_LS_COUNT, "backup io list count", ObStatClassIds::STORAGE, "backup io list count", 69019, true, true) STAT_EVENT_ADD_DEF(BACKUP_IO_READ_FAIL_COUNT, "backup io read failed count", ObStatClassIds::STORAGE, "backup io read failed count", 69020, true, true) STAT_EVENT_ADD_DEF(BACKUP_IO_WRITE_FAIL_COUNT, "backup io write failed count", ObStatClassIds::STORAGE, "backup io write failed count", 69021, true, true) STAT_EVENT_ADD_DEF(BACKUP_IO_DEL_FAIL_COUNT, "backup io delete failed count", ObStatClassIds::STORAGE, "backup io delete failed count", 69022, true, true) STAT_EVENT_ADD_DEF(BACKUP_IO_LS_FAIL_COUNT, "backup io list failed count", ObStatClassIds::STORAGE, "backup io list failed count", 69023, true, true) - STAT_EVENT_ADD_DEF(BACKUP_TAGGING_COUNT, "backup io tagging count", ObStatClassIds::STORAGE, "backup io tagging count", 69024, true, true) STAT_EVENT_ADD_DEF(BACKUP_TAGGING_FAIL_COUNT, "backup io tagging failed count", ObStatClassIds::STORAGE, "backup io tagging count", 69025, true, true) +STAT_EVENT_ADD_DEF(COS_IO_WRITE_BYTES, "cos io write bytes", ObStatClassIds::STORAGE, "cos io write bytes", 69026, true, true) +STAT_EVENT_ADD_DEF(COS_IO_WRITE_COUNT, "cos io write count", ObStatClassIds::STORAGE, "cos io write count", 69027, true, true) +STAT_EVENT_ADD_DEF(COS_DELETE_COUNT, "cos delete count", ObStatClassIds::STORAGE, "cos delete count", 69028, true, true) +STAT_EVENT_ADD_DEF(COS_DELETE_DELAY, "cos delete delay", ObStatClassIds::STORAGE, "cos delete delay", 69029, true, true) +STAT_EVENT_ADD_DEF(COS_IO_LS_LIMIT_COUNT, "cos list io limit count", ObStatClassIds::STORAGE, "cos list io limit count", 69030, true, true) +STAT_EVENT_ADD_DEF(COS_IO_LS_COUNT, "cos io list count", ObStatClassIds::STORAGE, "cos io list count", 69031, true, true) +STAT_EVENT_ADD_DEF(COS_IO_READ_COUNT, "cos io read count", ObStatClassIds::STORAGE, "cos io read count", 69032, true, true) +STAT_EVENT_ADD_DEF(COS_IO_READ_BYTES, "cos io read bytes", ObStatClassIds::STORAGE, "cos io read bytes", 69033, true, true) + // DEBUG STAT_EVENT_ADD_DEF(REFRESH_SCHEMA_COUNT, "refresh schema count", ObStatClassIds::DEBUG, "refresh schema count", 70000, false, true) STAT_EVENT_ADD_DEF(REFRESH_SCHEMA_TIME, "refresh schema time", ObStatClassIds::DEBUG, "refresh schema time", 70001, false, true) diff --git a/deps/oblib/unittest/lib/CMakeLists.txt b/deps/oblib/unittest/lib/CMakeLists.txt index bd36bd785..25da8ebe8 100644 --- a/deps/oblib/unittest/lib/CMakeLists.txt +++ b/deps/oblib/unittest/lib/CMakeLists.txt @@ -85,6 +85,8 @@ oblib_addtest(rc/test_context.cpp) oblib_addtest(resource/test_resource_mgr.cpp) #oblib_addtest(restore/test_storage_file.cpp) #oblib_addtest(restore/test_storage_oss.cpp) +oblib_addtest(restore/test_storage_cos.cpp) +oblib_addtest(restore/test_storage_info.cpp) #oblib_addtest(restore/test_storage.cpp) oblib_addtest(stat/test_di_cache.cpp) oblib_addtest(stat/test_diagnose_info.cpp) diff --git a/deps/oblib/unittest/lib/restore/test_storage_cos.cpp b/deps/oblib/unittest/lib/restore/test_storage_cos.cpp new file mode 100644 index 000000000..60d169f92 --- /dev/null +++ b/deps/oblib/unittest/lib/restore/test_storage_cos.cpp @@ -0,0 +1,681 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#include "lib/utility/ob_test_util.h" +#include "lib/restore/ob_storage.h" +#include "lib/restore/ob_storage_cos_base.h" +#include "lib/allocator/page_arena.h" +#include "test_storage_cos.h" + +using namespace oceanbase::common; + +class TestStorageCosCommon { +public: +TestStorageCosCommon() {} +~TestStorageCosCommon() {} + +void init() +{ + init_cos_env(); + ASSERT_EQ(OB_SUCCESS, + databuff_printf(account, sizeof(account), + "host=%s&access_id=%s&access_key=%s&appid=%s", + endpoint, secretid, secretkey, appid)); + //build cos_base + const ObString cos_storage_info(account); + ASSERT_EQ(OB_SUCCESS, cos_base.set(ObStorageType::OB_STORAGE_COS, cos_storage_info.ptr())); +} +void destory() +{ + fin_cos_env(); +} +protected: + char account[OB_MAX_URI_LENGTH]; + const char *dir_name = "cos_unittest_dir"; + char uri[OB_MAX_URI_LENGTH]; + char dir_uri[OB_MAX_URI_LENGTH]; + ObObjectStorageInfo cos_base; + + int object_prefix_len = 5; +}; + +//use to clean the env +class TestCosCleanOp : public ObBaseDirEntryOperator, public TestStorageCosCommon +{ +public: + TestCosCleanOp() + { + } + ~TestCosCleanOp() {} + int func(const dirent *entry) override; +private : +}; + +int TestCosCleanOp::func(const dirent *entry) +{ + int ret = OB_SUCCESS; + ObStorageCosUtil util; + if (OB_FAIL(util.open(&cos_base))) { + } else if (OB_FAIL(databuff_printf(uri, sizeof(uri), "%s/%s/%s", bucket, dir_name, entry->d_name))) { + } else if (OB_FAIL(util.del_file(uri))) { + } + util.close(); + + return ret; +} + +class TestStorageCos: public ::testing::Test, public TestStorageCosCommon +{ +public: + TestStorageCos() : enable_test_(enable_test) {} + virtual ~TestStorageCos(){} + virtual void SetUp() + { + init(); + } + virtual void TearDown() + { + destory(); + } + + static void SetUpTestCase() + { + } + + static void TearDownTestCase() + { + } + +private: + // disallow copy + DISALLOW_COPY_AND_ASSIGN(TestStorageCos); +protected: + bool enable_test_; +}; + +TEST_F(TestStorageCos, test_append) +{ + int ret = OB_SUCCESS; + if (enable_test_) { + ObStorageAppender appender; + ObStorageReader reader; + ObStorageUtil util; + ASSERT_EQ(OB_SUCCESS, util.open(&cos_base)); + const char *tmp_append_dir = "test_append"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_append_dir, ts)); + + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/test_append_file", dir_uri)); + + // operate before open + char tmp_buf[10] = {0}; + ASSERT_FALSE(appender.is_opened()); + ASSERT_EQ(OB_NOT_INIT, appender.write(tmp_buf, sizeof(tmp_buf))); + ASSERT_EQ(OB_NOT_INIT, appender.pwrite(tmp_buf, sizeof(tmp_buf), 0)); + ASSERT_EQ(OB_NOT_INIT, appender.pwrite(tmp_buf, sizeof(tmp_buf), 0)); + ASSERT_EQ(OB_NOT_INIT, appender.close()); + + // wrong uri + uri[0] = '\0'; + ASSERT_EQ(OB_INVALID_ARGUMENT, appender.open(uri, &cos_base)); + ASSERT_EQ(OB_NOT_INIT, appender.write(tmp_buf, sizeof(tmp_buf))); + ASSERT_EQ(OB_NOT_INIT, appender.pwrite(tmp_buf, sizeof(tmp_buf), 0)); + ASSERT_EQ(OB_NOT_INIT, appender.close()); + + uri[0] = 'a'; + uri[1] = '\0'; + ASSERT_EQ(OB_INVALID_BACKUP_DEST, appender.open(uri, &cos_base)); + ASSERT_EQ(OB_NOT_INIT, appender.write(tmp_buf, sizeof(tmp_buf))); + ASSERT_EQ(OB_NOT_INIT, appender.pwrite(tmp_buf, sizeof(tmp_buf), 0)); + ASSERT_EQ(OB_NOT_INIT, appender.close()); + + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/test_append_file_////", dir_uri)); + ASSERT_EQ(OB_INVALID_ARGUMENT, appender.open(uri, &cos_base)); + ASSERT_EQ(OB_NOT_INIT, appender.write(tmp_buf, sizeof(tmp_buf))); + ASSERT_EQ(OB_NOT_INIT, appender.pwrite(tmp_buf, sizeof(tmp_buf), 0)); + ASSERT_EQ(OB_NOT_INIT, appender.close()); + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/test_append_file", dir_uri)); + ASSERT_EQ(OB_SUCCESS, appender.open(uri, &cos_base)); + ASSERT_EQ(OB_SUCCESS, appender.close()); + + // invalid storage info + ASSERT_EQ(OB_INVALID_ARGUMENT, appender.open(uri, NULL)); + ASSERT_EQ(OB_NOT_INIT, appender.write(tmp_buf, sizeof(tmp_buf))); + ASSERT_EQ(OB_NOT_INIT, appender.pwrite(tmp_buf, sizeof(tmp_buf), 0)); + ASSERT_EQ(OB_NOT_INIT, appender.close()); + + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/test_append_file", dir_uri)); + ASSERT_EQ(OB_SUCCESS, appender.open(uri, &cos_base)); + ASSERT_TRUE(appender.is_opened()); + ASSERT_EQ(0LL, appender.get_length()); + + // first append + const char first_write[] = "123"; + ASSERT_EQ(OB_SUCCESS, appender.pwrite(first_write, strlen(first_write), 0)); + ASSERT_EQ(strlen(first_write), appender.get_length()); + + // second append + const char second_write[] = "4567"; + // repeatable_pwrite returned err code + ASSERT_EQ(OB_BACKUP_PWRITE_CONTENT_NOT_MATCH, + appender.pwrite(second_write, strlen(second_write), strlen(first_write) - 1)); + ASSERT_EQ(OB_BACKUP_PWRITE_OFFSET_NOT_MATCH, + appender.pwrite(second_write, strlen(second_write) - 1, strlen(first_write) + 1)); + ASSERT_EQ(OB_SUCCESS, appender.pwrite(second_write, strlen(second_write), strlen(first_write))); + ASSERT_EQ(strlen(first_write) + strlen(second_write), appender.get_length()); + + // check size + int64_t file_length = 0; + ASSERT_EQ(OB_SUCCESS, util.get_file_length(uri, file_length)); + ASSERT_EQ(strlen(first_write) + strlen(second_write), file_length); + + // check data and clean + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &cos_base)); + ASSERT_EQ(strlen(first_write) + strlen(second_write), reader.get_length()); + char read_buf[5] = {0}; + int64_t read_size = 0; + ASSERT_EQ(OB_SUCCESS, reader.pread(read_buf, 5, 2, read_size)); + ASSERT_EQ('3', read_buf[0]); + ASSERT_EQ('7', read_buf[4]); + ASSERT_EQ(5, read_size); + ASSERT_EQ(OB_SUCCESS, reader.close()); + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + ASSERT_EQ(OB_SUCCESS, appender.close()); + ASSERT_FALSE(appender.is_opened()); + + // append to not appendable object + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/test_append_to_not_appendable_file", dir_uri)); + ASSERT_EQ(OB_SUCCESS, util.write_single_file(uri, first_write, strlen(first_write))); + ASSERT_EQ(OB_SUCCESS, appender.open(uri, &cos_base)); + ASSERT_EQ(OB_CLOUD_OBJECT_NOT_APPENDABLE, appender.pwrite(second_write, strlen(second_write), strlen(first_write))); + ASSERT_EQ(OB_SUCCESS, appender.close()); + ASSERT_FALSE(appender.is_opened()); + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + + // operate after close + ASSERT_EQ(OB_NOT_INIT, appender.write(tmp_buf, sizeof(tmp_buf))); + ASSERT_EQ(OB_NOT_INIT, appender.pwrite(tmp_buf, sizeof(tmp_buf), 0)); + ASSERT_EQ(OB_NOT_INIT, appender.close()); + + util.close(); + } +} + +#define WRITE_SINGLE_FILE(file_suffix, file_content) \ + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%stest_write_file_%ld", dir_uri, file_suffix)); \ + ASSERT_EQ(OB_SUCCESS, writer.open(uri, &cos_base)); \ + const char write_content[] = file_content; \ + ASSERT_EQ(OB_SUCCESS, writer.write(write_content, strlen(write_content))); \ + ASSERT_EQ(OB_SUCCESS, writer.close()); \ + +TEST_F(TestStorageCos, test_basic_rw) +{ + int ret = OB_SUCCESS; + if (enable_test_) { + ObStorageWriter writer; + ObStorageReader reader; + ObStorageUtil util; + ASSERT_EQ(OB_SUCCESS, util.open(&cos_base)); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/", bucket, dir_name)); + + // operate before open + char tmp_buf[10] = {0}; + int64_t tmp_read_size = 0; + ASSERT_EQ(OB_NOT_INIT, writer.write(tmp_buf, sizeof(tmp_buf))); + ASSERT_EQ(OB_NOT_INIT, writer.close()); + ASSERT_EQ(OB_NOT_INIT, reader.pread(tmp_buf, sizeof(tmp_buf), 0, tmp_read_size)); + ASSERT_EQ(OB_NOT_INIT, reader.close()); + + // wrong uri + uri[0] = '\0'; + ASSERT_EQ(OB_INVALID_BACKUP_DEST, writer.open(uri, NULL)); + ASSERT_EQ(OB_NOT_INIT, writer.write(tmp_buf, sizeof(tmp_buf))); + ASSERT_EQ(OB_NOT_INIT, writer.close()); + ASSERT_EQ(OB_INVALID_BACKUP_DEST, reader.open(uri, NULL)); + ASSERT_EQ(OB_NOT_INIT, reader.pread(tmp_buf, sizeof(tmp_buf), 0, tmp_read_size)); + ASSERT_EQ(OB_NOT_INIT, reader.close()); + + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%stest_write_file_////", dir_uri)); + ASSERT_EQ(OB_INVALID_ARGUMENT, writer.open(uri, &cos_base)); + ASSERT_EQ(OB_NOT_INIT, writer.write(tmp_buf, sizeof(tmp_buf))); + ASSERT_EQ(OB_NOT_INIT, writer.close()); + ASSERT_EQ(OB_INVALID_ARGUMENT, reader.open(uri, &cos_base)); + ASSERT_EQ(OB_NOT_INIT, reader.pread(tmp_buf, sizeof(tmp_buf), 0, tmp_read_size)); + ASSERT_EQ(OB_NOT_INIT, reader.close()); + + const int64_t tmp_ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%stest_write_file_%ld", dir_uri, tmp_ts)); + ASSERT_EQ(OB_SUCCESS, writer.open(uri, &cos_base)); + ASSERT_EQ(OB_SUCCESS, writer.close()); + + // invalid storage info + ASSERT_EQ(OB_INVALID_ARGUMENT, writer.open(uri, NULL)); + ASSERT_EQ(OB_NOT_INIT, writer.write(tmp_buf, sizeof(tmp_buf))); + ASSERT_EQ(OB_NOT_INIT, writer.close()); + ASSERT_EQ(OB_INVALID_ARGUMENT, reader.open(uri, NULL)); + ASSERT_EQ(OB_NOT_INIT, reader.pread(tmp_buf, sizeof(tmp_buf), 0, tmp_read_size)); + ASSERT_EQ(OB_NOT_INIT, reader.close()); + + { + // rw empty object + const int64_t ts = ObTimeUtility::current_time(); + WRITE_SINGLE_FILE(ts, ""); + + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &cos_base)); + ASSERT_EQ(strlen(write_content), reader.get_length()); + ASSERT_EQ(OB_SUCCESS, reader.close()); + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + } + + { + // ObStorageWriter writer; + const int64_t ts = ObTimeUtility::current_time(); + WRITE_SINGLE_FILE(ts, "123456789"); + + // ObStorageReader reader; + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &cos_base)); + ASSERT_EQ(strlen(write_content), reader.get_length()); + char read_buf[7] = {0}; + int64_t read_size = 0; + ASSERT_EQ(OB_SUCCESS, reader.pread(read_buf, 7, 2, read_size)); + ASSERT_EQ('3', read_buf[0]); + ASSERT_EQ('9', read_buf[6]); + ASSERT_EQ(7, read_size); + ASSERT_EQ(OB_SUCCESS, reader.close()); + + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + } + + { + // ObStorageWriter writer; + const int64_t ts = ObTimeUtility::current_time(); + WRITE_SINGLE_FILE(ts, "123456789ABCDEF"); + + // ObStorageReader reader; + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &cos_base)); + ASSERT_EQ(strlen(write_content), reader.get_length()); + char read_buf[10] = {0}; + int64_t read_size = 0; + ASSERT_EQ(OB_SUCCESS, reader.pread(read_buf, 10, 9, read_size)); + ASSERT_EQ('A', read_buf[0]); + ASSERT_EQ('F', read_buf[5]); + ASSERT_EQ(6, read_size); + ASSERT_EQ(OB_SUCCESS, reader.close()); + + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + } + + { + // read not exist object + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%snot_exist_%ld", dir_uri, ts)); + ASSERT_EQ(OB_BACKUP_FILE_NOT_EXIST, reader.open(uri, &cos_base)); + ASSERT_EQ(OB_NOT_INIT, reader.close()); + + // open after fail + WRITE_SINGLE_FILE(ts, "123456789ABCDEF"); + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &cos_base)); + ASSERT_EQ(OB_SUCCESS, reader.close()); + } + + // open twice + ASSERT_EQ(OB_SUCCESS, writer.open(uri, &cos_base)); + ASSERT_EQ(OB_INIT_TWICE, writer.open(uri, &cos_base)); + ASSERT_EQ(OB_NOT_INIT, writer.close()); // reader/writer will be closed if init twice + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &cos_base)); + ASSERT_EQ(OB_INIT_TWICE, reader.open(uri, &cos_base)); + ASSERT_EQ(OB_NOT_INIT, reader.close()); + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + + // operate after close + ASSERT_EQ(OB_NOT_INIT, writer.write(tmp_buf, sizeof(tmp_buf))); + ASSERT_EQ(OB_NOT_INIT, writer.close()); + ASSERT_EQ(OB_NOT_INIT, reader.pread(tmp_buf, sizeof(tmp_buf), 0, tmp_read_size)); + ASSERT_EQ(OB_NOT_INIT, reader.close()); + + util.close(); + } +} + +TEST_F(TestStorageCos, test_util) +{ + int ret = OB_SUCCESS; + if (enable_test_) { + ObStorageUtil util; + const char *tmp_util_dir = "test_util"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld/", + bucket, dir_name, tmp_util_dir, ts)); + ASSERT_EQ(OB_SUCCESS, util.open(&cos_base)); + + ObStorageWriter writer; + { + WRITE_SINGLE_FILE(1L, "123456789ABC"); + + bool is_obj_exist = false; + ASSERT_EQ(OB_SUCCESS, util.is_exist(uri, is_obj_exist)); + ASSERT_TRUE(is_obj_exist); + + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + ASSERT_EQ(OB_SUCCESS, util.is_exist(uri, is_obj_exist)); + ASSERT_FALSE(is_obj_exist); + } + { + WRITE_SINGLE_FILE(2L, "123456789ABCDEF"); + + int64_t file_length = 0; + ASSERT_EQ(OB_SUCCESS, util.get_file_length(uri, file_length)); + ASSERT_EQ(strlen(write_content), file_length); + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + } + + // open twice + ASSERT_EQ(OB_INIT_TWICE, util.open(&cos_base)); + util.close(); + + // invalid storage info + ASSERT_EQ(OB_INVALID_ARGUMENT, util.open(NULL)); + } +} + +TEST_F(TestStorageCos, test_util_write_single_file) +{ + int ret = OB_SUCCESS; + if (enable_test_) { + ObStorageReader reader; + ObStorageUtil util; + const char *tmp_util_dir = "test_util_write_single_file"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_util_dir, ts)); + ASSERT_EQ(OB_SUCCESS, util.open(&cos_base)); + + // write data + const char *write_content = "123456789ABCDEF"; + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/0", dir_uri)); + ASSERT_EQ(OB_SUCCESS, util.write_single_file(uri, write_content, strlen(write_content))); + + // check data + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &cos_base)); + ASSERT_EQ(strlen(write_content), reader.get_length()); + char read_buf[10] = {0}; + int64_t read_size = 0; + ASSERT_EQ(OB_SUCCESS, reader.pread(read_buf, 10, 9, read_size)); + ASSERT_EQ('A', read_buf[0]); + ASSERT_EQ('F', read_buf[5]); + ASSERT_EQ(6, read_size); + ASSERT_EQ(OB_SUCCESS, reader.close()); + + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + + // wrong uri + uri[0] = '\0'; + ASSERT_EQ(OB_INVALID_BACKUP_DEST, util.write_single_file(uri, write_content, strlen(write_content))); + + util.close(); + } +} + +class TestCosListOp : public ObBaseDirEntryOperator +{ +public: + TestCosListOp() {} + virtual ~TestCosListOp() {} + int func(const dirent *entry) override; + + ObArray object_names_; +private : + ObArenaAllocator allocator_; +}; + +int TestCosListOp::func(const dirent *entry) +{ + int ret = OB_SUCCESS; + char *tmp_object_name = (char *)allocator_.alloc(strlen(entry->d_name) + 1); + if (OB_ISNULL(tmp_object_name)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN,"fail to alloc buf for tmp object name", K(ret)); + } else { + STRCPY(tmp_object_name, entry->d_name); + object_names_.push_back(tmp_object_name); + } + return ret; +} + +#define UTIL_WRITE_FILES(format, object_prefix_len, file_num, file_content) \ + for (int64_t file_idx = 0; file_idx < file_num; file_idx++) { \ + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), format, \ + dir_uri, object_prefix_len, file_idx, file_idx)); \ + ASSERT_EQ(OB_SUCCESS, util.write_single_file(uri, file_content, strlen(file_content))); \ + } \ + +#define UTIL_DELETE_FILES(format, object_prefix_len, file_num) \ + for (int64_t file_idx = 0; file_idx < file_num; file_idx++) { \ + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), format, \ + dir_uri, object_prefix_len, file_idx, file_idx)); \ + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); \ + } \ + +TEST_F(TestStorageCos, test_util_list_files) +{ + int ret = OB_SUCCESS; + if (enable_test_) { + ObStorageReader reader; + ObStorageUtil util; + const char *tmp_util_dir = "test_util_list_files"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_util_dir, ts)); + ASSERT_EQ(OB_SUCCESS, util.open(&cos_base)); + + { + // wrong uri + uri[0] = '\0'; + TestCosListOp op; + ASSERT_EQ(OB_INVALID_BACKUP_DEST, util.list_files(uri, op)); + } + + int64_t file_num = 1001; + const char *write_content = "0123456789"; + + // list objects + { + const char *format = "%s/%0*ld_%ld"; + UTIL_WRITE_FILES(format, object_prefix_len, file_num, write_content); + + // list and check and clean + TestCosListOp op; + ASSERT_EQ(OB_SUCCESS, util.list_files(dir_uri, op)); + ASSERT_EQ(file_num, op.object_names_.size()); + for (int64_t i = 0; i < file_num; i++) { + // listed files do not contain prefix + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%0*ld_%ld", object_prefix_len, i, i)); + ASSERT_STREQ(uri, op.object_names_[i]); + } + UTIL_DELETE_FILES(format, object_prefix_len, file_num); + } + + // list subfolders' objects + { + const char *format = "%s/%0*ld/%ld"; + UTIL_WRITE_FILES(format, object_prefix_len, file_num, write_content); + + // list and check and clean + TestCosListOp op; + ASSERT_EQ(OB_SUCCESS, util.list_files(dir_uri, op)); + ASSERT_EQ(file_num, op.object_names_.size()); + for (int64_t i = 0; i < file_num; i++) { + // listed files do not contain prefix + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%0*ld/%ld", object_prefix_len, i, i)); + ASSERT_STREQ(uri, op.object_names_[i]); + } + UTIL_DELETE_FILES(format, object_prefix_len, file_num); + } + + // list empty dir, now dir_uri should be empty after delete + TestCosListOp list_empty_op; + ASSERT_EQ(OB_SUCCESS, util.list_files(dir_uri, list_empty_op)); + ASSERT_EQ(0, list_empty_op.object_names_.size()); + + util.close(); + } +} + +TEST_F(TestStorageCos, test_util_list_directories) +{ + int ret = OB_SUCCESS; + if (enable_test_) { + ObStorageReader reader; + ObStorageUtil util; + const char *tmp_util_dir = "test_util_list_directories"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_util_dir, ts)); + ASSERT_EQ(OB_SUCCESS, util.open(&cos_base)); + + { + // wrong uri + uri[0] = '\0'; + TestCosListOp op; + ASSERT_EQ(OB_INVALID_ARGUMENT, util.list_directories(uri, op)); + } + + int64_t file_num = 1001; + const char *write_content = "0123456789"; + + { + const char *format = "%s/%0*ld/%ld"; + UTIL_WRITE_FILES(format, object_prefix_len, file_num, write_content); + + // list and check and clean + TestCosListOp op; + ASSERT_EQ(OB_SUCCESS, util.list_directories(dir_uri, op)); + ASSERT_EQ(file_num, op.object_names_.size()); + for (int64_t i = 0; i < file_num; i++) { + // listed directories do not contain prefix + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%0*ld", object_prefix_len, i)); + ASSERT_STREQ(uri, op.object_names_[i]); + } + UTIL_DELETE_FILES(format, object_prefix_len, file_num); + } + + // no sub directories + { + const char *format = "%s/%0*ld_%ld"; + UTIL_WRITE_FILES(format, object_prefix_len, file_num, write_content); + + // list and check and clean + TestCosListOp op; + ASSERT_EQ(OB_SUCCESS, util.list_directories(dir_uri, op)); + ASSERT_EQ(0, op.object_names_.size()); + UTIL_DELETE_FILES(format, object_prefix_len, file_num); + } + + util.close(); + } +} + +TEST_F(TestStorageCos, test_util_is_tagging) +{ + int ret = OB_SUCCESS; + if (enable_test_) { + ObStorageUtil util; + const char *tmp_util_dir = "test_util_is_tagging"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_util_dir, ts)); + + bool is_tagging = true; + char tmp_account[OB_MAX_URI_LENGTH]; + ObObjectStorageInfo tmp_cos_base; + const char *write_content = "123456789ABCDEF"; + + // wrong tag mode + ASSERT_EQ(OB_SUCCESS, + databuff_printf(tmp_account, sizeof(tmp_account), + "host=%s&access_id=%s&access_key=%s&appid=%s&delete_mode=tag", + endpoint, secretid, secretkey, appid)); + ASSERT_EQ(OB_INVALID_ARGUMENT, tmp_cos_base.set(ObStorageType::OB_STORAGE_COS, tmp_account)); + tmp_cos_base.reset(); + + ASSERT_EQ(OB_SUCCESS, + databuff_printf(tmp_account, sizeof(tmp_account), + "host=%s&access_id=%s&access_key=%s&appid=%s&delete_mode=delete_delete", + endpoint, secretid, secretkey, appid)); + ASSERT_EQ(OB_INVALID_ARGUMENT, tmp_cos_base.set(ObStorageType::OB_STORAGE_COS, tmp_account)); + tmp_cos_base.reset(); + + // delete mode + ASSERT_EQ(OB_SUCCESS, + databuff_printf(tmp_account, sizeof(tmp_account), + "host=%s&access_id=%s&access_key=%s&appid=%s&delete_mode=delete", + endpoint, secretid, secretkey, appid)); + ASSERT_EQ(OB_SUCCESS, tmp_cos_base.set(ObStorageType::OB_STORAGE_COS, tmp_account)); + + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/delete_mode", dir_uri)); + ASSERT_EQ(OB_SUCCESS, util.open(&tmp_cos_base)); + ASSERT_EQ(OB_SUCCESS, util.write_single_file(uri, write_content, strlen(write_content))); + + ASSERT_EQ(OB_SUCCESS, util.is_tagging(uri, is_tagging)); + ASSERT_FALSE(is_tagging); + + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + ASSERT_EQ(OB_BACKUP_FILE_NOT_EXIST, util.is_tagging(uri, is_tagging)); + tmp_cos_base.reset(); + util.close(); + + // tagging mode + ASSERT_EQ(OB_SUCCESS, + databuff_printf(tmp_account, sizeof(tmp_account), + "host=%s&access_id=%s&access_key=%s&appid=%s&delete_mode=tagging", + endpoint, secretid, secretkey, appid)); + ASSERT_EQ(OB_SUCCESS, tmp_cos_base.set(ObStorageType::OB_STORAGE_COS, tmp_account)); + + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/tagging_mode", dir_uri)); + ASSERT_EQ(OB_SUCCESS, util.open(&tmp_cos_base)); + ASSERT_EQ(OB_SUCCESS, util.write_single_file(uri, write_content, strlen(write_content))); + + is_tagging = true; + ASSERT_EQ(OB_SUCCESS, util.is_tagging(uri, is_tagging)); + ASSERT_FALSE(is_tagging); + + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + ASSERT_EQ(OB_SUCCESS, util.is_tagging(uri, is_tagging)); + ASSERT_TRUE(is_tagging); + tmp_cos_base.reset(); + util.close(); + + // clean + ASSERT_EQ(OB_SUCCESS, + databuff_printf(tmp_account, sizeof(tmp_account), + "host=%s&access_id=%s&access_key=%s&appid=%s", + endpoint, secretid, secretkey, appid)); + ASSERT_EQ(OB_SUCCESS, tmp_cos_base.set(ObStorageType::OB_STORAGE_COS, tmp_account)); + + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/tagging_mode", dir_uri)); + ASSERT_EQ(OB_SUCCESS, util.open(&tmp_cos_base)); + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + ASSERT_EQ(OB_BACKUP_FILE_NOT_EXIST, util.is_tagging(uri, is_tagging)); + util.close(); + } +} + +int main(int argc, char **argv) +{ + system("rm -f test_storage_cos.log*"); + OB_LOGGER.set_file_name("test_storage_cos.log", true, true); + OB_LOGGER.set_log_level("INFO"); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/deps/oblib/unittest/lib/restore/test_storage_cos.h b/deps/oblib/unittest/lib/restore/test_storage_cos.h new file mode 100644 index 000000000..6f3ad9322 --- /dev/null +++ b/deps/oblib/unittest/lib/restore/test_storage_cos.h @@ -0,0 +1,23 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef TEST_STORAGE_COS_H_ +#define TEST_STORAGE_COS_H_ + +const bool enable_test = false; +const char *bucket = "cos://xxx"; +const char *endpoint = "xxx"; +const char *secretid = "xxx"; +const char *secretkey = "xxx"; +const char *appid = "xxx"; + +#endif \ No newline at end of file diff --git a/deps/oblib/unittest/lib/restore/test_storage_info.cpp b/deps/oblib/unittest/lib/restore/test_storage_info.cpp new file mode 100644 index 000000000..6ddbf50bb --- /dev/null +++ b/deps/oblib/unittest/lib/restore/test_storage_info.cpp @@ -0,0 +1,114 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#include "lib/utility/ob_test_util.h" +#include "lib/restore/ob_storage_info.h" + +using namespace oceanbase::common; + +TEST(ObObjectStorageInfo, file) +{ + const char *uri = "file:///backup_dir/?&delete_mode=tagging"; + ObObjectStorageInfo info1; + ASSERT_FALSE(info1.is_valid()); + + // wrong storage info + const char *storage_info = NULL; + ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info)); + storage_info = "access_key=xxx"; // nfs, endpoint/access_id/access_key should be empty + ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info)); + storage_info = "access_key=xxx&encrypt_key=xxx"; + ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info)); + storage_info = "delete_mode=wrong_mode"; + ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info)); + storage_info = "delete_mode=delete"; + ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info)); + + ObObjectStorageInfo info2; + ObObjectStorageInfo info3; + storage_info = ""; + ObStorageType device_type = OB_STORAGE_FILE; + ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info)); + ASSERT_EQ(OB_SUCCESS, info2.set(device_type, storage_info)); + ASSERT_EQ(OB_SUCCESS, info3.assign(info1)); + + ASSERT_EQ(info1, info2); + ASSERT_EQ(info1, info3); + + // get + ASSERT_EQ(device_type, info1.get_type()); + + char buf[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 }; + ASSERT_EQ(OB_SUCCESS, info1.get_storage_info_str(buf, sizeof(buf))); + ASSERT_STREQ("", buf); + ASSERT_EQ(OB_SUCCESS, info1.get_storage_info_str(buf, sizeof(buf))); + ASSERT_STREQ("", buf); + + // clean + ASSERT_TRUE(info1.is_valid()); + info1.reset(); + ASSERT_FALSE(info1.is_valid()); +} + +TEST(ObObjectStorageInfo, oss) +{ + const char *uri = "oss://backup_dir?host=xxx.com&access_id=111&access_key=222"; + ObObjectStorageInfo info1; + + const char *storage_info = ""; + ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info)); + storage_info = "access_id=111&access_key=222"; + ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info)); + + storage_info = "host=xxx.com&access_id=111&access_key=222"; + ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info)); + + char buf[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 }; + ASSERT_EQ(OB_SUCCESS, info1.get_storage_info_str(buf, sizeof(buf))); + ASSERT_STREQ(storage_info, buf); + + storage_info = "host=xxx.com&access_id=111&access_key=222&delete_mode=delete"; + info1.reset(); + ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info)); +} + +TEST(ObObjectStorageInfo, cos) +{ + const char *uri = "cos://backup_dir?host=xxx.com&access_id=111&access_key=222&appid=333"; + ObObjectStorageInfo info1; + + const char *storage_info = ""; + ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info)); + storage_info = "host=xxx.com&access_id=111&access_key=222"; + ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info)); + + storage_info = "host=xxx.com&access_id=111&access_key=222&appid=333"; + ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info)); + + char buf[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 }; + ASSERT_EQ(OB_SUCCESS, info1.get_storage_info_str(buf, sizeof(buf))); + ASSERT_STREQ(storage_info, buf); + + storage_info = "host=xxx.com&access_id=111&access_key=222&appid=333&delete_mode=delete"; + info1.reset(); + ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info)); +} + +int main(int argc, char **argv) +{ + system("rm -f test_storage_info.log*"); + OB_LOGGER.set_file_name("test_storage_info.log", true, true); + OB_LOGGER.set_log_level("INFO"); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/src/logservice/archiveservice/ob_archive_sender.cpp b/src/logservice/archiveservice/ob_archive_sender.cpp index 31f277263..ee2477cf9 100644 --- a/src/logservice/archiveservice/ob_archive_sender.cpp +++ b/src/logservice/archiveservice/ob_archive_sender.cpp @@ -793,7 +793,8 @@ bool ObArchiveSender::is_retry_ret_code_(const int ret_code) const return is_io_error(ret_code) || OB_ALLOCATE_MEMORY_FAILED == ret_code || OB_BACKUP_DEVICE_OUT_OF_SPACE == ret_code - || OB_BACKUP_PWRITE_OFFSET_NOT_MATCH == ret_code; + || OB_BACKUP_PWRITE_OFFSET_NOT_MATCH == ret_code + || OB_IO_LIMIT == ret_code; } bool ObArchiveSender::is_ignore_ret_code_(const int ret_code) const diff --git a/src/logservice/ob_log_external_storage_io_task.h b/src/logservice/ob_log_external_storage_io_task.h index a6dfe7265..8b688c19f 100644 --- a/src/logservice/ob_log_external_storage_io_task.h +++ b/src/logservice/ob_log_external_storage_io_task.h @@ -162,7 +162,7 @@ protected: ObString uri_ob_str_; char uri_str_[OB_MAX_URI_LENGTH]; ObString storage_info_ob_str_; - char storage_info_str_[share::OB_MAX_BACKUP_STORAGE_INFO_LENGTH]; + char storage_info_str_[common::OB_MAX_BACKUP_STORAGE_INFO_LENGTH]; int64_t generate_ts_; int64_t do_task_ts_; RunningStatus *running_status_; diff --git a/src/logservice/restoreservice/ob_remote_data_generator.cpp b/src/logservice/restoreservice/ob_remote_data_generator.cpp index 5e9634823..87c8650fe 100644 --- a/src/logservice/restoreservice/ob_remote_data_generator.cpp +++ b/src/logservice/restoreservice/ob_remote_data_generator.cpp @@ -472,7 +472,10 @@ int LocationDataGenerator::read_file_(const ObString &base, ObString uri(path.get_obstr()); char storage_info_cstr[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = {'\0'}; int64_t real_size = 0; - if (OB_FAIL(storage_info->get_storage_info_str(storage_info_cstr, OB_MAX_BACKUP_STORAGE_INFO_LENGTH, false))) { + common::ObObjectStorageInfo storage_info_base; + if (OB_FAIL(storage_info_base.assign(*storage_info))) { + OB_LOG(WARN, "fail to assign storage info base!", K(ret), KP(storage_info)); + } else if (OB_FAIL(storage_info_base.get_storage_info_str(storage_info_cstr, OB_MAX_BACKUP_STORAGE_INFO_LENGTH))) { LOG_WARN("get_storage_info_str failed", K(ret), K(uri), K(storage_info)); } else { ObString storage_info_ob_str(storage_info_cstr); diff --git a/src/share/backup/ob_backup_io_adapter.cpp b/src/share/backup/ob_backup_io_adapter.cpp index e403ddc84..808a4879b 100644 --- a/src/share/backup/ob_backup_io_adapter.cpp +++ b/src/share/backup/ob_backup_io_adapter.cpp @@ -86,16 +86,20 @@ int ObBackupIoAdapter::get_and_init_device(ObIODevice*& dev_handle, const common::ObString &storage_type_prefix) { int ret = OB_SUCCESS; - char storage_info_str[share::OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 }; + char storage_info_str[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 }; ObIODOpts opts; ObIODOpt opt; //only one para opts.opts_ = &(opt); opts.opt_cnt_ = 1; opt.key_ = "storage_info"; + common::ObObjectStorageInfo storage_info_base; if (NULL == storage_info) { ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "storage info is null", K(ret)); - } else if (OB_FAIL(storage_info->get_storage_info_str(storage_info_str, sizeof(storage_info_str), false/*no need encrypt*/))) { + } else if (OB_FAIL(storage_info_base.assign(*storage_info))) { + OB_LOG(WARN, "fail to assign storage info base!", K(ret), KP(storage_info)); + } else if (OB_FAIL(storage_info_base.get_storage_info_str(storage_info_str, sizeof(storage_info_str)))) { + // no need encrypt OB_LOG(WARN, "fail to get storage info str!", K(ret), KP(storage_info)); } else if (FALSE_IT(opt.value_.value_str = storage_info_str)) { } else if (OB_FAIL(ObDeviceManager::get_instance().get_device(storage_info_str, storage_type_prefix, dev_handle))) { @@ -603,34 +607,6 @@ int ObBackupIoAdapter::is_empty_directory(const common::ObString &uri, return ret; } -int ObBackupIoAdapter::check_backup_dest_lifecycle(const common::ObString &path, - const share::ObBackupStorageInfo *storage_info, - bool &is_set_lifecycle) -{ - int ret = OB_SUCCESS; - is_set_lifecycle = false; - ObIODevice* device_handle = NULL; - if (OB_FAIL(get_and_init_device(device_handle, storage_info, path))) { - OB_LOG(WARN, "fail to get device!", K(ret)); - } else { - ObIODOpts opts; - ObIODOpt opt[2]; - - opts.opts_ = opt; - opts.opt_cnt_ = 2; - opt[0].set("get_bucket_lifecycle", false); - opt[1].set("path", path.ptr()); - - if (OB_FAIL(device_handle->get_config(opts))) { - OB_LOG(WARN, "fail to get config for backup lifecycle!", K(ret), K(path), KP(storage_info)); - } else { - is_set_lifecycle = opt[0].value_.value_bool; - } - } - release_device(device_handle); - return ret; -} - int ObBackupIoAdapter::set_access_type(ObIODOpts* opts, bool is_appender, int max_opt_num) { int ret = OB_SUCCESS; diff --git a/src/share/backup/ob_backup_io_adapter.h b/src/share/backup/ob_backup_io_adapter.h index 7ce431c6f..b0e858b62 100644 --- a/src/share/backup/ob_backup_io_adapter.h +++ b/src/share/backup/ob_backup_io_adapter.h @@ -54,9 +54,6 @@ public: int is_empty_directory(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, bool &is_empty_directory); - int check_backup_dest_lifecycle(const common::ObString &path, - const share::ObBackupStorageInfo *storage_info, - bool &is_set_lifecycle); int open_with_access_type(ObIODevice*& device_handle, ObIOFd &fd, const share::ObBackupStorageInfo *storage_info, const common::ObString &uri, ObStorageAccessType access_type); diff --git a/src/share/backup/ob_backup_path.cpp b/src/share/backup/ob_backup_path.cpp index 206802113..0b04d4445 100755 --- a/src/share/backup/ob_backup_path.cpp +++ b/src/share/backup/ob_backup_path.cpp @@ -1292,7 +1292,7 @@ int ObBackupPathUtil::construct_backup_set_dest(const share::ObBackupDest &backu ret = OB_ERR_UNEXPECTED; LOG_WARN("storage info must not be nullptr", K(ret)); } else if (OB_FAIL(storage_info->get_storage_info_str( - storage_info_buf, OB_MAX_BACKUP_STORAGE_INFO_LENGTH, true/*need_encrypt*/))) { + storage_info_buf, OB_MAX_BACKUP_STORAGE_INFO_LENGTH))) { LOG_WARN("fail to get storage info", K(ret)); } else if (OB_FAIL(backup_set_dest.set(path.get_ptr(), storage_info_buf))) { LOG_WARN("fail to set backup set dest", K(ret), K(path), K(storage_info_buf)); @@ -1324,7 +1324,7 @@ int ObBackupPathUtil::construct_backup_complement_log_dest(const share::ObBackup ret = OB_ERR_UNEXPECTED; LOG_WARN("storage info must not be nullptr", K(ret)); } else if (OB_FAIL(storage_info->get_storage_info_str( - storage_info_buf, OB_MAX_BACKUP_STORAGE_INFO_LENGTH, true/*need_encrypt*/))) { + storage_info_buf, OB_MAX_BACKUP_STORAGE_INFO_LENGTH))) { LOG_WARN("fail to get storage info", K(ret)); } else if (OB_FAIL(backup_set_dest.set(path.get_ptr(), storage_info_buf))) { LOG_WARN("fail to set backup set dest", K(ret), K(path), K(storage_info_buf)); @@ -1353,7 +1353,7 @@ int ObBackupPathUtil::construct_backup_complement_log_dest(const share::ObBackup ret = OB_ERR_UNEXPECTED; LOG_WARN("storage info must not be nullptr", K(ret)); } else if (OB_FAIL(storage_info->get_storage_info_str( - storage_info_buf, OB_MAX_BACKUP_STORAGE_INFO_LENGTH, true/*need_encrypt*/))) { + storage_info_buf, OB_MAX_BACKUP_STORAGE_INFO_LENGTH))) { LOG_WARN("fail to get storage info", K(ret)); } else if (OB_FAIL(backup_set_dest.set(path.get_ptr(), storage_info_buf))) { LOG_WARN("fail to set backup set dest", K(ret), K(path), K(storage_info_buf)); diff --git a/src/share/backup/ob_backup_struct.cpp b/src/share/backup/ob_backup_struct.cpp index 5c67bec5f..a3888ed23 100755 --- a/src/share/backup/ob_backup_struct.cpp +++ b/src/share/backup/ob_backup_struct.cpp @@ -1102,39 +1102,11 @@ int ObLogArchiveBackupInfo::get_backup_dest(ObBackupDest &backup_dest) const } //***********************ObBackupStorageInfo*************************** -ObBackupStorageInfo::ObBackupStorageInfo() - : device_type_(ObStorageType::OB_STORAGE_MAX_TYPE) -{ - endpoint_[0] = '\0'; - access_id_[0] = '\0'; - access_key_[0] = '\0'; - extension_[0] = '\0'; -} - ObBackupStorageInfo::~ObBackupStorageInfo() { reset(); } -void ObBackupStorageInfo::reset() -{ - device_type_ = ObStorageType::OB_STORAGE_MAX_TYPE; - endpoint_[0] = '\0'; - access_id_[0] = '\0'; - access_key_[0] = '\0'; - extension_[0] = '\0'; -} - -bool ObBackupStorageInfo::is_valid() const -{ - return device_type_ >= 0 && device_type_ < ObStorageType::OB_STORAGE_MAX_TYPE; -} - -const char *ObBackupStorageInfo::get_type_str() const -{ - return get_storage_type_str(device_type_); -} - int ObBackupStorageInfo::set( const common::ObStorageType device_type, const char *endpoint, @@ -1142,98 +1114,77 @@ int ObBackupStorageInfo::set( const char *extension) { int ret = OB_SUCCESS; + char storage_info[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 }; if (is_valid()) { ret = OB_INIT_TWICE; LOG_WARN("storage info init twice", K(ret)); - } else if (OB_ISNULL(endpoint) || OB_ISNULL(authorization) || OB_ISNULL(extension) || OB_STORAGE_MAX_TYPE == device_type) { + } else if (OB_ISNULL(endpoint) + || OB_ISNULL(authorization) || OB_ISNULL(extension) || OB_STORAGE_MAX_TYPE == device_type) { ret = OB_INVALID_BACKUP_DEST; LOG_WARN("invalid args", K(ret), KP(endpoint), KP(authorization), KP(extension), K(device_type)); - } else if (FALSE_IT(device_type_ = device_type)) { - } else if (0 != strlen(endpoint) && OB_FAIL(databuff_printf(endpoint_, sizeof(endpoint_), "%s", endpoint))) { - LOG_WARN("failed to set endpoint", K(ret)); - } else if (0 != strlen(authorization) && OB_FAIL(parse_authorization_(authorization))) { - LOG_WARN("failed to parse authorization", K(ret)); - } else if (0 != strlen(extension) && OB_FAIL(databuff_printf(extension_, sizeof(extension_), "%s", extension))) { - LOG_WARN("failed to set extension", K(ret)); - } - - if (OB_FAIL(ret)) { - } else if (ObStorageType::OB_STORAGE_FILE != device_type_ && (0 == strlen(endpoint_) || 0 == strlen(access_id_) || 0 == strlen(access_key_))) { - ret = OB_INVALID_BACKUP_DEST; - LOG_WARN("backup device is not nfs, endpoint/access_id/access_key do not allow to be empty", K(ret), K_(device_type), K_(endpoint), K_(access_id)); - } else if (ObStorageType::OB_STORAGE_FILE == device_type_ && (0 != strlen(endpoint_) || 0 != strlen(access_id_) || 0 != strlen(access_key_))) { - ret = OB_INVALID_BACKUP_DEST; - LOG_WARN("backup device is nfs, endpoint/access_id/access_key not be empty", K(ret), K_(device_type), K_(endpoint), K_(access_id)); + } else if (0 != strlen(endpoint) + && OB_FAIL(set_storage_info_field_(endpoint, storage_info, sizeof(storage_info)))) { + LOG_WARN("failed to set storage info", K(ret)); + } else if (0 != strlen(authorization) + && OB_FAIL(set_storage_info_field_(authorization, storage_info, sizeof(storage_info)))) { + LOG_WARN("failed to set storage info", K(ret)); + } else if (0 != strlen(extension) + && OB_FAIL(set_storage_info_field_(extension, storage_info, sizeof(storage_info)))) { + LOG_WARN("failed to set storage info", K(ret)); + } else if (OB_FAIL(set(device_type, storage_info))) { + LOG_WARN("failed to set storage info", K(ret), KPC(this)); } return ret; } -int ObBackupStorageInfo::set_access_key_(const char *buf, const bool need_decrypt) +int ObBackupStorageInfo::get_authorization_info(char *authorization, const int64_t length) const { int ret = OB_SUCCESS; - if (OB_ISNULL(buf)) { + const int64_t key_len = MAX(OB_MAX_BACKUP_SERIALIZEKEY_LENGTH, OB_MAX_BACKUP_ACCESSKEY_LENGTH); + char access_key_buf[key_len] = { 0 }; + STATIC_ASSERT(OB_MAX_BACKUP_AUTHORIZATION_LENGTH > (OB_MAX_BACKUP_ACCESSID_LENGTH + key_len), "array length overflow"); + if (!is_valid()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("buf is null", K(ret), K(need_decrypt)); - } else if (0 != strlen(access_key_)) { + LOG_WARN("storage info not init", K(ret)); + } else if (OB_ISNULL(authorization) || length <= 0) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("access_key has been set value", K(ret), K_(access_key)); + LOG_WARN("invalid args", K(ret), KP(authorization), K(length)); + } else if (OB_STORAGE_FILE == device_type_) { + // do nothing + } else if (OB_FAIL(get_access_key_(access_key_buf, sizeof(access_key_buf)))) { + LOG_WARN("failed to get access key", K(ret)); + } else if (OB_FAIL(databuff_printf(authorization, length, "%s&%s", access_id_, access_key_buf))) { + LOG_WARN("failed to set authorization", K(ret), K(length), K_(access_id), K(strlen(access_key_buf))); + } + + return ret; +} + #ifdef OB_BUILD_TDE_SECURITY - } else if (need_decrypt) { - char encrypted_key[OB_MAX_BACKUP_ENCRYPTKEY_LENGTH] = { 0 }; - if (OB_FAIL(set_storage_info_field_(buf, encrypted_key, sizeof(encrypted_key)))) { - LOG_WARN("failed to set access_key", K(ret), K(buf)); - } else if (OB_FAIL(decrypt_access_key_(encrypted_key))) { - LOG_WARN("failed to set access key", K(ret), K(buf)); - } -#endif - } else if (OB_FAIL(set_storage_info_field_(buf, access_key_, sizeof(access_key_)))) { - LOG_WARN("failed to set endpoint",K(ret), K(buf)); +int ObBackupStorageInfo::get_access_key_(char *key_buf, const int64_t key_buf_len) const +{ + int ret = OB_SUCCESS; + // encrypt_access_key_ will check args' validity + if (OB_FAIL(encrypt_access_key_(key_buf, key_buf_len))) { + LOG_WARN("failed to encrypt access key", K(ret)); } return ret; } -int ObBackupStorageInfo::check_delete_mode_(const char *delete_mode) +int ObBackupStorageInfo::parse_storage_info_(const char *storage_info, bool &has_appid) { int ret = OB_SUCCESS; - if (NULL == delete_mode) { - ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "invalid args", K(ret), KP(delete_mode)); - } else if (0 != strcmp(delete_mode, "delete") - && 0 != strcmp(delete_mode, "tagging")) { - ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "delete mode is invalid", K(ret), K(delete_mode)); - } - return ret; -} - -// oss:host=xxxx&access_id=xxx&access_key=xxx -// cos:host=xxxx&access_id=xxx&access_key=xxxappid=xxx -int ObBackupStorageInfo::set( - const common::ObStorageType device_type, - const char *storage_info) -{ - int ret = OB_SUCCESS; - if (is_valid()) { - ret = OB_INIT_TWICE; - LOG_WARN("storage info init twice", K(ret)); - } else if (OB_ISNULL(storage_info) || strlen(storage_info) >= OB_MAX_BACKUP_STORAGE_INFO_LENGTH) { + if (OB_ISNULL(storage_info) || strlen(storage_info) >= OB_MAX_BACKUP_STORAGE_INFO_LENGTH) { ret = OB_INVALID_BACKUP_DEST; - LOG_WARN("storage info is too long", K(ret), K(storage_info), K(strlen(storage_info))); - } else if (FALSE_IT(device_type_ = device_type)) { - } else if (0 == strlen(storage_info)) { - if (OB_STORAGE_FILE != device_type_) { - ret = OB_INVALID_BACKUP_DEST; - LOG_WARN("backup dest is valid", K(ret), K_(device_type)); - } + LOG_WARN("storage info is invalid", K(ret), K(storage_info), K(strlen(storage_info))); + } else if (OB_FAIL(ObObjectStorageInfo::parse_storage_info_(storage_info, has_appid))) { + LOG_WARN("failed to parse storage info", K(ret), K(storage_info)); } else { - int64_t info_len = strlen(storage_info); char tmp[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 }; + char serialize_key[OB_MAX_BACKUP_SERIALIZEKEY_LENGTH] = { 0 }; char *token = NULL; char *saved_ptr = NULL; - const char *HOST = "host="; - const char *APPID = "appid="; - const char *DELETE_MODE = "delete_mode="; - bool has_appid = false; + int64_t info_len = strlen(storage_info); MEMCPY(tmp, storage_info, info_len); tmp[info_len] = '\0'; @@ -1242,232 +1193,22 @@ int ObBackupStorageInfo::set( token = ::strtok_r(str, "&", &saved_ptr); if (NULL == token) { break; - } else if (0 == strncmp(HOST, token, strlen(HOST))) { - if (OB_FAIL(set_storage_info_field_(token, endpoint_, sizeof(endpoint_)))) { - LOG_WARN("failed to set endpoint",K(ret), K(token)); - } - } else if (0 == strncmp(ACCESS_ID, token, strlen(ACCESS_ID))) { - if (OB_FAIL(set_storage_info_field_(token, access_id_, sizeof(access_id_)))) { - LOG_WARN("failed to set access id", K(ret), K(token)); - } -#ifdef OB_BUILD_TDE_SECURITY } else if (0 == strncmp(ENCRYPT_KEY, token, strlen(ENCRYPT_KEY))) { - if (OB_FAIL(set_access_key_(token, true/*need decrypt*/))) { - LOG_WARN("failed to set oss extension", K(ret), K(token)); - } -#endif - } else if (0 == strncmp(ACCESS_KEY, token, strlen(ACCESS_KEY))) { - if (OB_FAIL(set_access_key_(token, false/*need decrypt*/))) { - LOG_WARN("failed to set oss extension", K(ret), K(token)); - } - } else if (OB_STORAGE_COS == device_type && 0 == strncmp(APPID, token, strlen(APPID))) { - has_appid = true; - if (OB_FAIL(set_storage_info_field_(token, extension_, sizeof(extension_)))) { - LOG_WARN("failed to set cos extension", K(ret), K(token)); - } - } else if (OB_STORAGE_OSS == device_type && 0 == strncmp(DELETE_MODE, token, strlen(DELETE_MODE))) { - if (OB_FAIL(check_delete_mode_(token + strlen(DELETE_MODE)))) { - OB_LOG(WARN, "failed to set delete mode", K(ret), K(token)); - } else if (OB_FAIL(set_storage_info_field_(token, extension_, sizeof(extension_)))) { - LOG_WARN("failed to set cos extension", K(ret), K(token)); - } - } else { - LOG_DEBUG("unkown storage info", K(*token)); - } - } - - if (OB_FAIL(ret)) { - } else if (OB_STORAGE_OSS == device_type &&(0 == strlen(endpoint_) || 0 == strlen(access_id_) || 0 == strlen(access_key_))) { - ret = OB_INVALID_BACKUP_DEST; - LOG_WARN("failed to parse storage info", K(ret), K_(endpoint), K_(access_id), K_(access_key)); - } else if (OB_STORAGE_COS == device_type && !has_appid) { - ret = OB_INVALID_BACKUP_DEST; - LOG_WARN("failed to parse cos info", K(ret), K_(extension)); - } else { - LOG_INFO("success to parse storage info", K(ret), K_(endpoint), K_(access_id), K_(extension)); - } - } - - return ret; -} - -int ObBackupStorageInfo::set( - const char *uri, - const char *storage_info) -{ - int ret = OB_SUCCESS; - common::ObStorageType device_type; - if (OB_FAIL(get_storage_type_from_path(uri, device_type))) { - LOG_WARN("failed to get storage type from path", K(ret), KPC(this)); - } else if (OB_FAIL(set(device_type, storage_info))) { - LOG_WARN("failed to set storage info", K(ret), KPC(this)); - } - return ret; -} - -int ObBackupStorageInfo::parse_authorization_(const char *authorization) -{ - int ret = OB_SUCCESS; - char tmp_authorization[OB_MAX_BACKUP_AUTHORIZATION_LENGTH] = { 0 }; - char *token = NULL; - char *saved_ptr = NULL; - if (OB_ISNULL(authorization) || 0 == strlen(authorization)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("authorization is invalid", K(ret)); - } else if (OB_FAIL(databuff_printf(tmp_authorization, sizeof(tmp_authorization), "%s", authorization))) { - LOG_WARN("failed to set token", K(ret)); - } else { - token = tmp_authorization; - for (char *str = token; OB_SUCC(ret); str = NULL) { - token = ::strtok_r(str, "&", &saved_ptr); - if (NULL == token) { - break; - } else if (0 == strncmp(ACCESS_ID, token, strlen(ACCESS_ID))) { - if (OB_FAIL(set_storage_info_field_(token, access_id_, sizeof(access_id_)))) { - LOG_WARN("failed to set access id", K(ret), K(token)); - } -#ifdef OB_BUILD_TDE_SECURITY - } else if (0 == strncmp(ENCRYPT_KEY, token, strlen(ENCRYPT_KEY))) { - char serialize_key[OB_MAX_BACKUP_SERIALIZEKEY_LENGTH] = { 0 }; - if (OB_FAIL(set_storage_info_field_(token, serialize_key, sizeof(serialize_key)))) { - LOG_WARN("failed to set encrypt_key", K(ret), K(token)); + if (0 != strlen(access_key_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("access_key has been set value", K(ret), K(strlen(access_key_))); + } else if (OB_FAIL(set_storage_info_field_(token, serialize_key, sizeof(serialize_key)))) { + LOG_WARN("failed to set encrypted_key", K(ret), K(token)); } else if (OB_FAIL(decrypt_access_key_(serialize_key))) { LOG_WARN("failed to decrypt access key", K(ret), K(token)); } -#endif - } else if (0 == strncmp(ACCESS_KEY, token, strlen(ACCESS_KEY))) { - if (OB_FAIL(set_storage_info_field_(token, access_key_, sizeof(access_key_)))) { - LOG_WARN("failed to set access key", K(ret), K(token)); - } } else { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(token)); } } } return ret; } -int ObBackupStorageInfo::get_authorization_info(char *authorization, int64_t length) -{ - int ret = OB_SUCCESS; - char encrypt_key[OB_MAX_BACKUP_SERIALIZEKEY_LENGTH] = { 0 }; - STATIC_ASSERT(OB_MAX_BACKUP_AUTHORIZATION_LENGTH > (OB_MAX_BACKUP_ACCESSID_LENGTH + OB_MAX_BACKUP_SERIALIZEKEY_LENGTH), "array length overflow"); - if (!is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("storage info not init", K(ret)); - } else if (OB_ISNULL(authorization) || length <= 0) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("storage info is invalid", K(ret)); - } else if (OB_STORAGE_FILE == device_type_) { - // do nothing -#ifndef OB_BUILD_TDE_SECURITY - } else if (OB_FAIL(databuff_printf(authorization, length, "%s&%s", access_id_, access_key_))) { - LOG_WARN("failed to set authorization", K(ret), K_(access_id), K(access_key_)); -#else - } else if (OB_FAIL(encrypt_access_key_(encrypt_key, sizeof(encrypt_key)))) { - LOG_WARN("failed to encrept access key", K(ret)); - } else if (OB_FAIL(databuff_printf(authorization, length, "%s&%s", access_id_, encrypt_key))) { - LOG_WARN("failed to set authorization", K(ret), K_(access_id), K(encrypt_key)); -#endif - } - return ret; -} - -int ObBackupStorageInfo::get_storage_info_str(char *storage_info, int64_t info_len, const bool need_encrypt) const -{ - int ret = OB_SUCCESS; - char encrypt_key[OB_MAX_BACKUP_SERIALIZEKEY_LENGTH] = { 0 }; - const char *key = NULL; - if (!is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("storage info not init", K(ret)); - } else if (OB_ISNULL(storage_info) || info_len <= 0) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("storage info is invalid", K(ret)); - } else if (OB_STORAGE_FILE == device_type_) { - // do nothing -#ifdef OB_BUILD_TDE_SECURITY - } else if (need_encrypt) { - if (OB_FAIL(encrypt_access_key_(encrypt_key, sizeof(encrypt_key)))) { - LOG_WARN("failed to encrypt access key", K(ret)); - } else { - key = encrypt_key; - } -#endif - } else { - key = access_key_; - } - - if (OB_FAIL(ret)) { - } else if (OB_STORAGE_FILE != device_type_ && OB_FAIL(databuff_printf(storage_info, info_len, "%s&%s&%s", endpoint_, access_id_, key))) { - LOG_WARN("failed to set storage info", K(ret)); - } else if (0 != strlen(extension_) && info_len > strlen(storage_info)) { - const int64_t str_len = strlen(storage_info); - if (OB_FAIL(databuff_printf(storage_info + str_len, info_len - str_len, "&%s", extension_))) { - LOG_WARN("failed to add extension", K(ret), K_(extension)); - } - } - return ret; -} - -int ObBackupStorageInfo::assign(const ObBackupStorageInfo &storage_info) -{ - int ret = OB_SUCCESS; - device_type_ = storage_info.device_type_; - MEMCPY(endpoint_, storage_info.endpoint_, sizeof(endpoint_)); - MEMCPY(access_id_, storage_info.access_id_, sizeof(access_id_)); - MEMCPY(access_key_, storage_info.access_key_, sizeof(access_key_)); - MEMCPY(extension_, storage_info.extension_, sizeof(extension_)); - - return ret; -} - -bool ObBackupStorageInfo::operator ==(const ObBackupStorageInfo &storage_info) const -{ - return device_type_ == storage_info.device_type_ - && (0 == STRCMP(endpoint_, storage_info.endpoint_)) - && (0 == STRCMP(access_id_, storage_info.access_id_)) - && (0 == STRCMP(access_key_, storage_info.access_key_)) - && (0 == STRCMP(extension_, storage_info.extension_)); -} - -bool ObBackupStorageInfo::operator !=(const ObBackupStorageInfo &storage_info) const -{ - return !(*this == storage_info); -} - -int64_t ObBackupStorageInfo::hash() const -{ - int64_t hash_value = 0; - hash_value = murmurhash(&device_type_, static_cast(sizeof(device_type_)), hash_value); - hash_value = murmurhash(endpoint_, static_cast(strlen(endpoint_)), hash_value); - hash_value = murmurhash(access_id_, static_cast(strlen(access_id_)), hash_value); - hash_value = murmurhash(access_key_, static_cast(strlen(access_key_)), hash_value); - hash_value = murmurhash(extension_, static_cast(strlen(extension_)), hash_value); - return hash_value; -} - -int ObBackupStorageInfo::set_storage_info_field_(const char *info, char *field, const int64_t length) -{ - int ret = OB_SUCCESS; - if (NULL == info || NULL == field) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), KP(info), KP(field)); - } else { - const int64_t info_len = strlen(info); - int64_t pos = strlen(field); - if (info_len >= length) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("info is too long ", K(ret), K(info), K(length)); - } else if (OB_FAIL(databuff_printf(field, length, pos, "%s", info))){ - LOG_WARN("failed to set storage info field", K(ret), K(info), K(field)); - } - } - return ret; -} - -#ifdef OB_BUILD_TDE_SECURITY int ObBackupStorageInfo::encrypt_access_key_(char *encrypt_key, int64_t length) const { int ret = OB_SUCCESS; @@ -1475,7 +1216,10 @@ int ObBackupStorageInfo::encrypt_access_key_(char *encrypt_key, int64_t length) char serialize_buf[OB_MAX_BACKUP_SERIALIZEKEY_LENGTH] = { 0 }; int64_t serialize_pos = 0; int64_t key_len = 0; - if (0 == strlen(access_key_)) { + if (OB_ISNULL(encrypt_key) || length <= 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", K(ret), KP(encrypt_key), K(length)); + } else if (0 == strlen(access_key_)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("access_key is empty, shouldn't encrypt", K(ret)); } else if (0 != strncmp(ACCESS_KEY, access_key_, strlen(ACCESS_KEY))) { @@ -1504,11 +1248,12 @@ int ObBackupStorageInfo::decrypt_access_key_(const char *buf) int64_t key_len = 0; char deserialize_buf[OB_MAX_BACKUP_SERIALIZEKEY_LENGTH] = { 0 }; char decrypt_key[OB_MAX_BACKUP_ACCESSKEY_LENGTH] = { 0 }; - int64_t buf_len = strlen(buf); + int64_t buf_len = 0; int64_t deserialize_size = 0; - if (0 == buf_len) { + if (OB_ISNULL(buf) || strlen(buf) == 0) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("encrypted_key is empty", K(ret)); + LOG_WARN("encrypted_key is empty", K(ret), KP(buf), K(strlen(buf))); + } else if (FALSE_IT(buf_len = strlen(buf))) { } else if (OB_FAIL(hex_to_cstr(buf + strlen(ENCRYPT_KEY), buf_len - strlen(ENCRYPT_KEY), deserialize_buf, sizeof(deserialize_buf), deserialize_size))) { LOG_WARN("failed to get cstr from hex", KR(ret), K(buf_len), K(sizeof(deserialize_buf))); @@ -1861,7 +1606,7 @@ int ObBackupDest::get_backup_dest_str(char *buf, const int64_t buf_size) const LOG_WARN("invalid argument", K(ret), KP(buf), K(buf_size)); } else if (OB_FAIL(databuff_printf(buf, buf_size, "%s", root_path_))) { LOG_WARN("failed to get backup dest str", K(ret), K(root_path_), K(storage_info_)); - } else if (OB_FAIL(storage_info_->get_storage_info_str(storage_info_str, sizeof(storage_info_str), true/*no need encrypt*/))) { + } else if (OB_FAIL(storage_info_->get_storage_info_str(storage_info_str, sizeof(storage_info_str)))) { OB_LOG(WARN, "fail to get storage info str!", K(ret), K(storage_info_)); } else if (0 != strlen(storage_info_str) && OB_FAIL(databuff_printf(buf + strlen(buf), buf_size - strlen(buf), "?%s",storage_info_str))) { LOG_WARN("failed to get backup dest str", K(ret), K(root_path_), K(storage_info_)); @@ -1885,7 +1630,7 @@ int ObBackupDest::get_backup_dest_str_with_primary_attr(char *buf, const int64_t LOG_WARN("invalid argument", K(ret), KP(buf), K(buf_size)); } else if (OB_FAIL(databuff_printf(buf, buf_size, "%s", root_path_))) { LOG_WARN("failed to get backup dest str", K(ret), K(root_path_), K(storage_info_)); - } else if (OB_FAIL(storage_info_->get_storage_info_str(storage_info_str, sizeof(storage_info_str), true/*no need encrypt*/))) { + } else if (OB_FAIL(storage_info_->get_storage_info_str(storage_info_str, sizeof(storage_info_str)))) { OB_LOG(WARN, "fail to get storage info str!", K(ret), K(storage_info_)); } else if (0 != strlen(storage_info_str) && OB_FAIL(databuff_printf(buf + strlen(buf), buf_size - strlen(buf), "?%s",storage_info_str))) { LOG_WARN("failed to get backup dest str", K(ret), K(root_path_), K(storage_info_)); diff --git a/src/share/backup/ob_backup_struct.h b/src/share/backup/ob_backup_struct.h index 175fd2614..6e516193d 100755 --- a/src/share/backup/ob_backup_struct.h +++ b/src/share/backup/ob_backup_struct.h @@ -55,14 +55,7 @@ const int64_t OB_BACKUP_DEFAULT_PG_NUM = 10000; const int64_t OB_MAX_BACKUP_DEST_LENGTH = 2048; const int64_t OB_MAX_RESTORE_DEST_LENGTH = OB_MAX_BACKUP_DEST_LENGTH * 10; const int64_t OB_MAX_BACKUP_PATH_LENGTH = 1024; -const int64_t OB_MAX_BACKUP_STORAGE_INFO_LENGTH = 1536; -const int64_t OB_MAX_BACKUP_ENDPOINT_LENGTH = 256; -const int64_t OB_MAX_BACKUP_ACCESSID_LENGTH = 256; -const int64_t OB_MAX_BACKUP_ACCESSKEY_LENGTH = 256; -const int64_t OB_MAX_BACKUP_ENCRYPTKEY_LENGTH = OB_MAX_BACKUP_ACCESSKEY_LENGTH + 32; -const int64_t OB_MAX_BACKUP_SERIALIZEKEY_LENGTH = OB_MAX_BACKUP_ENCRYPTKEY_LENGTH * 2; const int64_t OB_MAX_BACKUP_AUTHORIZATION_LENGTH = 1024; -const int64_t OB_MAX_BACKUP_EXTENSION_LENGTH = 512; const int64_t OB_MAX_BACKUP_CHECK_FILE_LENGTH = OB_MAX_BACKUP_PATH_LENGTH; const int64_t OB_MAX_BACKUP_CHECK_FILE_NAME_LENGTH = 256; const int64_t OB_MAX_KEPT_LOG_ARCHIVE_BACKUP_ROUND = 10 * 10000; // 10w @@ -358,9 +351,6 @@ const char *const OB_RESTORE_PREVIEW_BACKUP_CLUSTER_ID_SESSION_STR = "__ob_resto const char *const MULTI_BACKUP_SET_PATH_PREFIX = "BACKUPSET"; const char *const MULTI_BACKUP_PIECE_PATH_PREFIX = "BACKUPPIECE"; -const char *const ACCESS_ID = "access_id="; -const char *const ACCESS_KEY = "access_key="; - const char *const ENCRYPT_KEY = "encrypt_key="; const char *const OB_STR_INITIATOR_JOB_ID = "initiator_job_id"; const char *const OB_STR_EXECUTOR_TENANT_ID = "executor_tenant_id"; @@ -855,47 +845,29 @@ struct ObNonFrozenBackupPieceInfo final DECLARE_TO_STRING; }; -class ObBackupStorageInfo final +class ObBackupStorageInfo final : public common::ObObjectStorageInfo { public: - ObBackupStorageInfo(); - ~ObBackupStorageInfo(); - int set( - const common::ObStorageType device_type, - const char *storage_info); + using common::ObObjectStorageInfo::set; + +public: + ObBackupStorageInfo() {} + virtual ~ObBackupStorageInfo(); + int set( const common::ObStorageType device_type, const char *endpoint, const char *authorization, const char *extension); - int set( - const char *uri, - const char *storage_info); - int get_storage_info_str(char *storage_info, int64_t info_len, const bool need_encrypt) const; - int get_authorization_info(char *authorization, int64_t length); - bool is_valid() const; - void reset(); - bool operator ==(const ObBackupStorageInfo &storage_info) const; - bool operator !=(const ObBackupStorageInfo &storage_info) const; - const char *get_type_str() const; - int64_t hash() const; - int assign(const ObBackupStorageInfo &storage_info); - TO_STRING_KV(K_(endpoint), K_(access_id), K_(extension), "type", get_type_str()); + int get_authorization_info(char *authorization, const int64_t length) const; + private: - int set_storage_info_field_(const char *info, char *field, const int64_t length); #ifdef OB_BUILD_TDE_SECURITY - int encrypt_access_key_(char *encrypt_key, int64_t length) const; + virtual int get_access_key_(char *key_buf, const int64_t key_buf_len) const override; + virtual int parse_storage_info_(const char *storage_info, bool &has_appid) override; + int encrypt_access_key_(char *encrypt_key, const int64_t length) const; int decrypt_access_key_(const char *buf); #endif - int set_access_key_(const char *buf, const bool need_decrypt); - int parse_authorization_(const char *authorization); - int check_delete_mode_(const char *delete_mode); -public: - common::ObStorageType device_type_; - char endpoint_[OB_MAX_BACKUP_ENDPOINT_LENGTH]; - char access_id_[OB_MAX_BACKUP_ACCESSID_LENGTH]; - char access_key_[OB_MAX_BACKUP_ACCESSKEY_LENGTH]; - char extension_[OB_MAX_BACKUP_EXTENSION_LENGTH]; }; class ObBackupDest final diff --git a/src/share/ob_device_manager.cpp b/src/share/ob_device_manager.cpp index 07fb34e47..c7f69cd33 100644 --- a/src/share/ob_device_manager.cpp +++ b/src/share/ob_device_manager.cpp @@ -48,6 +48,8 @@ int ObDeviceManager::init_devices_env() OB_LOG(WARN, "Fail to init allocator ", K(ret)); } else if (OB_FAIL(init_oss_env())) { OB_LOG(WARN, "fail to init oss storage", K(ret)); + } else if (OB_FAIL(init_cos_env())) { + OB_LOG(WARN, "fail to init cos storage", K(ret)); } } @@ -81,6 +83,7 @@ void ObDeviceManager::destroy() } allocator_.reset(); fin_oss_env(); + fin_cos_env(); is_init_ = false; device_count_ = 0; OB_LOG_RET(WARN, ret_dev, "release the init resource", K(ret_dev), K(ret_handle)); diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.cpp b/src/sql/resolver/ddl/ob_ddl_resolver.cpp index a5bc13e86..7460d130e 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.cpp +++ b/src/sql/resolver/ddl/ob_ddl_resolver.cpp @@ -2001,7 +2001,7 @@ int ObDDLResolver::resolve_table_option(const ParseNode *option_node, const bool OZ (ObSQLUtils::split_remote_object_storage_url(url, storage_info)); } OZ (tmp_location.append(url)); - OZ (storage_info.get_storage_info_str(storage_info_buf, sizeof(storage_info_buf), true)); + OZ (storage_info.get_storage_info_str(storage_info_buf, sizeof(storage_info_buf))); OZ (arg.schema_.set_external_file_location(tmp_location.string())); OZ (arg.schema_.set_external_file_location_access_info(storage_info_buf)); } diff --git a/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp b/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp index aa3bef664..f779f06c9 100644 --- a/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp +++ b/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp @@ -510,7 +510,7 @@ ObAdminDumpBackupDataExecutor::ObAdminDumpBackupDataExecutor() : offset_(), leng allocator_() { MEMSET(backup_path_, 0, common::OB_MAX_URI_LENGTH); - MEMSET(storage_info_, 0, share::OB_MAX_BACKUP_STORAGE_INFO_LENGTH); + MEMSET(storage_info_, 0, common::OB_MAX_BACKUP_STORAGE_INFO_LENGTH); } ObAdminDumpBackupDataExecutor::~ObAdminDumpBackupDataExecutor() diff --git a/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.h b/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.h index 50a97599a..978336bb5 100644 --- a/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.h +++ b/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.h @@ -193,7 +193,7 @@ private: storage::ObExternTenantLocalityInfoDesc &locality_info); private: char backup_path_[common::OB_MAX_URI_LENGTH]; - char storage_info_[share::OB_MAX_BACKUP_STORAGE_INFO_LENGTH]; + char storage_info_[common::OB_MAX_BACKUP_STORAGE_INFO_LENGTH]; int64_t offset_; int64_t length_; int64_t file_type_; diff --git a/unittest/share/backup/CMakeLists.txt b/unittest/share/backup/CMakeLists.txt index b116e7750..aaec41242 100644 --- a/unittest/share/backup/CMakeLists.txt +++ b/unittest/share/backup/CMakeLists.txt @@ -1,4 +1,5 @@ storage_unittest(test_backup_path) storage_unittest(test_backup_struct) storage_unittest(test_log_archive_backup_info_mgr) -storage_unittest(test_archive_checkpoint_mgr) \ No newline at end of file +storage_unittest(test_backup_access_cos) +storage_unittest(test_archive_checkpoint_mgr) diff --git a/unittest/share/backup/test_backup_access_cos.cpp b/unittest/share/backup/test_backup_access_cos.cpp new file mode 100644 index 000000000..a41205db1 --- /dev/null +++ b/unittest/share/backup/test_backup_access_cos.cpp @@ -0,0 +1,326 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#include "lib/utility/ob_test_util.h" +#include "lib/restore/ob_storage.h" +#include "lib/restore/ob_storage_cos_base.h" +#include "lib/allocator/page_arena.h" +#include "share/backup/ob_backup_io_adapter.h" +#include "test_backup_access_cos.h" + +using namespace oceanbase::common; + +class TestStorageCosCommon { +public: +TestStorageCosCommon() {} +~TestStorageCosCommon() {} + +void init() +{ + // ObDeviceManager will init env in first call + // init_cos_env(); + + ASSERT_EQ(OB_SUCCESS, + databuff_printf(account, sizeof(account), + "host=%s&access_id=%s&access_key=%s&appid=%s", + endpoint, secretid, secretkey, appid)); + //build cos_base + const ObString cos_storage_info(account); + ASSERT_EQ(OB_SUCCESS, cos_base.set(ObStorageType::OB_STORAGE_COS, cos_storage_info.ptr())); +} +void destory() +{ + // fin_cos_env(); +} +protected: + char account[OB_MAX_URI_LENGTH]; + const char *dir_name = "test_backup_io_adapter_access_cos_dir"; + char uri[OB_MAX_URI_LENGTH]; + char dir_uri[OB_MAX_URI_LENGTH]; + oceanbase::share::ObBackupStorageInfo cos_base; + + int object_prefix_len = 5; +}; + +class TestBackupIOAdapterAccessCos : public ::testing::Test, public TestStorageCosCommon +{ +public: + TestBackupIOAdapterAccessCos() : enable_test_(enable_test) {} + virtual ~TestBackupIOAdapterAccessCos() {} + virtual void SetUp() + { + init(); + } + virtual void TearDown() + { + destory(); + } + + static void SetUpTestCase() + { + } + + static void TearDownTestCase() + { + } + +private: + // disallow copy + DISALLOW_COPY_AND_ASSIGN(TestBackupIOAdapterAccessCos); +protected: + bool enable_test_; +}; + +TEST_F(TestBackupIOAdapterAccessCos, test_basic_rw) +{ + int ret = OB_SUCCESS; + if (enable_test_) { + ObBackupIoAdapter adapter; + const char *tmp_dir = "test_basic_rw"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_dir, ts)); + + // write + const char *write_content = "123456789ABCDEF"; + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/0", dir_uri)); + ASSERT_EQ(OB_SUCCESS, + adapter.write_single_file(uri, &cos_base, write_content, strlen(write_content))); + + // read + char read_buf[100] = {0}; + int64_t read_size = 0; + ASSERT_EQ(OB_SUCCESS, + adapter.read_single_file(uri, &cos_base, read_buf, sizeof(read_buf), read_size)); + ASSERT_STREQ(write_content, read_buf); + ASSERT_EQ(strlen(write_content), read_size); + + ASSERT_EQ(OB_SUCCESS, + adapter.read_part_file(uri, &cos_base, read_buf, sizeof(read_buf), 0, read_size)); + ASSERT_STREQ(write_content, read_buf); + ASSERT_EQ(strlen(write_content), read_size); + + int64_t offset = 5; + ASSERT_EQ(OB_SUCCESS, + adapter.read_part_file(uri, &cos_base, read_buf, sizeof(read_buf), offset, read_size)); + ASSERT_EQ('6', read_buf[0]); + ASSERT_EQ('F', read_buf[9]); + ASSERT_EQ(strlen(write_content) - offset, read_size); + + offset = 6; + ASSERT_EQ(OB_SUCCESS, + adapter.read_part_file(uri, &cos_base, read_buf, 5, offset, read_size)); + ASSERT_EQ('7', read_buf[0]); + ASSERT_EQ('B', read_buf[4]); + ASSERT_EQ(5, read_size); + + offset = strlen(write_content); + ASSERT_EQ(OB_COS_ERROR, + adapter.read_part_file(uri, &cos_base, read_buf, sizeof(read_buf), offset, read_size)); + + ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &cos_base)); + } +} + +TEST_F(TestBackupIOAdapterAccessCos, test_util) +{ + int ret = OB_SUCCESS; + if (enable_test_) { + ObBackupIoAdapter adapter; + const char *tmp_dir = "test_util"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_dir, ts)); + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/0", dir_uri)); + + bool is_obj_exist = true; + ASSERT_EQ(OB_SUCCESS, adapter.is_exist(uri, &cos_base, is_obj_exist)); + ASSERT_FALSE(is_obj_exist); + + const char *write_content = "123456789ABCDEF"; + ASSERT_EQ(OB_SUCCESS, + adapter.write_single_file(uri, &cos_base, write_content, strlen(write_content))); + ASSERT_EQ(OB_SUCCESS, adapter.is_exist(uri, &cos_base, is_obj_exist)); + ASSERT_TRUE(is_obj_exist); + + int64_t file_length = -1; + ASSERT_EQ(OB_SUCCESS, adapter.get_file_length(uri, &cos_base, file_length)); + ASSERT_EQ(strlen(write_content), file_length); + + ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &cos_base)); + ASSERT_EQ(OB_SUCCESS, adapter.is_exist(uri, &cos_base, is_obj_exist)); + ASSERT_FALSE(is_obj_exist); + } +} + +TEST_F(TestBackupIOAdapterAccessCos, test_list_files) +{ + int ret = OB_SUCCESS; + if (enable_test_) { + ObBackupIoAdapter adapter; + const char *tmp_dir = "test_list_files"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_dir, ts)); + + int64_t file_num = 1001; + const char *write_content = "0123456789"; + const char *format = "%s/%0*ld_%ld"; + for (int64_t file_idx = 0; file_idx < file_num; file_idx++) { + ASSERT_EQ(OB_SUCCESS, + databuff_printf(uri, sizeof(uri), format, dir_uri, object_prefix_len, file_idx, file_idx)); + ASSERT_EQ(OB_SUCCESS, + adapter.write_single_file(uri, &cos_base, write_content, strlen(write_content))); + } + + ObArenaAllocator allocator; + ObArray name_array; + ObFileListArrayOp op(name_array, allocator); + ASSERT_EQ(OB_SUCCESS, adapter.list_files(dir_uri, &cos_base, op)); + ASSERT_EQ(file_num, name_array.size()); + for (int64_t file_idx = 0; file_idx < file_num; file_idx++) { + // listed files do not contain prefix + ASSERT_EQ(OB_SUCCESS, + databuff_printf(uri, sizeof(uri), "%0*ld_%ld", object_prefix_len, file_idx, file_idx)); + ASSERT_STREQ(uri, name_array[file_idx].ptr()); + ASSERT_EQ(OB_SUCCESS, + databuff_printf(uri, sizeof(uri), format, dir_uri, object_prefix_len, file_idx, file_idx)); + ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &cos_base)); + } + } +} + +TEST_F(TestBackupIOAdapterAccessCos, test_list_directories) +{ + int ret = OB_SUCCESS; + if (enable_test_) { + ObBackupIoAdapter adapter; + const char *tmp_dir = "test_list_directories"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_dir, ts)); + + int64_t file_num = 1001; + const char *write_content = "0123456789"; + const char *format = "%s/%0*ld/%ld"; + for (int64_t file_idx = 0; file_idx < file_num; file_idx++) { + ASSERT_EQ(OB_SUCCESS, + databuff_printf(uri, sizeof(uri), format, dir_uri, object_prefix_len, file_idx, file_idx)); + ASSERT_EQ(OB_SUCCESS, + adapter.write_single_file(uri, &cos_base, write_content, strlen(write_content))); + } + + ObArenaAllocator allocator; + ObArray name_array; + ObFileListArrayOp op(name_array, allocator); + ASSERT_EQ(OB_SUCCESS, adapter.list_directories(dir_uri, &cos_base, op)); + ASSERT_EQ(file_num, name_array.size()); + for (int64_t file_idx = 0; file_idx < file_num; file_idx++) { + // listed files do not contain prefix + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%0*ld", object_prefix_len, file_idx)); + ASSERT_STREQ(uri, name_array[file_idx].ptr()); + ASSERT_EQ(OB_SUCCESS, + databuff_printf(uri, sizeof(uri), format, dir_uri, object_prefix_len, file_idx, file_idx)); + ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &cos_base)); + } + } +} + +TEST_F(TestBackupIOAdapterAccessCos, test_is_tagging) +{ + int ret = OB_SUCCESS; + if (enable_test_) { + ObBackupIoAdapter adapter; + const char *tmp_util_dir = "test_util_is_tagging"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_util_dir, ts)); + + bool is_tagging = true; + char tmp_account[OB_MAX_URI_LENGTH]; + oceanbase::share::ObBackupStorageInfo tmp_cos_base; + const char *write_content = "123456789ABCDEF"; + + // wrong tag mode + ASSERT_EQ(OB_SUCCESS, + databuff_printf(tmp_account, sizeof(tmp_account), + "host=%s&access_id=%s&access_key=%s&appid=%s&delete_mode=tag", + endpoint, secretid, secretkey, appid)); + ASSERT_EQ(OB_INVALID_ARGUMENT, tmp_cos_base.set(ObStorageType::OB_STORAGE_COS, tmp_account)); + tmp_cos_base.reset(); + + ASSERT_EQ(OB_SUCCESS, + databuff_printf(tmp_account, sizeof(tmp_account), + "host=%s&access_id=%s&access_key=%s&appid=%s&delete_mode=delete_delete", + endpoint, secretid, secretkey, appid)); + ASSERT_EQ(OB_INVALID_ARGUMENT, tmp_cos_base.set(ObStorageType::OB_STORAGE_COS, tmp_account)); + tmp_cos_base.reset(); + + // delete mode + ASSERT_EQ(OB_SUCCESS, + databuff_printf(tmp_account, sizeof(tmp_account), + "host=%s&access_id=%s&access_key=%s&appid=%s&delete_mode=delete", + endpoint, secretid, secretkey, appid)); + ASSERT_EQ(OB_SUCCESS, tmp_cos_base.set(ObStorageType::OB_STORAGE_COS, tmp_account)); + + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/delete_mode", dir_uri)); + ASSERT_EQ(OB_SUCCESS, + adapter.write_single_file(uri, &tmp_cos_base, write_content, strlen(write_content))); + ASSERT_EQ(OB_SUCCESS, adapter.is_tagging(uri, &tmp_cos_base, is_tagging)); + ASSERT_FALSE(is_tagging); + + ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &tmp_cos_base)); + ASSERT_EQ(OB_BACKUP_FILE_NOT_EXIST, adapter.is_tagging(uri, &tmp_cos_base, is_tagging)); + tmp_cos_base.reset(); + + // tagging mode + ASSERT_EQ(OB_SUCCESS, + databuff_printf(tmp_account, sizeof(tmp_account), + "host=%s&access_id=%s&access_key=%s&appid=%s&delete_mode=tagging", + endpoint, secretid, secretkey, appid)); + ASSERT_EQ(OB_SUCCESS, tmp_cos_base.set(ObStorageType::OB_STORAGE_COS, tmp_account)); + + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/tagging_mode", dir_uri)); + ASSERT_EQ(OB_SUCCESS, + adapter.write_single_file(uri, &tmp_cos_base, write_content, strlen(write_content))); + + is_tagging = true; + ASSERT_EQ(OB_SUCCESS, adapter.is_tagging(uri, &tmp_cos_base, is_tagging)); + ASSERT_FALSE(is_tagging); + + ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &tmp_cos_base)); + ASSERT_EQ(OB_SUCCESS, adapter.is_tagging(uri, &tmp_cos_base, is_tagging)); + ASSERT_TRUE(is_tagging); + tmp_cos_base.reset(); + + // clean + ASSERT_EQ(OB_SUCCESS, + databuff_printf(tmp_account, sizeof(tmp_account), + "host=%s&access_id=%s&access_key=%s&appid=%s", + endpoint, secretid, secretkey, appid)); + ASSERT_EQ(OB_SUCCESS, tmp_cos_base.set(ObStorageType::OB_STORAGE_COS, tmp_account)); + + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/tagging_mode", dir_uri)); + ASSERT_EQ(OB_SUCCESS, adapter.del_file(uri, &tmp_cos_base)); + ASSERT_EQ(OB_BACKUP_FILE_NOT_EXIST, adapter.is_tagging(uri, &tmp_cos_base, is_tagging)); + } +} + +int main(int argc, char **argv) +{ + system("rm -f test_backup_access_cos.log*"); + OB_LOGGER.set_file_name("test_backup_access_cos.log", true, true); + OB_LOGGER.set_log_level("INFO"); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/unittest/share/backup/test_backup_access_cos.h b/unittest/share/backup/test_backup_access_cos.h new file mode 100644 index 000000000..263c3e896 --- /dev/null +++ b/unittest/share/backup/test_backup_access_cos.h @@ -0,0 +1,23 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef UNITTEST_BACKUP_ACCESS_COS_H_ +#define UNITTEST_BACKUP_ACCESS_COS_H_ + +const bool enable_test = false; +const char *bucket = "cos://xxx"; +const char *endpoint = "xxx"; +const char *secretid = "xxx"; +const char *secretkey = "xxx"; +const char *appid = "xxx"; + +#endif \ No newline at end of file diff --git a/unittest/share/backup/test_backup_struct.cpp b/unittest/share/backup/test_backup_struct.cpp index 2aa506821..f9182f59f 100644 --- a/unittest/share/backup/test_backup_struct.cpp +++ b/unittest/share/backup/test_backup_struct.cpp @@ -52,6 +52,9 @@ TEST(ObBackupDest, nfs) const char *backup_test = "file:///backup_dir/?&delete_mode=tagging"; ObBackupDest dest; ObBackupDest dest1; + ASSERT_EQ(OB_INVALID_BACKUP_DEST, dest.set(backup_test)); + backup_test = "file:///backup_dir/"; + char backup_dest_str[OB_MAX_BACKUP_DEST_LENGTH] = { 0 }; char backup_path_str[OB_MAX_BACKUP_DEST_LENGTH] = { 0 }; ASSERT_EQ(OB_SUCCESS, dest.set(backup_test)); @@ -144,10 +147,79 @@ TEST(ObBackupDest, oss_encrypt) ObMasterKeyGetter::instance().wait(); ObMasterKeyGetter::instance().reset(); } -#endif // + +TEST(ObBackupDest, cos) +{ + const char *backup_test = "cos://backup_dir/?host=xxx.com&access_id=111&access_key=222&delete_mode=tagging&appid=333"; + ObBackupDest dest; + ObBackupDest dest1; + char backup_dest_str[OB_MAX_BACKUP_DEST_LENGTH] = { 0 }; + char backup_path_str[OB_MAX_BACKUP_DEST_LENGTH] = { 0 }; + ASSERT_EQ(OB_SUCCESS, dest.set(backup_test)); + ASSERT_EQ(OB_SUCCESS, dest1.set(backup_test)); + LOG_INFO("dump backup dest", K(dest), K(dest.get_root_path()), K(*(dest.get_storage_info()))); + ASSERT_EQ(0, strcmp(dest.root_path_, "cos://backup_dir")); + ASSERT_TRUE(dest.storage_info_->device_type_ == 2); + + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().init(NULL)); + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().set_root_key(OB_SYS_TENANT_ID, + obrpc::RootKeyType::DEFAULT, ObString())); + ASSERT_EQ(OB_SUCCESS, dest.get_backup_dest_str(backup_dest_str, sizeof(backup_dest_str))); + ASSERT_EQ(0, strcmp(backup_dest_str, "cos://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F&delete_mode=tagging&appid=333")); + ASSERT_EQ(OB_SUCCESS, dest.get_backup_path_str(backup_path_str, sizeof(backup_path_str))); + ASSERT_EQ(0, strcmp(backup_path_str, "cos://backup_dir?host=xxx.com")); + ASSERT_TRUE(dest.is_root_path_equal(dest1)); + bool is_equal = false; + ASSERT_EQ(OB_SUCCESS, dest.is_backup_path_equal(dest1, is_equal)); + ASSERT_TRUE(is_equal); + ASSERT_TRUE(dest == dest1); + dest1.reset(); + ASSERT_EQ(OB_SUCCESS, dest1.set(dest.get_root_path().ptr(), dest.get_storage_info())); + ASSERT_TRUE(dest == dest1); + ObMasterKeyGetter::instance().stop(); + ObMasterKeyGetter::instance().wait(); + ObMasterKeyGetter::instance().reset(); +} + +TEST(ObBackupDest, cos_encrypt) +{ + const char *backup_test = "cos://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F&appid=333"; + ObBackupDest dest; + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().init(NULL)); + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().set_root_key(OB_SYS_TENANT_ID, + obrpc::RootKeyType::DEFAULT, ObString())); + ASSERT_EQ(OB_SUCCESS, dest.set(backup_test)); + LOG_INFO("dump backup dest", K(dest.get_root_path()), K(*(dest.get_storage_info()))); + ASSERT_EQ(0, strcmp(dest.root_path_, "cos://backup_dir")); + ASSERT_TRUE(dest.storage_info_->device_type_ == 2); + const char *path = "cos://backup_dir/"; + const char *endpoint = "host=xxx.com"; + const char *authorization = "access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F"; + const char *extension = "appid=333"; + ObBackupDest dest1; + ASSERT_EQ(OB_SUCCESS, dest1.set(path, endpoint, authorization, extension)); + ASSERT_TRUE(dest == dest1); + + char backup_dest_str[OB_MAX_BACKUP_DEST_LENGTH] = { 0 }; + char backup_path_str[OB_MAX_BACKUP_DEST_LENGTH] = { 0 }; + ASSERT_EQ(OB_SUCCESS, dest.get_backup_dest_str(backup_dest_str, sizeof(backup_dest_str))); + ASSERT_EQ(0, strcmp(backup_dest_str, "cos://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F&appid=333")); + ASSERT_EQ(OB_SUCCESS, dest.get_backup_path_str(backup_path_str, sizeof(backup_path_str))); + ASSERT_EQ(0, strcmp(backup_path_str, "cos://backup_dir?host=xxx.com")); + + dest1.reset(); + ASSERT_EQ(OB_SUCCESS, dest1.set(path, endpoint, authorization, extension)); + ASSERT_TRUE(dest == dest1); + ObMasterKeyGetter::instance().stop(); + ObMasterKeyGetter::instance().wait(); + ObMasterKeyGetter::instance().reset(); +} +#endif int main(int argc, char **argv) { + system("rm -f test_backup_struct.log*"); + OB_LOGGER.set_file_name("test_backup_struct.log", true, true); OB_LOGGER.set_log_level("INFO"); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS();