[CP] Add checksum type check and multipart upload check in backup connectivity check

deps/oblib/src/lib/restore/ob_object_device.cpp

@@ -20,6 +20,16 @@ namespace common
 const char *OB_STORAGE_ACCESS_TYPES_STR[] = {"reader", "adaptive_reader", "overwriter",
                                              "appender", "random_write", "multipart_writer"};
 
+const char *get_storage_access_type_str(const ObStorageAccessType &type)
+{
+  const char *str = "UNKNOWN";
+  STATIC_ASSERT(static_cast<int64_t>(OB_STORAGE_ACCESS_MAX_TYPE) == ARRAYSIZEOF(OB_STORAGE_ACCESS_TYPES_STR), "ObStorageAccessType count mismatch");
+  if (type >= OB_STORAGE_ACCESS_READER && type < OB_STORAGE_ACCESS_MAX_TYPE) {
+    str = OB_STORAGE_ACCESS_TYPES_STR[type];
+  }
+  return str;
+}
+
 ObObjectDevice::ObObjectDevice()
   : storage_info_(), is_started_(false), lock_(common::ObLatchIds::OBJECT_DEVICE_LOCK)
 {

deps/oblib/src/lib/restore/ob_object_device.h

@@ -21,7 +21,7 @@ namespace oceanbase
 {
 namespace common
 {
-
+const char *get_storage_access_type_str(const ObStorageAccessType &type);
 /*
 there are three write mode
 ------use write interface----

deps/oblib/src/lib/restore/ob_storage_info.cpp

@@ -21,6 +21,17 @@ namespace oceanbase
 
 namespace common
 {
+const char *OB_STORAGE_CHECKSUM_TYPE_STR[] = {CHECKSUM_TYPE_NO_CHECKSUM, CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_CRC32};
+
+const char *get_storage_checksum_type_str(const ObStorageChecksumType &type)
+{
+  const char *str = "UNKNOWN";
+  STATIC_ASSERT(static_cast<int64_t>(OB_STORAGE_CHECKSUM_MAX_TYPE) == ARRAYSIZEOF(OB_STORAGE_CHECKSUM_TYPE_STR), "ObStorageChecksumType count mismatch");
+  if (type >= OB_NO_CHECKSUM_ALGO && type < OB_STORAGE_CHECKSUM_MAX_TYPE) {
+    str = OB_STORAGE_CHECKSUM_TYPE_STR[type];
+  }
+  return str;
+}
 
 //***********************ObObjectStorageInfo***************************
 ObObjectStorageInfo::ObObjectStorageInfo()
@@ -95,6 +106,11 @@ ObStorageChecksumType ObObjectStorageInfo::get_checksum_type() const
   return checksum_type_;
 }
 
+const char *ObObjectStorageInfo::get_checksum_type_str() const
+{
+  return get_storage_checksum_type_str(checksum_type_);
+}
+
 // oss:host=xxxx&access_id=xxx&access_key=xxx
 // cos:host=xxxx&access_id=xxx&access_key=xxxappid=xxx
 // s3:host=xxxx&access_id=xxx&access_key=xxx&s3_region=xxx

deps/oblib/src/lib/restore/ob_storage_info.h

@@ -50,11 +50,13 @@ enum ObStorageChecksumType
   OB_NO_CHECKSUM_ALGO = 0,
   OB_MD5_ALGO = 1,
   OB_CRC32_ALGO = 2,
+  OB_STORAGE_CHECKSUM_MAX_TYPE
 };
 
 bool is_oss_supported_checksum(const ObStorageChecksumType checksum_type);
 bool is_cos_supported_checksum(const ObStorageChecksumType checksum_type);
 bool is_s3_supported_checksum(const ObStorageChecksumType checksum_type);
+const char *get_storage_checksum_type_str(const ObStorageChecksumType &type);
 
 class ObObjectStorageInfo
 {
@@ -69,6 +71,7 @@ public:
   ObStorageType get_type() const;
   const char *get_type_str() const;
   ObStorageChecksumType get_checksum_type() const;
+  const char *get_checksum_type_str() const;
   int get_storage_info_str(char *storage_info, const int64_t info_len) const;
 
   bool is_valid() const;

src/share/backup/ob_backup_connectivity.cpp

@@ -20,10 +20,12 @@
 #include "share/ob_srv_rpc_proxy.h"
 #include "share/backup/ob_backup_data_table_operator.h"
 #include "share/schema/ob_multi_version_schema_service.h"
+#include "lib/restore/ob_object_device.h"
 
-using namespace oceanbase;
+namespace oceanbase
-using namespace share;
+{
-
+namespace share
+{
 ObBackupConnectivityCheckManager::ObBackupConnectivityCheckManager()
   : is_inited_(false),
     tenant_id_(OB_INVALID_TENANT_ID),
@@ -417,7 +419,10 @@ int ObBackupCheckFile::create_connectivity_check_file(
     } else if (OB_FAIL(store.init(backup_dest))) {
       LOG_WARN("failed to set buffer", K(ret), K_(tenant_id));
     } else if (OB_FAIL(store.write_check_file(path.get_ptr(), check_desc))) {
-      LOG_WARN("failed to set buffer", K(ret), K_(tenant_id));
+      if (OB_CHECKSUM_TYPE_NOT_SUPPORTED == ret) {
+        LOG_USER_ERROR(OB_CHECKSUM_TYPE_NOT_SUPPORTED, backup_dest.get_storage_info()->get_checksum_type_str());
+      }
+      LOG_WARN("failed to write check file", K(ret), K(path), K(check_desc));
     } else {
       is_new_create = true;
       FLOG_INFO("[BACKUP_DEST_CHECK] succeed to create new check file", K(path), K(is_match));
@@ -478,17 +483,21 @@ int ObBackupCheckFile::delete_permission_check_file(const ObBackupDest &backup_d
 
 int ObBackupCheckFile::get_permission_check_file_path_(
     const ObBackupDest &backup_dest,
-    bool is_appender,
+    const ObStorageAccessType access_type,
     share::ObBackupPath &path)
 {
   int ret = OB_SUCCESS;
   int64_t check_time_s = ObTimeUtility::current_time() / 1000/ 1000;
   char buff[OB_BACKUP_MAX_TIME_STR_LEN] = { 0 };
-  const char *prefix = is_appender ? "append" : "put";
+  const char *prefix = nullptr;
   int64_t pos = 0;
   if (!is_inited_) {
     ret = OB_NOT_INIT;
     LOG_WARN("backup check file not init", K(ret));
+  } else if (ObStorageAccessType::OB_STORAGE_ACCESS_MAX_TYPE <= access_type) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid access type", K(ret), K(access_type));
+  } else if (OB_FALSE_IT(prefix = get_storage_access_type_str(access_type))) {
   } else if (OB_FAIL(get_check_file_path(backup_dest, path))) {
     LOG_WARN("failed to get check file path", K(ret), K(backup_dest));
   } else if (OB_FAIL(backup_time_to_strftime(check_time_s, buff, sizeof(buff), pos, 'T'/* concat */))) {
@@ -524,7 +533,7 @@ int ObBackupCheckFile::check_appender_permission_(const ObBackupDest &backup_des
   if (!is_inited_) {
     ret = OB_NOT_INIT;
     LOG_WARN("backup check file not init", K(ret));
-  } else if (OB_FAIL(get_permission_check_file_path_(backup_dest, true/*is_appender*/, path))) {
+  } else if (OB_FAIL(get_permission_check_file_path_(backup_dest, ObStorageAccessType::OB_STORAGE_ACCESS_APPENDER, path))) {
       LOG_WARN("failed to get permission check file path", K(ret), K_(tenant_id), K(backup_dest)); 
   } else if (OB_FAIL(util.set_access_type(&iod_opts, true/*is_appender*/, DEFAULT_OPT_ARG_NUM))) {
     LOG_WARN("fail to set access type");
@@ -539,7 +548,7 @@ int ObBackupCheckFile::check_appender_permission_(const ObBackupDest &backup_des
   } else if (OB_FAIL(device_handle->write(fd, data, strlen(data),  write_size))) {
     LOG_WARN("fail to write file", K(ret), K(path.get_ptr()), K(data));
   } else if (OB_FAIL(util.adaptively_del_file(path.get_obstr(), backup_dest.get_storage_info()))) {
-    LOG_WARN("failed to del file", K(ret));
+    LOG_WARN("failed to del file", K(ret), K(path));
   }
 
   if (OB_SUCCESS != (tmp_ret = util.close_device_and_fd(device_handle, fd))) {
@@ -550,6 +559,49 @@ int ObBackupCheckFile::check_appender_permission_(const ObBackupDest &backup_des
   return ret;
 }
 
+int ObBackupCheckFile::check_multipart_upload_permission_(const ObBackupDest &backup_dest)
+{
+  int ret = OB_SUCCESS;
+  int tmp_ret = OB_SUCCESS;
+  ObBackupIoAdapter util;
+  ObStorageAccessType access_type = OB_STORAGE_ACCESS_MULTIPART_WRITER;
+  int64_t write_size = 0;
+  int64_t offset = 0;
+  ObIODevice *device_handle = NULL;
+  ObIOFd fd;
+  const static int64_t BUF_LENGTH = 64;
+  char data[BUF_LENGTH];
+  ObBackupPath path;
+  if (!is_inited_) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("backup check file not init", K(ret));
+  } else if (OB_FAIL(get_permission_check_file_path_(backup_dest, access_type, path))) {
+    LOG_WARN("failed to get permission check file path", K(ret), K_(tenant_id), K(backup_dest));
+  } else if (OB_FAIL(util.open_with_access_type(device_handle, fd, backup_dest.get_storage_info(), path.get_obstr(), access_type))) {
+    LOG_WARN("fail to open device or fd", K(ret), K(backup_dest), K(path));
+  } else if (OB_FAIL(databuff_printf(data, sizeof(data), "tenant(%lu) multipart writer at %ld", tenant_id_, ObTimeUtility::current_time()))) {
+    LOG_WARN("fail to set data", K(ret), K(path.get_ptr()));
+  } else if (OB_FAIL(device_handle->pwrite(fd, offset, strlen(data), data, write_size))) {
+    LOG_WARN("fail to write file", K(ret), K(path.get_ptr()), K(data));
+  } else if (OB_FAIL(device_handle->complete(fd))) {
+    STORAGE_LOG(WARN, "fail to complete multipart upload", K(ret), K(device_handle), K(fd));
+  } else if (OB_FAIL(util.del_file(path.get_obstr(), backup_dest.get_storage_info()))) {
+    LOG_WARN("failed to del file", K(ret));
+  }
+
+  if (OB_FAIL(ret)) {
+    if (OB_TMP_FAIL(device_handle->abort(fd))) {
+      ret = COVER_SUCC(tmp_ret);
+      STORAGE_LOG(WARN, "fail to abort multipart upload", K(ret), K(tmp_ret), K(device_handle), K(fd));
+    }
+  }
+  if (OB_TMP_FAIL(util.close_device_and_fd(device_handle, fd))) {
+    ret = COVER_SUCC(tmp_ret);
+    STORAGE_LOG(WARN, "fail to close device and fd", K(ret), K(tmp_ret), K(device_handle), K(fd));
+  }
+  return ret;
+}
+
 bool ObBackupCheckFile::is_permission_error_(const int32_t result) 
 { 
   int ret = OB_SUCCESS;
@@ -582,7 +634,7 @@ int ObBackupCheckFile::check_io_permission(const ObBackupDest &backup_dest)
   } else if (!backup_dest.is_valid()) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("backup dest is valid", K(ret), K_(tenant_id)); 
-  } else if (OB_FAIL(get_permission_check_file_path_(backup_dest, false/*is_appender*/, path))) {
+  } else if (OB_FAIL(get_permission_check_file_path_(backup_dest, ObStorageAccessType::OB_STORAGE_ACCESS_OVERWRITER, path))) {
       LOG_WARN("failed to get permission check file path", K(ret), K_(tenant_id));
   } else if (OB_FAIL(generate_format_desc_(backup_dest, check_desc))) {
     LOG_WARN("failed to set buffer", K(ret), K_(tenant_id));
@@ -590,17 +642,17 @@ int ObBackupCheckFile::check_io_permission(const ObBackupDest &backup_dest)
     LOG_WARN("failed to set buffer", K(ret), K_(tenant_id));
   } else if (OB_FAIL(store.write_check_file(path.get_ptr(), check_desc))) {
     if (is_permission_error_(ret)) {
+      ret = OB_BACKUP_PERMISSION_DENIED;
       ROOTSERVICE_EVENT_ADD("connectivity_check", "permission check", 
           "tenant_id", tenant_id_, "error_code", ret, "comment", "write single file");
-      ret = OB_BACKUP_PERMISSION_DENIED;
     }
     LOG_WARN("failed to write single file", K(ret), K_(tenant_id), K(backup_dest));
   } else if (FALSE_IT(write_ok = true)
       || OB_FAIL(util.adaptively_get_file_length(path.get_obstr(), backup_dest.get_storage_info(), file_len))) {
     if (is_permission_error_(ret)) { 
+      ret = OB_BACKUP_PERMISSION_DENIED;
       ROOTSERVICE_EVENT_ADD("connectivity_check", "permission check", 
           "tenant_id", tenant_id_, "error_code", ret, "comment", "get file length");
-      ret = OB_BACKUP_PERMISSION_DENIED;
     }
     LOG_WARN("failed to get file length", K(ret));
   } else if (OB_ISNULL(buf = reinterpret_cast<char*>(allocator.alloc(file_len)))) {
@@ -608,17 +660,17 @@ int ObBackupCheckFile::check_io_permission(const ObBackupDest &backup_dest)
     LOG_WARN("failed to alloc buf", K(ret), K(file_len));
   } else if (OB_FAIL(util.adaptively_read_single_file(path.get_obstr(), backup_dest.get_storage_info(), buf, file_len, read_size))) {
     if (is_permission_error_(ret)) {
+      ret = OB_BACKUP_PERMISSION_DENIED;
       ROOTSERVICE_EVENT_ADD("connectivity_check", "permission check", 
           "tenant_id", tenant_id_, "error_code", ret, "comment", "read single file");
-      ret = OB_BACKUP_PERMISSION_DENIED;
     }
     LOG_WARN("failed to read single file", K(ret));
   }
   if (write_ok && (OB_SUCCESS != (tmp_ret = util.adaptively_del_file(path.get_obstr(), backup_dest.get_storage_info())))) {
     if (is_permission_error_(tmp_ret)) {
+      tmp_ret = OB_BACKUP_PERMISSION_DENIED;
       ROOTSERVICE_EVENT_ADD("connectivity_check", "permission check", 
           "tenant_id", tenant_id_, "error_code", tmp_ret, "comment", "delete file");
-      tmp_ret = OB_BACKUP_PERMISSION_DENIED;
     }
     ret = (OB_SUCCESS == ret) ? tmp_ret : ret;
     LOG_WARN("failed to del file", K(tmp_ret), K(ret), K(path), K(backup_dest));
@@ -626,11 +678,18 @@ int ObBackupCheckFile::check_io_permission(const ObBackupDest &backup_dest)
   if (OB_FAIL(ret)) {
   } else if (OB_FAIL(check_appender_permission_(backup_dest))){
     if (is_permission_error_(ret)) {
+      ret = OB_BACKUP_PERMISSION_DENIED;
       ROOTSERVICE_EVENT_ADD("connectivity_check", "permission check",
           "tenant_id", tenant_id_, "error_code", ret, "comment", "appender write");
-      ret = OB_BACKUP_PERMISSION_DENIED;
     }
     LOG_WARN("failed to appender permission", K(ret));
+  } else if (OB_FAIL(check_multipart_upload_permission_(backup_dest))) {
+    if (is_permission_error_(ret)) {
+      ret = OB_BACKUP_PERMISSION_DENIED;
+      ROOTSERVICE_EVENT_ADD("connectivity_check", "permission check",
+          "tenant_id", tenant_id_, "error_code", ret, "comment", "multipart upload write");
+    }
+    LOG_WARN("failed to check multipart permission", K(ret), K(backup_dest));
   }
 
   return ret;
@@ -1068,3 +1127,5 @@ int ObBackupStorageInfoOperator::get_backup_dest(
   }
   return ret;
 }
+}//share
+}//oceanbase

src/share/backup/ob_backup_connectivity.h

@@ -67,9 +67,10 @@ public:
 private:
   int get_permission_check_file_path_(
       const share::ObBackupDest &backup_dest,
-      bool is_appender,
+      const ObStorageAccessType access_type,
       share::ObBackupPath &path);
   int check_appender_permission_(const share::ObBackupDest &backup_dest);
+  int check_multipart_upload_permission_(const share::ObBackupDest &backup_dest);
   int set_connectivity_check_name_();
   int create_check_file_dir_(
       const share::ObBackupDest &backup_dest,

src/share/ob_errno.cpp

@@ -23044,10 +23044,10 @@ static const _error _error_OB_CHECKSUM_TYPE_NOT_SUPPORTED = {
       .mysql_errno           = -1,
       .sqlstate              = "HY000",
       .str_error             = "checksum type is not supported",
-      .str_user_error        = "checksum type is not supported",
+      .str_user_error        = "checksum type %s is not supported for the input destination",
       .oracle_errno          = 600,
       .oracle_str_error      = "ORA-00600: internal error code, arguments: -9113, checksum type is not supported",
-      .oracle_str_user_error = "ORA-00600: internal error code, arguments: -9113, checksum type is not supported"
+      .oracle_str_user_error = "ORA-00600: internal error code, arguments: -9113, checksum type %s is not supported for the input destination"
 };
 static const _error _error_OB_INVALID_STORAGE_DEST = {
       .error_name            = "OB_INVALID_STORAGE_DEST",

src/share/ob_errno.def

@@ -2260,7 +2260,7 @@ DEFINE_ERROR_EXT_DEP(OB_ERR_CLONE_TENANT, -9110, -1, "HY000", "Tenant clone job
 DEFINE_ERROR_EXT_DEP(OB_ERR_TENANT_SNAPSHOT, -9111, -1, "HY000", "Tenant snapshot task failed", "%.*s");
 DEFINE_ERROR_EXT_DEP(OB_TENANT_SNAPSHOT_LOCK_CONFLICT, -9112, -1, "HY000", "Tenant snapshot lock conflict", "%s");
 
-DEFINE_ERROR_DEP(OB_CHECKSUM_TYPE_NOT_SUPPORTED, -9113, -1, "HY000", "checksum type is not supported");
+DEFINE_ERROR_EXT_DEP(OB_CHECKSUM_TYPE_NOT_SUPPORTED, -9113, -1, "HY000", "checksum type is not supported", "checksum type %s is not supported for the input destination");
 DEFINE_ERROR_DEP(OB_INVALID_STORAGE_DEST, -9114, -1, "HY000", "storage destination is not valid");
 DEFINE_ERROR(OB_STORAGE_DEST_NOT_CONNECT, -9115, -1, "HY000", "can not connect to storage destination");
 DEFINE_ERROR_DEP(OB_OBJECT_STORAGE_PERMISSION_DENIED, -9116, -1, "HY000", "no I/O operation permission of the object storage");

src/share/ob_errno.h

@@ -3780,7 +3780,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219;
 #define OB_ERR_CLONE_TENANT__USER_ERROR_MSG "%.*s"
 #define OB_ERR_TENANT_SNAPSHOT__USER_ERROR_MSG "%.*s"
 #define OB_TENANT_SNAPSHOT_LOCK_CONFLICT__USER_ERROR_MSG "%s"
-#define OB_CHECKSUM_TYPE_NOT_SUPPORTED__USER_ERROR_MSG "checksum type is not supported"
+#define OB_CHECKSUM_TYPE_NOT_SUPPORTED__USER_ERROR_MSG "checksum type %s is not supported for the input destination"
 #define OB_INVALID_STORAGE_DEST__USER_ERROR_MSG "storage destination is not valid"
 #define OB_STORAGE_DEST_NOT_CONNECT__USER_ERROR_MSG "can not connect to storage destination"
 #define OB_OBJECT_STORAGE_PERMISSION_DENIED__USER_ERROR_MSG "no I/O operation permission of the object storage"
@@ -6010,7 +6010,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219;
 #define OB_ERR_CLONE_TENANT__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9110, %.*s"
 #define OB_ERR_TENANT_SNAPSHOT__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9111, %.*s"
 #define OB_TENANT_SNAPSHOT_LOCK_CONFLICT__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9112, %s"
-#define OB_CHECKSUM_TYPE_NOT_SUPPORTED__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9113, checksum type is not supported"
+#define OB_CHECKSUM_TYPE_NOT_SUPPORTED__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9113, checksum type %s is not supported for the input destination"
 #define OB_INVALID_STORAGE_DEST__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9114, storage destination is not valid"
 #define OB_STORAGE_DEST_NOT_CONNECT__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9115, can not connect to storage destination"
 #define OB_OBJECT_STORAGE_PERMISSION_DENIED__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9116, no I/O operation permission of the object storage"