From 2cafbe380d4c46b16b48eb79ca8cdb1aa002a2b4 Mon Sep 17 00:00:00 2001 From: obdev Date: Wed, 21 Feb 2024 04:21:27 +0000 Subject: [PATCH] Correct NFS's Erroneous Verification of Simulated Appendable Files --- deps/oblib/src/lib/restore/ob_object_device.h | 4 ++++ .../oblib/src/lib/restore/ob_storage_file.cpp | 18 ++++++++++-------- .../src/lib/restore/ob_storage_s3_base.cpp | 19 ++++++++++++++----- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/deps/oblib/src/lib/restore/ob_object_device.h b/deps/oblib/src/lib/restore/ob_object_device.h index 2f8b8c1a9e..67d0d5b2dc 100644 --- a/deps/oblib/src/lib/restore/ob_object_device.h +++ b/deps/oblib/src/lib/restore/ob_object_device.h @@ -62,6 +62,10 @@ public: virtual int close(const ObIOFd &fd) override; virtual int mkdir(const char *pathname, mode_t mode) override; virtual int rmdir(const char *pathname) override; + // When attempting to delete a non-existent file, NFS will return an OB_BACKUP_FILE_NOT_EXIST error. + // OSS/COS/S3/OBS will not report any error, while GCS will return a 'file not found' error. + // Since GCS is accessed using the S3 SDK, to maintain consistency across different object storage services + // that are accessed via the S3 SDK, no error code is returned when attempting to delete a non-existent object. virtual int unlink(const char *pathname) override; virtual int exist(const char *pathname, bool &is_exist) override; //sync io interfaces diff --git a/deps/oblib/src/lib/restore/ob_storage_file.cpp b/deps/oblib/src/lib/restore/ob_storage_file.cpp index a2b06d2d5e..bd6516da52 100644 --- a/deps/oblib/src/lib/restore/ob_storage_file.cpp +++ b/deps/oblib/src/lib/restore/ob_storage_file.cpp @@ -518,16 +518,18 @@ int ObStorageFileUtil::check_is_appendable( { int ret = OB_SUCCESS; ObStorageObjectMetaBase obj_meta; + char logic_apendable_obj_name[OB_MAX_URI_LENGTH] = { 0 }; char tmp_uri_buf[OB_MAX_URI_LENGTH] = ""; - int pos = snprintf(tmp_uri_buf, OB_MAX_URI_LENGTH, "%s/%s/%s%s", uri.ptr(), cur_entry.d_name, - OB_S3_APPENDABLE_FRAGMENT_PREFIX, OB_S3_APPENDABLE_FORMAT_META); - if (pos < 0 || pos >= OB_MAX_URI_LENGTH) { - ret = OB_BUF_NOT_ENOUGH; - OB_LOG(WARN, "fail to build format meta file path", K(ret), K(pos), K(uri), KCSTRING(cur_entry.d_name)); + if (OB_FAIL(databuff_printf(logic_apendable_obj_name, sizeof(logic_apendable_obj_name), "%s/%s", + uri.ptr(), cur_entry.d_name))) { + OB_LOG(WARN, "fail to construct logic_apendable_obj_name", K(ret), K(uri), K(cur_entry.d_name)); + } else if (OB_FAIL(construct_fragment_full_name(logic_apendable_obj_name, + OB_S3_APPENDABLE_FORMAT_META, + tmp_uri_buf, sizeof(tmp_uri_buf)))) { + OB_LOG(WARN, "fail to construct fragment full name", K(ret), K(uri), K(cur_entry.d_name)); } else { - common::ObString format_meta_uri(pos, tmp_uri_buf); - if (OB_FAIL(head_object_meta(format_meta_uri, obj_meta))) { - OB_LOG(WARN, "fail to head object meta", K(ret), K(format_meta_uri)); + if (OB_FAIL(head_object_meta(tmp_uri_buf, obj_meta))) { + OB_LOG(WARN, "fail to head object meta", K(ret), K(tmp_uri_buf)); } else { is_appendable_file = obj_meta.is_exist_; } diff --git a/deps/oblib/src/lib/restore/ob_storage_s3_base.cpp b/deps/oblib/src/lib/restore/ob_storage_s3_base.cpp index 0c16320047..eeef46ebc0 100644 --- a/deps/oblib/src/lib/restore/ob_storage_s3_base.cpp +++ b/deps/oblib/src/lib/restore/ob_storage_s3_base.cpp @@ -115,7 +115,7 @@ int ObS3Client::init_s3_client_configuration_(const ObS3Account &account, } else { config.region = account.region_; config.scheme = Aws::Http::Scheme::HTTP; // if change to HTTPS, be careful about checksum logic. - config.verifySSL = false; + config.verifySSL = true; config.connectTimeoutMs = S3_CONNECT_TIMEOUT_MS; config.requestTimeoutMs = S3_REQUEST_TIMEOUT_MS; config.maxConnections = MAX_S3_CONNECTIONS_PER_CLIENT; @@ -1002,7 +1002,6 @@ static int init_put_object_request(const char *bucket, const char *object, data_stream->write(buf, size); data_stream->flush(); request.SetBody(data_stream); - request.SetAdditionalCustomHeaderValue("Connection", "keep-alive"); request.SetContentLength(static_cast(request.GetBody()->tellp())); } return ret; @@ -1154,10 +1153,11 @@ int ObStorageS3Reader::pread_(char *buf, } else { Aws::S3::Model::GetObjectRequest request; Aws::S3::Model::GetObjectOutcome outcome; - request.SetChecksumMode(Aws::S3::Model::ChecksumMode::ENABLED); + if (get_data_size == file_length_) { + request.SetChecksumMode(Aws::S3::Model::ChecksumMode::ENABLED); + } char range_read[64] = { 0 }; request.WithBucket(bucket_.ptr()).WithKey(object_.ptr()); - request.SetAdditionalCustomHeaderValue("Connection", "keep-alive"); if (OB_FAIL(databuff_printf(range_read, sizeof(range_read), "bytes=%ld-%ld", offset, offset + get_data_size - 1))) { @@ -1331,7 +1331,16 @@ int ObStorageS3Util::del_file_(const ObString &uri) const int64_t delete_mode = s3_base.s3_account_.delete_mode_; if (ObIStorageUtil::DELETE == delete_mode) { if (OB_FAIL(delete_object_(s3_base))) { - OB_LOG(WARN, "failed to delete s3 object", K(ret), K(uri)); + if (OB_BACKUP_FILE_NOT_EXIST == ret) { + // Uniform handling of 'object not found' scenarios across different object storage services: + // GCS returns the OB_BACKUP_FILE_NOT_EXIST error when an object does not exist, + // whereas other object storage services may not report an error. + // Therefore, to maintain consistency, + // no error code is returned when attempting to delete a non-existent object + ret = OB_SUCCESS; + } else { + OB_LOG(WARN, "failed to delete s3 object", K(ret), K(uri)); + } } } else if (ObIStorageUtil::TAGGING == delete_mode) { if (OB_FAIL(tagging_object_(s3_base))) {