hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Correct NFS's Erroneous Verification of Simulated Appendable Files

Browse Source
This commit is contained in:
obdev
2024-02-21 04:21:27 +00:00
committed by ob-robot
parent e1d63f0600
commit 2cafbe380d
3 changed files with 28 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
deps/oblib/src/lib/restore/ob_object_device.h vendored
View File

@ -62,6 +62,10 @@ public:
virtual int close(const ObIOFd &fd) override;
virtual int mkdir(const char *pathname, mode_t mode) override;
virtual int rmdir(const char *pathname) override;
// When attempting to delete a non-existent file, NFS will return an OB_BACKUP_FILE_NOT_EXIST error.
// OSS/COS/S3/OBS will not report any error, while GCS will return a 'file not found' error.
// Since GCS is accessed using the S3 SDK, to maintain consistency across different object storage services
// that are accessed via the S3 SDK, no error code is returned when attempting to delete a non-existent object.
virtual int unlink(const char *pathname) override;
virtual int exist(const char *pathname, bool &is_exist) override;
//sync io interfaces

18
deps/oblib/src/lib/restore/ob_storage_file.cpp vendored
View File

@ -518,16 +518,18 @@ int ObStorageFileUtil::check_is_appendable(
{
int ret = OB_SUCCESS;
ObStorageObjectMetaBase obj_meta;
char logic_apendable_obj_name[OB_MAX_URI_LENGTH] = { 0 };
char tmp_uri_buf[OB_MAX_URI_LENGTH] = "";
int pos = snprintf(tmp_uri_buf, OB_MAX_URI_LENGTH, "%s/%s/%s%s", uri.ptr(), cur_entry.d_name,
OB_S3_APPENDABLE_FRAGMENT_PREFIX, OB_S3_APPENDABLE_FORMAT_META);
if (pos < 0 || pos >= OB_MAX_URI_LENGTH) {
ret = OB_BUF_NOT_ENOUGH;
OB_LOG(WARN, "fail to build format meta file path", K(ret), K(pos), K(uri), KCSTRING(cur_entry.d_name));
if (OB_FAIL(databuff_printf(logic_apendable_obj_name, sizeof(logic_apendable_obj_name), "%s/%s",
uri.ptr(), cur_entry.d_name))) {
OB_LOG(WARN, "fail to construct logic_apendable_obj_name", K(ret), K(uri), K(cur_entry.d_name));
} else if (OB_FAIL(construct_fragment_full_name(logic_apendable_obj_name,
OB_S3_APPENDABLE_FORMAT_META,
tmp_uri_buf, sizeof(tmp_uri_buf)))) {
OB_LOG(WARN, "fail to construct fragment full name", K(ret), K(uri), K(cur_entry.d_name));
} else {
common::ObString format_meta_uri(pos, tmp_uri_buf);
if (OB_FAIL(head_object_meta(format_meta_uri, obj_meta))) {
OB_LOG(WARN, "fail to head object meta", K(ret), K(format_meta_uri));
if (OB_FAIL(head_object_meta(tmp_uri_buf, obj_meta))) {
OB_LOG(WARN, "fail to head object meta", K(ret), K(tmp_uri_buf));
} else {
is_appendable_file = obj_meta.is_exist_;
}

19
deps/oblib/src/lib/restore/ob_storage_s3_base.cpp vendored
View File

@ -115,7 +115,7 @@ int ObS3Client::init_s3_client_configuration_(const ObS3Account &account,
} else {
config.region = account.region_;
config.scheme = Aws::Http::Scheme::HTTP; // if change to HTTPS, be careful about checksum logic.
config.verifySSL = false;
config.verifySSL = true;
config.connectTimeoutMs = S3_CONNECT_TIMEOUT_MS;
config.requestTimeoutMs = S3_REQUEST_TIMEOUT_MS;
config.maxConnections = MAX_S3_CONNECTIONS_PER_CLIENT;
@ -1002,7 +1002,6 @@ static int init_put_object_request(const char *bucket, const char *object,
data_stream->write(buf, size);
data_stream->flush();
request.SetBody(data_stream);
request.SetAdditionalCustomHeaderValue("Connection", "keep-alive");
request.SetContentLength(static_cast<long>(request.GetBody()->tellp()));
}
return ret;
@ -1154,10 +1153,11 @@ int ObStorageS3Reader::pread_(char *buf,
} else {
Aws::S3::Model::GetObjectRequest request;
Aws::S3::Model::GetObjectOutcome outcome;
request.SetChecksumMode(Aws::S3::Model::ChecksumMode::ENABLED);
if (get_data_size == file_length_) {
request.SetChecksumMode(Aws::S3::Model::ChecksumMode::ENABLED);
}
char range_read[64] = { 0 };
request.WithBucket(bucket_.ptr()).WithKey(object_.ptr());
request.SetAdditionalCustomHeaderValue("Connection", "keep-alive");
if (OB_FAIL(databuff_printf(range_read, sizeof(range_read),
"bytes=%ld-%ld", offset, offset + get_data_size - 1))) {
@ -1331,7 +1331,16 @@ int ObStorageS3Util::del_file_(const ObString &uri)
const int64_t delete_mode = s3_base.s3_account_.delete_mode_;
if (ObIStorageUtil::DELETE == delete_mode) {
if (OB_FAIL(delete_object_(s3_base))) {
OB_LOG(WARN, "failed to delete s3 object", K(ret), K(uri));
if (OB_BACKUP_FILE_NOT_EXIST == ret) {
// Uniform handling of 'object not found' scenarios across different object storage services:
// GCS returns the OB_BACKUP_FILE_NOT_EXIST error when an object does not exist,
// whereas other object storage services may not report an error.
// Therefore, to maintain consistency,
// no error code is returned when attempting to delete a non-existent object
ret = OB_SUCCESS;
} else {
OB_LOG(WARN, "failed to delete s3 object", K(ret), K(uri));
}
}
} else if (ObIStorageUtil::TAGGING == delete_mode) {
if (OB_FAIL(tagging_object_(s3_base))) {