hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[CP] disable OSS/COS crc checksum

Browse Source
This commit is contained in:
obdev
2024-04-16 10:56:55 +00:00
committed by ob-robot
parent 037fe7d9bb
commit 1044aef268
2 changed files with 21 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
deps/oblib/src/lib/restore/cos/ob_cos_wrapper.cpp vendored
View File

@ -329,6 +329,15 @@ int ObCosWrapper::create_cos_handle(
custom_mem.customFree(custom_mem.opaque, ctx); custom_mem.customFree(custom_mem.opaque, ctx);
ret = OB_ALLOCATE_MEMORY_FAILED; ret = OB_ALLOCATE_MEMORY_FAILED;
cos_warn_log("[COS]fail to create cos http controller, ret=%d\n", ret); cos_warn_log("[COS]fail to create cos http controller, ret=%d\n", ret);
// A separate instance of ctl->options is now allocated for each request,
// ensuring that disabling CRC checks is a request-specific action
// and does not impact the global setting for COS request options.
} else if (NULL ==
(ctx->options->ctl->options = cos_http_request_options_create(ctx->options->pool))) {
cos_pool_destroy(ctx->mem_pool);
custom_mem.customFree(custom_mem.opaque, ctx);
ret = OB_ALLOCATE_MEMORY_FAILED;
cos_warn_log("[COS]fail to create cos http request options, ret=%d\n", ret);
} else { } else {
*h = reinterpret_cast<Handle *>(ctx); *h = reinterpret_cast<Handle *>(ctx);
@ -349,7 +358,7 @@ int ObCosWrapper::create_cos_handle(
} else { } else {
cos_set_content_md5_enable(ctx->options->ctl, COS_FALSE); cos_set_content_md5_enable(ctx->options->ctl, COS_FALSE);
} }
ctx->options->ctl->options->enable_crc = true; ctx->options->ctl->options->enable_crc = false;
} }
} }
@ -410,30 +419,12 @@ int ObCosWrapper::put(
ret = OB_ALLOCATE_MEMORY_FAILED; ret = OB_ALLOCATE_MEMORY_FAILED;
cos_warn_log("[COS]fail to pack buf, ret=%d\n", ret); cos_warn_log("[COS]fail to pack buf, ret=%d\n", ret);
} else { } else {
uint64_t before_crc64 = cos_crc64(0, (void *)buf, buf_size);
cos_list_add_tail(&content->node, &buffer); cos_list_add_tail(&content->node, &buffer);
if (NULL == (cos_ret = cos_put_object_from_buffer(ctx->options, &bucket, &object, &buffer, nullptr, &resp_headers)) if (NULL == (cos_ret = cos_put_object_from_buffer(ctx->options, &bucket, &object, &buffer, nullptr, &resp_headers))
|| !cos_status_is_ok(cos_ret)) { || !cos_status_is_ok(cos_ret)) {
convert_io_error(cos_ret, ret); convert_io_error(cos_ret, ret);
cos_warn_log("[COS]fail to put one object to cos, ret=%d\n", ret); cos_warn_log("[COS]fail to put one object to cos, ret=%d\n", ret);
log_status(cos_ret, ret); log_status(cos_ret, ret);
char *resp_crc64 = (char*)(apr_table_get(resp_headers, COS_HASH_CRC64_ECMA));
uint64_t after_crc64 = cos_crc64(0, (void *)buf, buf_size);
cos_warn_log("[COS]=-=-=-=-=-=-=-=-=-=-= upload crc, ret=%d, before=%llu, after=%llu, resp=%s\n",
ret, before_crc64, after_crc64, resp_crc64);
cos_warn_log("[COS]=-=-=-=-=-=-=-=-=-=-= upload data, ret=%d, buf_size=%ld, buf=%s, buf=%p\n",
ret, buf_size, buf, buf);
cos_warn_log("[COS]=-=-=-=-=-=-=-=-=-=-= other, ret=%d, enable_crc=%d\n",
ret, ctx->options->ctl->options->enable_crc);
if (cos_ret != nullptr) {
if (cos_ret->code == -978
|| ((cos_ret->error_code != nullptr) && 0 == strcmp("InconsistentError", cos_ret->error_code))) {
sleep(5);
raise(SIGSEGV);
}
}
} }
} }
} }
@ -942,9 +933,6 @@ int ObCosWrapper::pread(
} else { } else {
apr_table_set(headers, COS_RANGE_KEY, range_size); apr_table_set(headers, COS_RANGE_KEY, range_size);
} }
} else {
// support crc checksum when reading entire object
ctx->options->ctl->options->enable_crc = true;
} }
if (OB_SUCCESS != ret) { if (OB_SUCCESS != ret) {
@ -953,42 +941,6 @@ int ObCosWrapper::pread(
convert_io_error(cos_ret, ret); convert_io_error(cos_ret, ret);
cos_warn_log("[COS]fail to get object to buffer, ret=%d\n", ret); cos_warn_log("[COS]fail to get object to buffer, ret=%d\n", ret);
log_status(cos_ret, ret); log_status(cos_ret, ret);
// FIXME @fangdan: 临时增加的debug info，发版前会删除
char *tmp_buf = static_cast<char *>(malloc(buf_size + 1));
memset(tmp_buf, 0, buf_size + 1);
int64_t tmp_buf_pos = 0;
int64_t size = 0;
cos_buf_t *content = NULL;
cos_list_for_each_entry(cos_buf_t, content, &buffer, node) {
size = cos_buf_size(content);
if (content->pos == nullptr) {
// ret = OB_OSS_ERROR;
cos_warn_log("[COS]=-=-=-=-=-=-=-=-=-=-= unexpected error, ret=%d, size=%ld\n", ret, size);
break;
} else {
memcpy(tmp_buf + tmp_buf_pos, content->pos, size);
tmp_buf_pos += size;
}
}
cos_warn_log("[COS]=-=-=-=-=-=-=-=-=-=-= read content, ret=%d, tmp_buf=%s, tmp_buf=%p, pos=%ld\n",
ret, tmp_buf, tmp_buf, tmp_buf_pos);
char *crc64 = (char*)(apr_table_get(resp_headers, COS_HASH_CRC64_ECMA));
uint64_t read_crc64 = cos_crc64(0, (void *)tmp_buf, tmp_buf_pos);
cos_warn_log("[COS]=-=-=-=-=-=-=-=-=-=-= resp crc64, ret=%d, crc64=%s, get_data_size=%ld, read_crc64=%llu\n",
ret, crc64, buf_size, read_crc64);
cos_warn_log("[COS]=-=-=-=-=-=-=-=-=-=-= other, ret=%d, enable_crc=%d, is_range_read=%d\n",
ret, ctx->options->ctl->options->enable_crc, is_range_read);
if (cos_ret != nullptr) {
if (cos_ret->code == -978
|| ((cos_ret->error_code != nullptr) && 0 == strcmp("InconsistentError", cos_ret->error_code))) {
sleep(5);
raise(SIGSEGV);
}
}
free(tmp_buf);
} else { } else {
read_size = 0; read_size = 0;
int64_t size = 0; int64_t size = 0;
@ -1586,31 +1538,12 @@ int ObCosWrapper::upload_part_from_buffer(
ret = OB_ALLOCATE_MEMORY_FAILED; ret = OB_ALLOCATE_MEMORY_FAILED;
cos_warn_log("[COS]fail to pack buf, ret=%d, buf_size=%d\n", ret, buf_size); cos_warn_log("[COS]fail to pack buf, ret=%d, buf_size=%d\n", ret, buf_size);
} else { } else {
uint64_t before_crc64 = cos_crc64(0, (void *)buf, buf_size);
cos_list_add_tail(&content->node, &buffer); cos_list_add_tail(&content->node, &buffer);
if (NULL == (cos_ret = cos_upload_part_from_buffer(ctx->options, &bucket, &object, &upload_id, part_num, &buffer, &resp_headers)) if (NULL == (cos_ret = cos_upload_part_from_buffer(ctx->options, &bucket, &object, &upload_id, part_num, &buffer, &resp_headers))
|| !cos_status_is_ok(cos_ret)) { || !cos_status_is_ok(cos_ret)) {
convert_io_error(cos_ret, ret); convert_io_error(cos_ret, ret);
cos_warn_log("[COS]fail to upload part to cos, ret=%d, part_num=%d\n", ret, part_num); cos_warn_log("[COS]fail to upload part to cos, ret=%d, part_num=%d\n", ret, part_num);
log_status(cos_ret, ret); log_status(cos_ret, ret);
char *resp_crc64 = (char*)(apr_table_get(resp_headers, COS_HASH_CRC64_ECMA));
uint64_t after_crc64 = cos_crc64(0, (void *)buf, buf_size);
cos_warn_log("[COS]=-=-=-=-=-=-=-=-=-=-= upload crc, ret=%d, before=%llu, after=%llu, resp=%s\n",
ret, before_crc64, after_crc64, resp_crc64);
cos_warn_log("[COS]=-=-=-=-=-=-=-=-=-=-= upload data, ret=%d, buf_size=%ld, buf=%s, buf=%p\n",
ret, buf_size, buf, buf);
cos_warn_log("[COS]=-=-=-=-=-=-=-=-=-=-= other, ret=%d, enable_crc=%d\n",
ret, ctx->options->ctl->options->enable_crc);
if (cos_ret != nullptr) {
if (cos_ret->code == -978
|| ((cos_ret->error_code != nullptr) && 0 == strcmp("InconsistentError", cos_ret->error_code))) {
sleep(5);
raise(SIGSEGV);
}
}
} }
} }
} }

77
deps/oblib/src/lib/restore/ob_storage_oss_base.cpp vendored
View File

@ -398,7 +398,6 @@ int ObStorageOssBase::init_with_storage_info(common::ObObjectStorageInfo *storag
OB_LOG(WARN, "that checksum algorithm is not supported for oss", K(ret), K_(checksum_type)); OB_LOG(WARN, "that checksum algorithm is not supported for oss", K(ret), K_(checksum_type));
} else { } else {
is_inited_ = true; is_inited_ = true;
oss_option_->ctl->options->enable_crc = true;
} }
} }
return ret; return ret;
@ -517,12 +516,20 @@ int ObStorageOssBase::init_oss_options(aos_pool_t *&aos_pool, oss_request_option
OB_LOG(WARN, "fail to create oss config", K(ret)); OB_LOG(WARN, "fail to create oss config", K(ret));
} else if (OB_FAIL(init_oss_endpoint())) { } else if (OB_FAIL(init_oss_endpoint())) {
OB_LOG(WARN, "fail to init oss endpoind", K(ret)); OB_LOG(WARN, "fail to init oss endpoind", K(ret));
} else if (OB_ISNULL(oss_option->ctl = aos_http_controller_create(oss_option->pool, 0))) {
ret = OB_OSS_ERROR;
OB_LOG(WARN, "fail to create aos http controller", K(ret));
// A separate instance of ctl->options is now allocated for each request,
// ensuring that disabling CRC checks is a request-specific action
// and does not impact the global setting for OSS request options.
} else if (OB_ISNULL(oss_option->ctl->options = aos_http_request_options_create(oss_option->pool))) {
ret = OB_OSS_ERROR;
OB_LOG(WARN, "fail to create aos http request options", K(ret));
} else { } else {
aos_str_set(&oss_option->config->endpoint, oss_endpoint_); aos_str_set(&oss_option->config->endpoint, oss_endpoint_);
aos_str_set(&oss_option->config->access_key_id, oss_account_.oss_id_); aos_str_set(&oss_option->config->access_key_id, oss_account_.oss_id_);
aos_str_set(&oss_option->config->access_key_secret, oss_account_.oss_key_); aos_str_set(&oss_option->config->access_key_secret, oss_account_.oss_key_);
oss_option->config->is_cname = 0; oss_option->config->is_cname = 0;
oss_option->ctl = aos_http_controller_create(oss_option->pool, 0);
// Set connection timeout, the default value is 10s // Set connection timeout, the default value is 10s
oss_option->ctl->options->connect_timeout = 60; oss_option->ctl->options->connect_timeout = 60;
@ -535,6 +542,8 @@ int ObStorageOssBase::init_oss_options(aos_pool_t *&aos_pool, oss_request_option
// The maximum time that the control can tolerate, the default is 15 seconds // The maximum time that the control can tolerate, the default is 15 seconds
oss_option->ctl->options->speed_limit = 16000; oss_option->ctl->options->speed_limit = 16000;
oss_option->ctl->options->speed_time = 60; oss_option->ctl->options->speed_time = 60;
oss_option->ctl->options->enable_crc = false;
} }
return ret; return ret;
} }
@ -849,8 +858,6 @@ int ObStorageOssMultiPartWriter::write_single_part()
&& OB_FAIL(add_content_md5(oss_option_, base_buf_, base_buf_pos_, headers))) { && OB_FAIL(add_content_md5(oss_option_, base_buf_, base_buf_pos_, headers))) {
OB_LOG(WARN, "fail to add content md5 when uploading part", K(ret)); OB_LOG(WARN, "fail to add content md5 when uploading part", K(ret));
} else { } else {
// FIXME @fangdan: 临时增加的debug info，发版前会删除
uint64_t before_crc64 = aos_crc64(0, (void *)base_buf_, base_buf_pos_);
aos_list_add_tail(&content->node, &buffer); aos_list_add_tail(&content->node, &buffer);
if (NULL == (aos_ret = oss_do_upload_part_from_buffer(oss_option_, &bucket, &object, if (NULL == (aos_ret = oss_do_upload_part_from_buffer(oss_option_, &bucket, &object,
@ -861,20 +868,6 @@ int ObStorageOssMultiPartWriter::write_single_part()
OB_LOG(WARN, "fail to upload one part from buffer", OB_LOG(WARN, "fail to upload one part from buffer",
K_(base_buf_pos), K_(bucket), K_(object), K(ret)); K_(base_buf_pos), K_(bucket), K_(object), K(ret));
print_oss_info(resp_headers, aos_ret, ret); print_oss_info(resp_headers, aos_ret, ret);
uint64_t after_crc64 = aos_crc64(0, (void *)base_buf_, base_buf_pos_);
char *resp_crc64 = (char*)(apr_table_get(resp_headers, OSS_HASH_CRC64_ECMA));
OB_LOG(WARN, "=-=-=-=-=-=-=-=-=-=-= upload crc", K(ret), K(before_crc64), K(after_crc64), K(resp_crc64));
OB_LOG(WARN, "=-=-=-=-=-=-=-=-=-=-= upload data", K(ret), K(base_buf_pos_), K(base_buf_), KP(base_buf_));
OB_LOG(WARN, "=-=-=-=-=-=-=-=-=-=-= other", K(ret), K(oss_option_->ctl->options->enable_crc));
if (OB_NOT_NULL(aos_ret)) {
if (aos_ret->code == -978
|| (OB_NOT_NULL(aos_ret->error_code) && 0 == STRCMP("InconsistentError", aos_ret->error_code))) {
sleep(5);
ob_abort();
}
}
} }
bool is_slow = false; bool is_slow = false;
print_access_storage_log("oss upload one part ", object_, start_time, base_buf_pos_, &is_slow); print_access_storage_log("oss upload one part ", object_, start_time, base_buf_pos_, &is_slow);
@ -1183,8 +1176,6 @@ int ObStorageOssReader::pread(
} else { } else {
apr_table_set(headers, OSS_RANGE_KEY, range_size); apr_table_set(headers, OSS_RANGE_KEY, range_size);
} }
} else {
oss_option->ctl->options->enable_crc = true;
} }
if (OB_FAIL(ret)) { if (OB_FAIL(ret)) {
@ -1193,36 +1184,6 @@ int ObStorageOssReader::pread(
convert_io_error(aos_ret, ret); convert_io_error(aos_ret, ret);
OB_LOG(WARN, "fail to get object to buffer", K_(bucket), K_(object), K(ret)); OB_LOG(WARN, "fail to get object to buffer", K_(bucket), K_(object), K(ret));
print_oss_info(resp_headers, aos_ret, ret); print_oss_info(resp_headers, aos_ret, ret);
ObArenaAllocator allocator;
char *tmp_buf = static_cast<char *>(allocator.alloc(buf_size + 1));
MEMSET(tmp_buf, 0, buf_size + 1);
int64_t tmp_buf_pos = 0;
aos_list_for_each_entry(aos_buf_t, content, &buffer, node) {
size = aos_buf_size(content);
if (OB_ISNULL(content->pos)) {
// ret = OB_OSS_ERROR;
OB_LOG(WARN, "=-=-=-=-=-=-=-=-=-=-= unexpected error, the data pos is null", K(size), K(ret));
break;
} else {
memcpy(tmp_buf + tmp_buf_pos, content->pos, size);
tmp_buf_pos += size;
}
}
OB_LOG(WARN, "=-=-=-=-=-=-=-=-=-=-= read content", K(ret), K(tmp_buf), KP(tmp_buf), K(file_length_), K(tmp_buf_pos));
char *crc64 = (char*)(apr_table_get(resp_headers, OSS_HASH_CRC64_ECMA));
uint64_t read_crc64 = aos_crc64(0, tmp_buf, tmp_buf_pos);
// std::cout << "=========== resp crc64=" << crc64 << std::endl;;
OB_LOG(WARN, "=-=-=-=-=-=-=-=-=-=-= resp crc64", K(ret), K(crc64), K(get_data_size), K(read_crc64));
OB_LOG(WARN, "=-=-=-=-=-=-=-=-=-=-= other", K(ret), K(oss_option->ctl->options->enable_crc), K(is_range_read));
if (OB_NOT_NULL(aos_ret)) {
if (aos_ret->code == -978
|| (OB_NOT_NULL(aos_ret->error_code) && 0 == STRCMP("InconsistentError", aos_ret->error_code))) {
sleep(5);
ob_abort();
}
}
} else { } else {
//check date len //check date len
aos_list_for_each_entry(aos_buf_t, content, &buffer, node) { aos_list_for_each_entry(aos_buf_t, content, &buffer, node) {
@ -2321,28 +2282,12 @@ int ObStorageOssWriter::write(const char *buf, const int64_t size)
&& OB_FAIL(add_content_md5(oss_option_, buf, size, headers))) { && OB_FAIL(add_content_md5(oss_option_, buf, size, headers))) {
OB_LOG(WARN, "fail to add content md5 when putting object", K(ret)); OB_LOG(WARN, "fail to add content md5 when putting object", K(ret));
} else { } else {
uint64_t before_crc64 = aos_crc64(0, (void *)buf, size);
aos_list_add_tail(&content->node, &buffer); aos_list_add_tail(&content->node, &buffer);
if (OB_ISNULL(aos_ret = oss_put_object_from_buffer(oss_option_, &bucket, &object, if (OB_ISNULL(aos_ret = oss_put_object_from_buffer(oss_option_, &bucket, &object,
&buffer, headers, &resp_headers)) || !aos_status_is_ok(aos_ret)) { &buffer, headers, &resp_headers)) || !aos_status_is_ok(aos_ret)) {
convert_io_error(aos_ret, ret); convert_io_error(aos_ret, ret);
OB_LOG(WARN, "fail to upload one object", K(bucket_), K(object_), K(ret)); OB_LOG(WARN, "fail to upload one object", K(bucket_), K(object_), K(ret));
print_oss_info(resp_headers, aos_ret, ret); print_oss_info(resp_headers, aos_ret, ret);
uint64_t after_crc64 = aos_crc64(0, (void *)buf, size);
char *resp_crc64 = (char*)(apr_table_get(resp_headers, OSS_HASH_CRC64_ECMA));
OB_LOG(WARN, "=-=-=-=-=-=-=-=-=-=-= write crc", K(ret), K(before_crc64), K(after_crc64), K(resp_crc64));
OB_LOG(WARN, "=-=-=-=-=-=-=-=-=-=-= write data", K(ret), K(size), K(buf), KP(buf));
OB_LOG(WARN, "=-=-=-=-=-=-=-=-=-=-= other", K(ret), K(oss_option_->ctl->options->enable_crc));
if (OB_NOT_NULL(aos_ret)) {
if (aos_ret->code == -978
|| (OB_NOT_NULL(aos_ret->error_code) && 0 == STRCMP("InconsistentError", aos_ret->error_code))) {
sleep(5);
ob_abort();
}
}
} else { } else {
file_length_ = size; file_length_ = size;
} }