hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

remove oss info.

Browse Source
This commit is contained in:
HaHaJeff
2023-08-22 10:10:35 +00:00
committed by ob-robot
parent ad74fb0068
commit 068a6d6bf5
3 changed files with 0 additions and 294 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
unittest/archive/CMakeLists.txt
View File

@ -1,2 +1 @@
# ob_unittest(test_archive_task)
ob_unittest(test_archive_overwrite)

87
unittest/archive/test_archive_overwrite.cpp
View File

@ -1,87 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "lib/allocator/ob_malloc.h"
#include "lib/ob_errno.h"
#include "lib/oblog/ob_log.h"
#include "lib/string/ob_string.h"
#include "share/backup/ob_backup_io_adapter.h"
#include "share/backup/ob_backup_struct.h"
#include "logservice/archiveservice/ob_archive_io.h"
#include <cstdlib>
#include <gtest/gtest.h>
#include <string>
using namespace oceanbase;
using namespace common;
using namespace share;
using namespace archive;
using namespace std;
TEST(TestObArchiveOverwrite, oss_push_log)
{
int64_t rand_num = rand();
ObBackupDest dest;
ObBackupIoAdapter util;
ObArchiveIO archive_io;
const int64_t part_size = 5 * 1024 * 1024L;
const int64_t total_size = 10 * 1024 * 1024L;
char *data = (char*)ob_malloc(total_size, "ArchiveTest");
string path = "oss://antsys-oceanbasebackup/test_archive/test_archive_overwrite/" + to_string(rand_num) + "?host=cn-hangzhou-alipay-b.oss-cdn.aliyun-inc.com&access_id=LTAI4Fdwx9iFgZso4CqyHPs7&" + "access_" + "key=ER51kMn" + "lmu3zXwcxczJ" + "MbYzJIgrY9O";
EXPECT_EQ(false, NULL == data);
EXPECT_EQ(0, dest.set(path.c_str()));
const ObString &uri = dest.get_root_path();
const ObBackupStorageInfo *storage_info = dest.get_storage_info();
// delete file
EXPECT_EQ(0, util.del_file(uri, storage_info));
// scene 1: append & append
EXPECT_EQ(0, archive_io.push_log(uri, storage_info, data, part_size, 0, false /*append*/));
EXPECT_EQ(0, archive_io.push_log(uri, storage_info, data, total_size, 0, false /*append*/));
// delete file
EXPECT_EQ(0, util.del_file(uri, storage_info));
// scene 2: append & put
EXPECT_EQ(0, archive_io.push_log(uri, storage_info, data, part_size, 0, false /*append*/));
EXPECT_EQ(0, archive_io.push_log(uri, storage_info, data, total_size, 0, true /*put*/));
// delete file
EXPECT_EQ(0, util.del_file(uri, storage_info));
// scene 3: put & put
EXPECT_EQ(0, archive_io.push_log(uri, storage_info, data, total_size, 0, true /*put*/));
EXPECT_EQ(0, archive_io.push_log(uri, storage_info, data, total_size, 0, true /*put*/));
// delete file
EXPECT_EQ(0, util.del_file(uri, storage_info));
// scene 4: put & append
EXPECT_EQ(0, archive_io.push_log(uri, storage_info, data, total_size, 0, true /*put*/));
EXPECT_EQ(0, archive_io.push_log(uri, storage_info, data, part_size, 0, false /*append*/));
// delete file
EXPECT_EQ(0, util.del_file(uri, storage_info));
if (NULL != data) {
ob_free(data);
data = NULL;
}
}
int main(int argc, char **argv)
{
ObLogger::get_logger().set_file_name("test_archive_overwrite.log", true);
ObLogger::get_logger().set_log_level("INFO");
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

206
unittest/logservice/test_log_external_storage_handler.cpp
View File

@ -299,212 +299,6 @@ TEST(TestLogExternalStorageHandler, test_log_external_storage_handler)
handler.handle_adapter_ = origin_adapter;
}
// 测试真实的oss object
TEST(TestLogExternalStorageHandler, test_oss_object)
{
// 验证oss 不可访问
{
// 构造错误的oss meta信息
CLOG_LOG(INFO, "test oss can not access");
std::string trick_code_first = "host=123&acc";
std::string trick_code_second = "ess_id=111&acce";
std::string trick_code_third = "ss_key=222";
std::string trick_code_fourth = trick_code_first + trick_code_second + trick_code_third;
ObString oss_path(trick_code_fourth.c_str());
ObString uri("oss://backup_dir/1234");
ObLogExternalStorageIOTaskHandleAdapter adapter;
bool exist = false;
EXPECT_EQ(OB_OSS_ERROR, adapter.exist(uri, oss_path, exist));
EXPECT_EQ(false, exist);
int64_t file_size = 0;
EXPECT_EQ(OB_OSS_ERROR, adapter.get_file_size(uri, oss_path, file_size));
EXPECT_EQ(0, file_size);
const int64_t buf_len = 4096;
char buf[buf_len];
int64_t real_read_size = 0;
EXPECT_EQ(OB_OSS_ERROR, adapter.pread(uri, oss_path, 0, buf, buf_len, real_read_size));
EXPECT_EQ(0, real_read_size);
}
// 验证oss 可访问
// 需要保证如下的OSS能持续访问
{
CLOG_LOG(INFO, "test oss can access");
int ret = OB_SUCCESS;
const char * oss_bucket="oss://antsys-oceanbasebackup/shuning.tsn/";
const char *oss_host="cn-hangzhou-alipay-b.oss-cdn.aliyun-inc.com";
std::string trick_id_first = "LTAI4Fdwx";
std::string trick_id_second = "9iFgZso4";
std::string trick_id_third = "CqyHPs7";
std::string oss_id = trick_id_first + trick_id_second + trick_id_third;
std::string trick_key_first = "ER51kMnlmu";
std::string trick_key_second = "3zXwcxczJMb";
std::string trick_key_third = "YzJIgrY9O";
std::string oss_key = trick_key_first + trick_key_second + trick_key_third;
std::string trick_code_first = "host=cn-hangzhou-alipay-b.oss-cdn.aliyun-inc.com&acc";
std::string trick_code_second = "ess_id=" + oss_id + "&acce";
std::string trick_code_third = "ss_key=" + oss_key;
std::string trick_code_fourth = trick_code_first + trick_code_second + trick_code_third;
ObString oss_path(trick_code_fourth.c_str());
ObLogExternalStorageIOTaskHandleAdapter adapter;
std::vector<std::string> uris;
std::vector<int64_t> object_checksums;
std::string base_uri = "oss://antsys-oceanbasebackup/shuning.tsn/runlin_test_pf/";
const int64_t total_oss_object = 5;
// 测试目录下存在5个clog文件
for (int i = 0; i < total_oss_object; i++) {
uris.push_back(base_uri+std::to_string(i));
}
int exist_num = 0;
const int64_t buf_len = 64*1024*1024;
char *buf_self_pread = reinterpret_cast<char*>(ob_malloc(buf_len, "unittest"));
memset(buf_self_pread, 'x', buf_len);
// oss地址可能变化，当oss可访问时，进行如下测试
for (int i = 0; i < total_oss_object; i++) {
bool exist = false;
int64_t real_read_size = 0;
// farm环境写oss太慢了，暂时先单测不运行
// if (OB_FAIL(generate_oss_data(uris[i].c_str(), oss_path, buf_self_pread, buf_len))) {
// CLOG_LOG(ERROR, "oss can not access", K(uris[i].c_str()), K(oss_path), K(exist));
// } else {
// int64_t checksum = ob_crc64(buf_self_pread, buf_len);
// object_checksums.push_back(checksum);
// exist_num++;
// memset(buf_self_pread, 'y', buf_len);
// }
}
if (exist_num == total_oss_object)
{
int64_t real_read_size = 0;
// 读的offer为文件尾，读长度为0
EXPECT_EQ(OB_SUCCESS, adapter.pread(uris[0].c_str(), oss_path, buf_len, buf_self_pread, buf_len, real_read_size));
EXPECT_EQ(0, real_read_size);
CLOG_LOG(INFO, "read outof upper bound", K(ret), K(real_read_size));
// 读的offer为文件尾-1000，读长度为0
EXPECT_EQ(OB_SUCCESS, adapter.pread(uris[0].c_str(), oss_path, buf_len-1000, buf_self_pread, buf_len, real_read_size));
EXPECT_EQ(1000, real_read_size);
// 读的offer为文件尾-4*1000*1000，读长度为0
EXPECT_EQ(OB_SUCCESS, adapter.pread(uris[0].c_str(), oss_path, buf_len-1000*1000*4, buf_self_pread, buf_len, real_read_size));
EXPECT_EQ(1000*1000*4, real_read_size);
// 读的offer为文件尾+1000，读长度为0, 返回invalid argument
// 底层读接口会判断读取的offset是否超过文件长度，超过报错invalid argment
// OSS的feature: 如果指定的offset超过文件长度或者offset+待读取长度大于文件长度，会返回整个文件的数据内容且不会报错
EXPECT_EQ(OB_INVALID_ARGUMENT, adapter.pread(uris[0].c_str(), oss_path, buf_len+1000, buf_self_pread, buf_len, real_read_size));
// 读的offer为文件尾+1000*1000*4，读长度为0, 返回invalid argument
EXPECT_EQ(OB_INVALID_ARGUMENT, adapter.pread(uris[0].c_str(), oss_path, buf_len+1000*1000*4, buf_self_pread, buf_len, real_read_size));
CLOG_LOG(INFO, "read outof upper bound", K(ret), K(real_read_size));
}
if (exist_num == total_oss_object)
// 验证性能
{
EXPECT_EQ(OB_SUCCESS, ObDeviceManager::get_instance().init_devices_env());
ObLogExternalStorageHandler handler;
const int64_t concurrency = 16;
EXPECT_EQ(OB_SUCCESS, handler.init());
EXPECT_EQ(OB_SUCCESS, handler.start(concurrency));
int64_t start_ts = ObTimeUtility::current_time();
int64_t real_read_size = 0;
for (int i = 0; i < total_oss_object; i++) {
EXPECT_EQ(OB_SUCCESS, handler.pread(
uris[i].c_str(), oss_path, 0, buf_self_pread, buf_len, real_read_size
));
EXPECT_EQ(real_read_size, buf_len);
}
int64_t cost_ts = ObTimeUtility::current_time() - start_ts;
int64_t bw = total_oss_object * buf_len / cost_ts;
CLOG_LOG(INFO, "band width", K(bw), K(cost_ts));
}
// 验证读取正确性
if (exist_num == total_oss_object)
{
ObLogExternalStorageHandler handler;
const int64_t concurrency = 16;
EXPECT_EQ(OB_SUCCESS, handler.init());
EXPECT_EQ(OB_SUCCESS, handler.start(concurrency));
ASSERT_NE(nullptr, buf_self_pread);
for (int i = 0; i < total_oss_object; i++) {
int64_t real_read_size = 0;
memset(buf_self_pread, '0', buf_len);
EXPECT_EQ(OB_SUCCESS, handler.pread(
uris[i].c_str(), oss_path, 0, buf_self_pread, buf_len, real_read_size
));
EXPECT_EQ(buf_len, real_read_size);
int64_t checksum = ob_crc64(buf_self_pread, buf_len);
EXPECT_EQ(checksum, object_checksums[i]);
}
if (NULL != buf_self_pread) {
ob_free(buf_self_pread);
buf_self_pread = NULL;
}
// 随机位置读取
{
auto read_func = [&]() {
for (int i = 0; i < 32; i++) {
srandom(ObTimeUtility::current_time());
const int64_t object_idx = random() % total_oss_object;
const int64_t file_size = palf::PALF_PHY_BLOCK_SIZE;
int64_t tmp_read_offset = random() % file_size;
const int64_t tmp_read_size = random() % file_size;
char *tmp_read_buf1 = reinterpret_cast<char*>(ob_malloc(tmp_read_size, "unittest"));
char *tmp_read_buf2 = reinterpret_cast<char*>(ob_malloc(tmp_read_size, "unittest"));
int64_t tmp_real_read_size = 0;
ASSERT_NE(nullptr, tmp_read_buf1);
ASSERT_NE(nullptr, tmp_read_buf2);
EXPECT_EQ(OB_SUCCESS, handler.pread(
uris[object_idx].c_str(), oss_path, tmp_read_offset, tmp_read_buf1, tmp_read_size, tmp_real_read_size
));
EXPECT_EQ(std::min(file_size - tmp_read_offset, tmp_read_size),
tmp_real_read_size);
tmp_real_read_size = 0;
EXPECT_EQ(OB_SUCCESS, adapter.pread(
uris[object_idx].c_str(), oss_path, tmp_read_offset, tmp_read_buf2, tmp_read_size, tmp_real_read_size
));
EXPECT_EQ(std::min(file_size - tmp_read_offset, tmp_read_size),
tmp_real_read_size);
EXPECT_EQ(0, memcmp(tmp_read_buf1, tmp_read_buf2, tmp_real_read_size));
if (NULL != tmp_read_buf1) {
ob_free(tmp_read_buf1);
tmp_read_buf1 = NULL;
}
if (NULL != tmp_read_buf2) {
ob_free(tmp_read_buf2);
tmp_read_buf2 = NULL;
}
}
};
std::vector<std::thread> tmp_pread_threads;
for (int i = 0; i < 4; i++) {
tmp_pread_threads.emplace_back(read_func);
}
auto tmp_resize_func = [&](){
for (int i = 0; i < 4; i++) {
srandom(ObTimeUtility::current_time());
handler.resize(random()%4);
usleep(1000 * 1000);
}
};
std::thread tmp_resize_thread(tmp_resize_func);
for (int i = 0; i < 4; i++) {
tmp_pread_threads[i].join();
}
tmp_resize_thread.join();
}
for (auto uri : uris) {
delete_oss_object(uri.c_str(), oss_path);
}
CLOG_LOG(INFO, "delete success", K(base_uri.c_str()), K(oss_path));
} else {
CLOG_LOG(ERROR, "the object of oss is not correct", K(exist_num), K(oss_path));
}
}
}
}
}