/** * Copyright (c) 2021 OceanBase * OceanBase CE is licensed under Mulan PubL v2. * You can use this software according to the terms and conditions of the Mulan PubL v2. * You may obtain a copy of Mulan PubL v2 at: * http://license.coscl.org.cn/MulanPubL-2.0 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. * See the Mulan PubL v2 for more details. */ #define USING_LOG_PREFIX COMMON #include #include #define private public #include "share/io/ob_io_define.h" #include "share/io/ob_io_manager.h" #include "share/io/ob_io_calibration.h" #include "share/io/io_schedule/ob_io_mclock.h" #include "share/resource_manager/ob_cgroup_ctrl.h" #include "mittest/mtlenv/mock_tenant_module_env.h" #undef private #include "share/ob_local_device.h" #include "lib/thread/thread_pool.h" #include "lib/file/file_directory_utils.h" #include "common/ob_clock_generator.h" #include "storage/blocksstable/ob_micro_block_cache.h" #include "storage/blocksstable/ob_tmp_file_cache.h" #include "storage/blocksstable/ob_tmp_file_store.h" #include "storage/meta_mem/ob_storage_meta_cache.h" #define ASSERT_SUCC(ret) ASSERT_EQ((ret), ::oceanbase::common::OB_SUCCESS) #define ASSERT_FAIL(ret) ASSERT_NE((ret), ::oceanbase::common::OB_SUCCESS) using namespace oceanbase::lib; using namespace oceanbase::common; using namespace oceanbase::share; using namespace oceanbase::blocksstable; #define TEST_ROOT_DIR "./" #define TEST_DATA_DIR TEST_ROOT_DIR "/data_dir" #define TEST_SSTABLE_DIR TEST_DATA_DIR "/sstable" static const int64_t IO_MEMORY_LIMIT = 10L * 1024L * 1024L * 1024L; static const uint64_t TEST_TENANT_ID = 1; int init_device(const int64_t media_id, ObLocalDevice &device) { int ret = OB_SUCCESS; const int64_t IO_OPT_COUNT = 6; const int64_t block_size = 1024L * 1024L * 2L; // 2MB const int64_t data_disk_size = 1024L * 1024L * 1024L; // 1GB const int64_t data_disk_percentage = 50L; ObIODOpt io_opts[IO_OPT_COUNT]; io_opts[0].key_ = "data_dir"; io_opts[0].value_.value_str = oceanbase::MockTenantModuleEnv::get_instance().storage_env_.data_dir_; io_opts[1].key_ = "sstable_dir"; io_opts[1].value_.value_str = oceanbase::MockTenantModuleEnv::get_instance().storage_env_.sstable_dir_; io_opts[2].key_ = "block_size"; io_opts[2].value_.value_int64 = block_size; io_opts[3].key_ = "datafile_disk_percentage"; io_opts[3].value_.value_int64 = data_disk_percentage; io_opts[4].key_ = "datafile_size"; io_opts[4].value_.value_int64 = data_disk_size; io_opts[5].key_ = "media_id"; io_opts[5].value_.value_int64 = media_id; ObIODOpts init_opts; init_opts.opts_ = io_opts; init_opts.opt_cnt_ = IO_OPT_COUNT; bool need_format = false; if (OB_FAIL(device.init(init_opts))) { LOG_WARN("init device failed", K(ret)); } else { int64_t reserved_size = 0; ObIODOpts opts_start; ObIODOpt opt_start; opts_start.opts_ = &(opt_start); opts_start.opt_cnt_ = 1; opt_start.set("reserved size", reserved_size); if (OB_FAIL(device.start(opts_start))) { LOG_WARN("start device failed", K(ret)); } } return ret; } class TestIOStruct : public ::testing::Test { public: static void SetUpTestCase() { // prepare test directory and file system("mkdir -p " TEST_DATA_DIR); system("mkdir -p " TEST_SSTABLE_DIR); ASSERT_SUCC(oceanbase::MockTenantModuleEnv::get_instance().init()); ObMallocAllocator::get_instance()->create_and_add_tenant_allocator(1001); ObMallocAllocator::get_instance()->create_and_add_tenant_allocator(1002); ObMallocAllocator::get_instance()->get_tenant_ctx_allocator(OB_SERVER_TENANT_ID, 0)->set_limit(IO_MEMORY_LIMIT); } static void TearDownTestCase() { ObMallocAllocator::get_instance()->recycle_tenant_allocator(1001); ObMallocAllocator::get_instance()->recycle_tenant_allocator(1002); } static ObIOInfo get_random_io_info() { ObIOInfo io_info; const int64_t max_group_num = 0; //不包括OTHER GROUPS的其他group io_info.tenant_id_ = OB_SERVER_TENANT_ID; io_info.fd_.first_id_ = ObRandom::rand(0, 10000); io_info.fd_.second_id_ = ObRandom::rand(0, 10000); io_info.flag_.set_mode(static_cast(ObRandom::rand(0, (int)ObIOMode::MAX_MODE - 1))); io_info.flag_.set_group_id(0); // 0 means default io_info.flag_.set_wait_event(ObRandom::rand(1, 9999)); io_info.timeout_us_ = DEFAULT_IO_WAIT_TIME_US; io_info.offset_ = ObRandom::rand(1, 1000L * 1000L * 1000L); io_info.size_ = ObRandom::rand(1, 1000L * 10L); io_info.flag_.set_read(); io_info.user_data_buf_ = static_cast(ob_malloc(io_info.size_, ObNewModIds::TEST)); return io_info; } static ObTenantIOConfig default_tenant_io_config() { ObTenantIOConfig tenant_config; tenant_config.callback_thread_count_ = 2; tenant_config.memory_limit_ = 1024L * 1024L * 1024L; tenant_config.group_num_ = 0; tenant_config.unit_config_.min_iops_ = 1000; tenant_config.unit_config_.max_iops_ = 1000; tenant_config.unit_config_.weight_ = 1000; tenant_config.other_group_config_.min_percent_ = 100; tenant_config.other_group_config_.max_percent_ = 100; tenant_config.other_group_config_.weight_percent_ = 100; tenant_config.group_config_change_ = false; return tenant_config; } }; class TestIOCallback : public ObIOCallback { public: TestIOCallback() : number_(nullptr), allocator_(nullptr), help_buf_(nullptr) {} virtual ~TestIOCallback(); virtual const char *get_data() override { return (char *)help_buf_; } virtual int64_t size() const override { return sizeof(TestIOCallback); } virtual int alloc_data_buf(const char *io_data_buffer, const int64_t data_size) override; virtual int inner_process(const char *data_buffer, const int64_t size) override; virtual ObIAllocator *get_allocator() override { return allocator_; } TO_STRING_KV(KP(number_), KP(allocator_), KP(help_buf_)); public: int64_t *number_; ObIAllocator *allocator_; char *help_buf_; }; TEST_F(TestIOStruct, IOFlag) { // default invalid ObIOFlag flag; ObIOFlag flag2; ASSERT_FALSE(flag.is_valid()); // normal usage flag.set_mode(ObIOMode::READ); flag.set_group_id(0); flag.set_wait_event(99); ASSERT_TRUE(flag.is_valid()); // test io mode flag2 = flag; ASSERT_TRUE(flag2.is_valid()); flag2.set_mode(ObIOMode::MAX_MODE); ASSERT_FALSE(flag2.is_valid()); flag2.set_mode((ObIOMode)88); ASSERT_FALSE(flag2.is_valid()); // test io group flag2 = flag; ASSERT_TRUE(flag2.is_valid()); flag2.set_group_id(-1); ASSERT_FALSE(flag2.is_valid()); // test wait event number flag2 = flag; ASSERT_TRUE(flag2.is_valid()); flag2.set_wait_event(0); ASSERT_FALSE(flag2.is_valid()); flag2.set_wait_event(-100); ASSERT_FALSE(flag2.is_valid()); // test reset flag2 = flag; ASSERT_TRUE(flag2.is_valid()); flag2.reset(); ASSERT_FALSE(flag2.is_valid()); } TEST_F(TestIOStruct, IOInfo) { // default invalid ObIOInfo info; ASSERT_FALSE(info.is_valid()); // normal usage ObIOFd fd; fd.first_id_ = 0; fd.second_id_ = 0; info.tenant_id_ = OB_SERVER_TENANT_ID; info.fd_ = fd; info.flag_.set_mode(ObIOMode::READ); info.flag_.set_group_id(0); info.flag_.set_wait_event(1); info.timeout_us_ = DEFAULT_IO_WAIT_TIME_US; info.offset_ = 80; info.size_ = 1; char user_buf[1] = { 0 }; info.user_data_buf_ = user_buf; ASSERT_TRUE(info.is_valid()); // write io info require write buf info.flag_.set_mode(ObIOMode::WRITE); ASSERT_FALSE(info.is_valid()); char tmp_buf[10] = { 0 }; info.buf_ = tmp_buf; ASSERT_TRUE(info.is_valid()); // test reset info.reset(); ASSERT_FALSE(info.is_valid()); } TEST_F(TestIOStruct, IOHandle) { ObIOResult result; ObIORequest req; req.inc_ref(); // prevent from free result.inc_ref(); req.set_result(result); ASSERT_EQ(1, req.ref_cnt_); ASSERT_EQ(2, result.result_ref_cnt_); ASSERT_EQ(0, result.out_ref_cnt_); // default invalid ObIOHandle handle; ASSERT_FALSE(handle.is_valid()); ASSERT_TRUE(handle.is_empty()); // normal usage handle.set_result(result); ASSERT_TRUE(handle.is_valid()); ASSERT_FALSE(handle.is_empty()); ASSERT_EQ(3, result.result_ref_cnt_); ASSERT_EQ(1, result.out_ref_cnt_); // copy assign ObIOHandle handle2 = handle; ASSERT_EQ(4, result.result_ref_cnt_); ASSERT_EQ(2, result.out_ref_cnt_); // reset handle2.reset(); ASSERT_EQ(3, result.result_ref_cnt_); ASSERT_EQ(1, result.out_ref_cnt_); ASSERT_FALSE(handle2.is_valid()); ASSERT_TRUE(handle2.is_empty()); // test log LOG_INFO("test log io handle", K(handle), K(handle2)); } TEST_F(TestIOStruct, IOAllocator) { ObIOAllocator allocator; ASSERT_FALSE(allocator.is_inited_); // init const int64_t memory_limit = 1024L * 1024L * 1024L; // 1GB ASSERT_SUCC(allocator.init(TEST_TENANT_ID, memory_limit)); ASSERT_TRUE(allocator.is_inited_); // alloc and free memory void *buf = allocator.alloc(1000); ASSERT_NE(buf, nullptr); allocator.free(buf); // alloc and free object ObIORequest *req = nullptr; allocator.alloc(req); ASSERT_NE(req, nullptr); allocator.free(req); // destroy allocator.destroy(); ASSERT_FALSE(allocator.is_inited_); } TEST_F(TestIOStruct, IORequest) { ObRefHolder holder; OB_IO_MANAGER.get_tenant_io_manager(OB_SERVER_TENANT_ID, holder); ObTenantIOManager &tenant_io_mgr = *(holder.get_ptr()); ObIOFd fd; fd.first_id_ = 0; fd.second_id_ = 1; // default invalid ObIOResult result; ASSERT_FALSE(result.is_inited_); ASSERT_FALSE(result.is_valid()); ObIORequest req; ASSERT_FALSE(req.is_inited_); ASSERT_FALSE(req.is_valid()); // prepare read request req.destroy(); result.destroy(); req.tenant_io_mgr_.hold(&tenant_io_mgr); result.tenant_io_mgr_.hold(&tenant_io_mgr); result.inc_ref(); req.inc_ref(); ObIOInfo read_info; read_info.tenant_id_ = OB_SERVER_TENANT_ID; read_info.fd_ = fd; read_info.flag_.set_mode(ObIOMode::READ); read_info.flag_.set_group_id(0); read_info.flag_.set_wait_event(1); read_info.timeout_us_ = DEFAULT_IO_WAIT_TIME_US; read_info.offset_ = 89; read_info.size_ = 1; char user_buf[1] = { 0 }; read_info.user_data_buf_ = user_buf; ASSERT_TRUE(read_info.is_valid()); ASSERT_SUCC(req.tenant_io_mgr_.get_ptr()->io_usage_.init(0)); ASSERT_FAIL(result.init(read_info)); //not init cond yet ASSERT_FALSE(result.is_inited_); ASSERT_SUCC(result.basic_init()); ASSERT_SUCC(result.init(read_info)); ASSERT_SUCC(req.init(read_info ,&result)); ASSERT_TRUE(req.is_inited_); ASSERT_TRUE(result.is_inited_); ASSERT_TRUE(req.is_valid()); ASSERT_TRUE(result.is_valid()); ASSERT_EQ(req.raw_buf_, nullptr); // read buf allocation is delayed ASSERT_SUCC(req.prepare()); ASSERT_NE(req.raw_buf_, nullptr); ASSERT_EQ(2, result.result_ref_cnt_); // prepare write request ObIOInfo write_info = read_info; write_info.flag_.set_mode(ObIOMode::WRITE); req.destroy(); ASSERT_EQ(1, result.result_ref_cnt_); result.reset(); ASSERT_EQ(0, result.result_ref_cnt_); req.tenant_io_mgr_.hold(&tenant_io_mgr); result.tenant_io_mgr_.hold(&tenant_io_mgr); ASSERT_FAIL(result.init(write_info)); // not aligned ASSERT_TRUE(req.init(write_info ,&result)); ASSERT_EQ(1, result.result_ref_cnt_); //inc ref even fail write_info.offset_ = DIO_READ_ALIGN_SIZE * 2; req.reset(); result.reset(); req.tenant_io_mgr_.hold(&tenant_io_mgr); result.tenant_io_mgr_.hold(&tenant_io_mgr); ASSERT_FAIL(result.init(write_info)); // only offset aligned, size not aligned ASSERT_TRUE(req.init(write_info ,&result)); write_info.size_ = DIO_READ_ALIGN_SIZE * 4; req.reset(); result.reset(); req.tenant_io_mgr_.hold(&tenant_io_mgr); result.tenant_io_mgr_.hold(&tenant_io_mgr); ASSERT_FAIL(result.init(write_info)); // offset and size aligned, but write buf is null ASSERT_TRUE(req.init(write_info ,&result)); write_info.buf_ = "test_write"; req.reset(); result.reset(); req.tenant_io_mgr_.hold(&tenant_io_mgr); result.tenant_io_mgr_.hold(&tenant_io_mgr); ASSERT_SUCC(result.init(write_info)); ASSERT_SUCC(req.init(write_info ,&result)); // normal usage ASSERT_TRUE(req.is_inited_); ASSERT_TRUE(result.is_inited_); ASSERT_NE(req.raw_buf_, nullptr); // write buf need copy immediately req.reset(); } TEST_F(TestIOStruct, IOAbility) { ObIOBenchResult item, item2; ASSERT_FALSE(item.is_valid()); ASSERT_TRUE(item == item2); ObIOAbility io_ability; ASSERT_FALSE(io_ability.is_valid()); ASSERT_FAIL(io_ability.add_measure_item(item)); // invalid measure item // normal usage item.mode_ = ObIOMode::READ; item.size_ = 10; item.rt_us_ = 10; item.iops_ = 100; ASSERT_TRUE(item.is_valid()); ASSERT_SUCC(io_ability.add_measure_item(item)); item.mode_ = ObIOMode::WRITE; ASSERT_SUCC(io_ability.add_measure_item(item)); ASSERT_TRUE(io_ability.is_valid()); // test equal ASSERT_FALSE(item == item2); item2 = item; ASSERT_TRUE(item == item2); item2.iops_ += 0.0001; ASSERT_FALSE(item == item2); // test sort item.size_ = 2; item.rt_us_ = 2; item.iops_ = 500; ASSERT_SUCC(io_ability.add_measure_item(item)); item.size_ = 5; item.rt_us_ = 5; item.iops_ = 200; ASSERT_SUCC(io_ability.add_measure_item(item)); ASSERT_EQ(2, io_ability.measure_items_[(int)ObIOMode::WRITE].at(0).size_); ASSERT_EQ(5, io_ability.measure_items_[(int)ObIOMode::WRITE].at(1).size_); ASSERT_EQ(10, io_ability.measure_items_[(int)ObIOMode::WRITE].at(2).size_); // test calculate smallest double iops = 0; double rt = 0; ASSERT_SUCC(io_ability.get_iops(ObIOMode::WRITE, 1, iops)); ASSERT_SUCC(io_ability.get_rt(ObIOMode::WRITE, 1, rt)); ASSERT_GE(iops, 500); ASSERT_LE(rt, 2); // test calculate middle ASSERT_SUCC(io_ability.get_iops(ObIOMode::WRITE, 8, iops)); ASSERT_SUCC(io_ability.get_rt(ObIOMode::WRITE, 8, rt)); ASSERT_GE(iops, 100); ASSERT_LE(iops, 200); ASSERT_GE(rt, 5); ASSERT_LE(rt, 10); // test calculate biggest ASSERT_SUCC(io_ability.get_iops(ObIOMode::WRITE, 100, iops)); ASSERT_SUCC(io_ability.get_rt(ObIOMode::WRITE, 100, rt)); ASSERT_LE(iops, 100); ASSERT_GE(rt, 10); // test assign and equal ObIOAbility io_ability2; ASSERT_SUCC(io_ability2.assign(io_ability)); ASSERT_TRUE(io_ability2.is_valid()); ASSERT_TRUE(io_ability == io_ability2); // test destroy io_ability2.reset(); ASSERT_FALSE(io_ability2.is_valid()); } TEST_F(TestIOStruct, IOCalibration) { ObIOCalibration bench; ASSERT_FALSE(bench.is_inited_); ASSERT_SUCC(bench.init()); ASSERT_TRUE(bench.is_inited_); ObIOAbility io_ability; ASSERT_FALSE(io_ability.is_valid()); ASSERT_FAIL(bench.update_io_ability(io_ability)); ASSERT_SUCC(bench.reset_io_ability()); ASSERT_SUCC(bench.get_io_ability(io_ability)); ASSERT_FALSE(io_ability.is_valid()); bench.destroy(); ASSERT_FALSE(bench.is_inited_); ObIOBenchResult item; ASSERT_SUCC(ObIOCalibration::parse_calibration_string("READ:4096M:120:100000", item)); ASSERT_TRUE(item.is_valid()); ASSERT_SUCC(ObIOCalibration::parse_calibration_string("read:4K:10ms:1000", item)); ASSERT_TRUE(item.is_valid()); ASSERT_SUCC(ObIOCalibration::parse_calibration_string("r:2M:1s:100", item)); ASSERT_TRUE(item.is_valid()); ASSERT_SUCC(ObIOCalibration::parse_calibration_string("write:2M:1s:100", item)); ASSERT_TRUE(item.is_valid()); ASSERT_SUCC(ObIOCalibration::parse_calibration_string("W:12KB:20us:10000", item)); ASSERT_TRUE(item.is_valid()); ASSERT_FAIL(ObIOCalibration::parse_calibration_string(":12KB:20us:10000", item)); ASSERT_FAIL(ObIOCalibration::parse_calibration_string("W::20us:10000", item)); ASSERT_FAIL(ObIOCalibration::parse_calibration_string("W:12KB::10000", item)); ASSERT_FAIL(ObIOCalibration::parse_calibration_string("W:12KB:20us:", item)); ASSERT_FAIL(ObIOCalibration::parse_calibration_string("W:12KB:20us:X1000", item)); ASSERT_FAIL(ObIOCalibration::parse_calibration_string("A:12KB:20us:1000", item)); ASSERT_FAIL(ObIOCalibration::parse_calibration_string("W:12KB:20ks:1000", item)); ASSERT_FAIL(ObIOCalibration::parse_calibration_string("W:12U:20us:1000", item)); } TEST_F(TestIOStruct, IOStat) { ObIOStat stat; stat.accumulate(100, 200, 300); ObIOStatDiff io_diff; ObCpuUsage cpu_diff; double avg_iops = 0; double avg_size = 0; double avg_rt = 0; double avg_cpu = 0; io_diff.diff(stat, avg_iops, avg_size, avg_rt); cpu_diff.get_cpu_usage(avg_cpu); // ASSERT_DOUBLE_EQ(avg_iops, 0); ASSERT_DOUBLE_EQ(avg_size, 2); ASSERT_DOUBLE_EQ(avg_rt, 3); ASSERT_DOUBLE_EQ(avg_cpu, 0); usleep(1000L * 1000L); // sleep 1s stat.accumulate(200, 600, 4000); io_diff.diff(stat, avg_iops, avg_size, avg_rt); ASSERT_NEAR(avg_iops, 200, 1); ASSERT_NEAR(avg_size, 3, 0.1); ASSERT_NEAR(avg_rt, 20, 0.1); ASSERT_GE(avg_cpu, 0); } TEST_F(TestIOStruct, IOScheduler) { ObIOCalibration::get_instance().init(); // test init ObIOConfig io_config = ObIOConfig::default_config(); ObIOAllocator io_allocator; ASSERT_SUCC(io_allocator.init(TEST_TENANT_ID, IO_MEMORY_LIMIT)); ASSERT_TRUE(io_config.is_valid()); ObIOScheduler &scheduler = *(OB_IO_MANAGER.get_scheduler()); // test schedule ObIOResult result; ObIORequest req; result.inc_ref(); ObIOInfo io_info = get_random_io_info(); ASSERT_TRUE(io_info.is_valid()); ASSERT_SUCC(result.basic_init()); ASSERT_SUCC(result.init(io_info)); ASSERT_SUCC(req.init(io_info, &result)); ObTenantIOConfig tenant_config = default_tenant_io_config(); ObTenantIOClock io_clock; ObIOUsage io_usage; ASSERT_SUCC(io_clock.init(tenant_config, &io_usage)); io_clock.destroy(); ASSERT_SUCC(io_usage.init(0)); ASSERT_SUCC(io_clock.init(tenant_config, &io_usage)); //ASSERT_SUCC(scheduler.schedule_request(io_clock, req)); // test destroy scheduler.destroy(); ASSERT_FALSE(scheduler.is_inited_); ObIOCalibration::get_instance().destroy(); } TEST_F(TestIOStruct, MClockQueue) { ObIOAllocator io_allocator; ASSERT_SUCC(io_allocator.init(TEST_TENANT_ID, IO_MEMORY_LIMIT)); ObTenantIOConfig io_config; io_config.callback_thread_count_ = 2; io_config.memory_limit_ = 1024L * 1024L * 1024L; io_config.group_num_ = 2; io_config.unit_config_.min_iops_ = 100; io_config.unit_config_.max_iops_ = 10000L; io_config.unit_config_.weight_ = 1000; ObTenantIOConfig::GroupConfig group_config_1; group_config_1.min_percent_ = 1; group_config_1.max_percent_ = 90; group_config_1.weight_percent_ = 60; ObTenantIOConfig::GroupConfig group_config_2; group_config_2.min_percent_ = 1; group_config_2.max_percent_ = 90; group_config_2.weight_percent_ = 30; io_config.group_configs_.push_back(group_config_1); io_config.group_configs_.push_back(group_config_2); io_config.group_ids_.push_back(1); io_config.group_ids_.push_back(2); io_config.other_group_config_.min_percent_ = 98; io_config.other_group_config_.max_percent_ = 100; io_config.other_group_config_.weight_percent_ = 10; ASSERT_TRUE(io_config.is_valid()); ObTenantIOClock tenant_clock; ObIOUsage io_usage; ASSERT_SUCC(io_usage.init(2)); ASSERT_SUCC(io_usage.refresh_group_num(2)); ASSERT_SUCC(tenant_clock.init(io_config, &io_usage)); ObMClockQueue mqueue1; ObMClockQueue mqueue2; const int64_t test_count = 6000L; ASSERT_SUCC(mqueue1.init()); ASSERT_SUCC(mqueue2.init()); ObArenaAllocator arena; for (int64_t i = 0; i < test_count; ++i) { char *buf = (char *)arena.alloc(sizeof(ObPhyQueue) * 2); ASSERT_TRUE(nullptr != buf); ObPhyQueue *phy_req = new (buf) ObPhyQueue; ObPhyQueue *phy_req2 = new (buf + sizeof(ObPhyQueue)) ObPhyQueue; ASSERT_SUCC(mqueue1.push_phyqueue(phy_req)); ASSERT_SUCC(mqueue2.push_phyqueue(phy_req2)); } } TEST_F(TestIOStruct, Test_Size) { //callback int64_t size1 = sizeof(ObAsyncSingleMicroBlockIOCallback); int64_t size2 = sizeof(ObMultiDataBlockIOCallback); int64_t size3 = sizeof(ObSyncSingleMicroBLockIOCallback); int64_t size4 = sizeof(ObTmpPageCache::ObTmpPageIOCallback); int64_t size5 = sizeof(ObTmpPageCache::ObTmpMultiPageIOCallback); int64_t size6 = sizeof(oceanbase::ObStorageMetaCache::ObStorageMetaIOCallback); int64_t max_callback_size = std::max({size1, size2, size3, size4, size5, size6}); int64_t size_request = sizeof(ObIORequest); int64_t size_result = sizeof(ObIOResult); int64_t size_info = sizeof(ObIOInfo); int64_t size_thread_cond = sizeof(ObThreadCond); int64_t size_flag = sizeof(ObIOFlag); ObRefHolder tenant_io_mgr_; int64_t ref_size = sizeof(tenant_io_mgr_); int64_t time_size = sizeof(ObIOTimeLog); int64_t return_size = sizeof(ObIORetCode); int64_t trace_size = sizeof(ObCurTraceId::TraceId); int64_t fd_size = sizeof(ObIOFd); ASSERT_LT(max_callback_size, 256); LOG_INFO("qilu :check size", K(size1), K(size2), K(size3), K(size4), K(size5), K(size6), K(max_callback_size)); LOG_INFO("qilu :check size", K(size_request), K(size_result), K(size_info), K(size_thread_cond), K(size_flag), K(ref_size), K(time_size), K(return_size), K(fd_size), K(trace_size)); //mark: max_callback_size=208(ObMultiDataBlockIOCallback、ObStorageMetaIOCallback) } TEST_F(TestIOStruct, IOResult) { ObRefHolder holder; ASSERT_SUCC(OB_IO_MANAGER.get_tenant_io_manager(TEST_TENANT_ID, holder)); ObIOFd fd; fd.first_id_ = 0; fd.second_id_ = 1; void *result_buf = holder.get_ptr()->io_allocator_.alloc(sizeof(ObIOResult)); ObIOResult *result = new (result_buf) ObIOResult; void *req_buf = holder.get_ptr()->io_allocator_.alloc(sizeof(ObIORequest)); ObIORequest *req = new (req_buf) ObIORequest; // default invalid ASSERT_FALSE(result->is_inited_); ASSERT_FALSE(result->is_valid()); ASSERT_FALSE(req->is_inited_); ASSERT_FALSE(req->is_valid()); // prepare test read request req->destroy(); result->destroy(); req->tenant_io_mgr_.hold(holder.get_ptr()); result->tenant_io_mgr_.hold(holder.get_ptr()); result->inc_ref(); req->inc_ref(); ObIOInfo read_info; read_info.tenant_id_ = OB_SERVER_TENANT_ID; read_info.fd_ = fd; read_info.flag_.set_mode(ObIOMode::READ); read_info.flag_.set_group_id(10005); read_info.flag_.set_wait_event(1); read_info.timeout_us_ = DEFAULT_IO_WAIT_TIME_US; read_info.offset_ = 89; read_info.size_ = 1; char user_buf[1] = { 0 }; read_info.user_data_buf_ = user_buf; ASSERT_TRUE(read_info.is_valid()); ASSERT_SUCC(req->tenant_io_mgr_.get_ptr()->io_usage_.init(0)); ASSERT_FAIL(result->init(read_info)); ASSERT_FALSE(result->is_inited_); ASSERT_SUCC(result->basic_init()); ASSERT_SUCC(result->init(read_info)); ASSERT_TRUE(result->is_inited_); ASSERT_TRUE(result->is_valid()); ASSERT_SUCC(req->init(read_info, result)); ASSERT_TRUE(req->is_inited_); ASSERT_EQ(req->raw_buf_, nullptr); // read buf allocation is delayed ASSERT_SUCC(req->prepare()); ASSERT_NE(req->raw_buf_, nullptr); ASSERT_EQ(result->get_group_id(), 10005); ASSERT_EQ(req->get_group_id(), 10005); // test finish result->finish_without_accumulate(OB_CANCELED); ASSERT_EQ(result->ret_code_.io_ret_, OB_CANCELED); result->finish_without_accumulate(OB_IO_ERROR); ASSERT_NE(result->ret_code_.io_ret_, OB_IO_ERROR); // finish only once //test free req->io_result_->dec_ref(); ASSERT_EQ(1, req->io_result_->result_ref_cnt_); req->destroy(); ASSERT_EQ(nullptr, req->io_result_); } TEST_F(TestIOStruct, IOCallbackManager) { // test init ObIOCallbackManager callback_mgr; ASSERT_FALSE(callback_mgr.is_inited_); ASSERT_FAIL(callback_mgr.init(TEST_TENANT_ID, 0, 1000, nullptr)); ObIOAllocator io_allocator; ASSERT_SUCC(io_allocator.init(TEST_TENANT_ID, IO_MEMORY_LIMIT)); ASSERT_SUCC(callback_mgr.init(TEST_TENANT_ID, 2, 1000, &io_allocator)); ASSERT_TRUE(callback_mgr.is_inited_); // test enqueue and dequeue ObIOResult result; ObIORequest req; req.inc_ref(); result.inc_ref(); ObIOInfo io_info = get_random_io_info(); ASSERT_TRUE(io_info.is_valid()); ASSERT_SUCC(result.basic_init()); ASSERT_SUCC(result.init(io_info)); ASSERT_SUCC(req.init(io_info, &result)); ASSERT_FAIL(callback_mgr.enqueue_callback(req)); char buf[32] = "test"; req.raw_buf_ = buf; char callback_buf_[ObIOCallback::CALLBACK_BUF_SIZE] __attribute__ ((aligned (16))); TestIOCallback *test_callback = new (callback_buf_) TestIOCallback(); result.io_callback_ = test_callback; // ObIOManager::get_instance().io_config_ = ObIOConfig::default_config(); ASSERT_SUCC(callback_mgr.enqueue_callback(req)); // test destroy callback_mgr.destroy(); ASSERT_FALSE(callback_mgr.is_inited_); } TEST_F(TestIOStruct, IOFaultDetector) { ObIOFaultDetector &detector = OB_IO_MANAGER.get_device_health_detector(); ObIOConfig &io_config = (ObIOConfig &)detector.io_config_; // test get device health ObDeviceHealthStatus dhs = DEVICE_HEALTH_NORMAL; int64_t disk_abnormal_time = 0; ASSERT_SUCC(detector.get_device_health_status(dhs, disk_abnormal_time)); ASSERT_TRUE(DEVICE_HEALTH_NORMAL == dhs); ASSERT_TRUE(0 == disk_abnormal_time); // test read failure detection ObIOInfo io_info = get_random_io_info(); ObIOResult result; ObIORequest req; req.inc_ref(); result.inc_ref(); ASSERT_SUCC(result.basic_init()); ASSERT_SUCC(result.init(io_info)); ASSERT_SUCC(req.init(io_info, &result)); detector.reset_device_health(); ASSERT_SUCC(detector.get_device_health_status(dhs, disk_abnormal_time)); ASSERT_TRUE(DEVICE_HEALTH_NORMAL == dhs); ASSERT_TRUE(0 == disk_abnormal_time); result.flag_.set_mode(ObIOMode::READ); io_config.data_storage_warning_tolerance_time_ = 1000L * 1000L; io_config.data_storage_error_tolerance_time_ = 3000L * 1000L; // io manager not init, skip this test // detector.record_io_err_failure(req); // usleep(2000L * 1000L); // ASSERT_SUCC(detector.get_device_health(is_device_warning, is_device_error)); // ASSERT_TRUE(is_device_warning); // ASSERT_FALSE(is_device_error); // usleep(2000L * 1000L); // ASSERT_SUCC(detector.get_device_health(is_device_warning, is_device_error)); // ASSERT_TRUE(is_device_warning); // ASSERT_TRUE(is_device_error); // test auto clean device warning, but not clean device error detector.reset_device_health(); ASSERT_SUCC(detector.get_device_health_status(dhs, disk_abnormal_time)); ASSERT_TRUE(DEVICE_HEALTH_NORMAL == dhs); ASSERT_TRUE(0 == disk_abnormal_time); io_config.read_failure_black_list_interval_ = 1000L * 100L; // 100ms detector.set_device_warning(); ASSERT_SUCC(detector.get_device_health_status(dhs, disk_abnormal_time)); ASSERT_TRUE(DEVICE_HEALTH_WARNING == dhs); ASSERT_TRUE(disk_abnormal_time > 0); usleep(io_config.read_failure_black_list_interval_ * 2); ASSERT_SUCC(detector.get_device_health_status(dhs, disk_abnormal_time)); ASSERT_TRUE(DEVICE_HEALTH_NORMAL == dhs); ASSERT_TRUE(0 == disk_abnormal_time); detector.set_device_error(); usleep(io_config.read_failure_black_list_interval_ * 2); ASSERT_SUCC(detector.get_device_health_status(dhs, disk_abnormal_time)); ASSERT_TRUE(DEVICE_HEALTH_ERROR == dhs); ASSERT_TRUE(disk_abnormal_time > 0); } class TestIOManager : public TestIOStruct { // basic use resource manager public: static void SetUpTestCase() { } static void TearDownTestCase() { oceanbase::MockTenantModuleEnv::get_instance().destroy(); } virtual void SetUp() { OB_IO_MANAGER.destroy(); const int64_t memory_limit = 10L * 1024L * 1024L * 1024L; // 10GB ASSERT_SUCC(OB_IO_MANAGER.init(memory_limit)); ASSERT_SUCC(OB_IO_MANAGER.start()); // add io device ASSERT_SUCC(OB_IO_MANAGER.add_device_channel(THE_IO_DEVICE, 16, 2, 1024)); // add tenant io manager const uint64_t tenant_id = OB_SERVER_TENANT_ID; ObTenantIOConfig io_config; io_config.memory_limit_ = memory_limit; io_config.callback_thread_count_ = 2; io_config.unit_config_.min_iops_ = 10000; io_config.unit_config_.max_iops_ = 100000; io_config.unit_config_.weight_ = 100; io_config.other_group_config_.min_percent_ = 100; io_config.other_group_config_.max_percent_ = 100; io_config.other_group_config_.weight_percent_ = 100; } virtual void TearDown() { OB_IO_MANAGER.stop(); OB_IO_MANAGER.destroy(); } }; TEST_F(TestIOManager, memory_pool) { ObRefHolder tenant_holder; ASSERT_SUCC(OB_IO_MANAGER.get_tenant_io_manager(500, tenant_holder)); ASSERT_NE(nullptr, tenant_holder.get_ptr()); ObIORequest *io_request = nullptr; ASSERT_SUCC(tenant_holder.get_ptr()->io_request_pool_.alloc(io_request)); ASSERT_NE(nullptr, io_request); io_request->tenant_io_mgr_.hold(tenant_holder.get_ptr()); ASSERT_TRUE(tenant_holder.get_ptr()->io_request_pool_.contain(io_request)); ASSERT_SUCC(tenant_holder.get_ptr()->io_request_pool_.recycle(io_request)); io_request->tenant_io_mgr_.reset(); ObIOResult *io_result = nullptr; ASSERT_SUCC(tenant_holder.get_ptr()->io_result_pool_.alloc(io_result)); ASSERT_NE(nullptr, io_result); io_result->tenant_io_mgr_.hold(tenant_holder.get_ptr()); ASSERT_TRUE(tenant_holder.get_ptr()->io_result_pool_.contain(io_result)); ASSERT_SUCC(tenant_holder.get_ptr()->io_result_pool_.recycle(io_result)); io_result->tenant_io_mgr_.reset(); void *result_buf = tenant_holder.get_ptr()->io_allocator_.alloc(sizeof(ObIOResult)); ObIOResult *result1 = new (result_buf) ObIOResult; result1->tenant_io_mgr_.hold(tenant_holder.get_ptr()); ASSERT_FALSE(tenant_holder.get_ptr()->io_result_pool_.contain(result1)); ASSERT_FAIL(tenant_holder.get_ptr()->io_result_pool_.recycle(result1)); result1->~ObIOResult(); tenant_holder.get_ptr()->io_allocator_.free(result1); void *req_buf = tenant_holder.get_ptr()->io_allocator_.alloc(sizeof(ObIORequest)); ObIORequest *req1 = new (req_buf) ObIORequest; req1->tenant_io_mgr_.hold(tenant_holder.get_ptr()); ASSERT_FALSE(tenant_holder.get_ptr()->io_request_pool_.contain(req1)); ASSERT_FAIL(tenant_holder.get_ptr()->io_request_pool_.recycle(req1)); req1->~ObIORequest(); tenant_holder.get_ptr()->io_allocator_.free(req1); } TEST_F(TestIOManager, simple) { ObIOFd fd; ASSERT_SUCC(THE_IO_DEVICE->open(TEST_ROOT_DIR "/test_io_file", O_CREAT | O_DIRECT | O_TRUNC | O_RDWR, 0644, fd)); ASSERT_TRUE(fd.is_valid()); ObIOManager &io_mgr = ObIOManager::get_instance(); // fallocate const int64_t FILE_SIZE = 4 * 1024 * 1024; ASSERT_SUCC(THE_IO_DEVICE->fallocate(fd, 0, 0, FILE_SIZE)); // 4M { oceanbase::common::ObIODFileStat stat_buf; ASSERT_SUCC(THE_IO_DEVICE->stat(TEST_ROOT_DIR "/test_io_file", stat_buf)); ASSERT_EQ(FILE_SIZE, stat_buf.size_); } // sync write const int64_t io_timeout_ms = 1000L * 5L; const int64_t write_io_size = DIO_READ_ALIGN_SIZE * 2; ObIOInfo io_info; io_info.tenant_id_ = OB_SERVER_TENANT_ID; io_info.fd_ = fd; io_info.flag_.set_write(); io_info.flag_.set_group_id(0); io_info.flag_.set_wait_event(100); io_info.offset_ = 0; io_info.size_ = write_io_size; io_info.timeout_us_ = DEFAULT_IO_WAIT_TIME_US; char buf[write_io_size] = { 0 }; const int64_t user_offset = write_io_size / 2; memset(buf, 'a', user_offset); memset(buf + user_offset, 'b', write_io_size - user_offset); io_info.buf_ = buf; io_info.user_data_buf_ = buf; ASSERT_SUCC(io_mgr.write(io_info)); // check size { oceanbase::common::ObIODFileStat stat_buf; ASSERT_SUCC(THE_IO_DEVICE->stat(TEST_ROOT_DIR "/test_io_file", stat_buf)); ASSERT_EQ(FILE_SIZE, stat_buf.size_); } // sync read and compare data buffer io_info.flag_.set_read(); ObIOHandle io_handle; ASSERT_SUCC(io_mgr.read(io_info, io_handle)); ASSERT_NE(nullptr, io_handle.get_buffer()); ASSERT_EQ(io_info.size_, io_handle.get_data_size()); ASSERT_EQ(0, memcmp(buf, io_handle.get_buffer(), write_io_size)); // read end io_handle.reset(); io_info.offset_ = FILE_SIZE; io_info.size_ = DIO_READ_ALIGN_SIZE; ASSERT_NE(OB_SUCCESS, io_mgr.read(io_info, io_handle)); ASSERT_EQ(0, io_handle.get_data_size()); // read tail io_handle.reset(); io_info.offset_ = FILE_SIZE - DIO_READ_ALIGN_SIZE; io_info.size_ = DIO_READ_ALIGN_SIZE; ASSERT_SUCC(io_mgr.read(io_info, io_handle)); ASSERT_EQ(DIO_READ_ALIGN_SIZE, io_handle.get_data_size()); // read tail part io_handle.reset(); io_info.offset_ = FILE_SIZE - DIO_READ_ALIGN_SIZE; io_info.size_ = DIO_READ_ALIGN_SIZE * 2; ASSERT_NE(OB_SUCCESS, io_mgr.read(io_info, io_handle)); ASSERT_EQ(DIO_READ_ALIGN_SIZE, io_handle.get_data_size()); // read with callback io_handle.reset(); io_info.offset_ = user_offset; io_info.size_ = write_io_size - user_offset; ObArenaAllocator allocator; int64_t tmp_number = 0; TestIOCallback callback; callback.number_ = &tmp_number; callback.allocator_ = &allocator; io_info.callback_ = &callback; ASSERT_SUCC(io_mgr.read(io_info, io_handle)); ASSERT_NE(nullptr, io_handle.get_buffer()); ASSERT_EQ(io_info.size_, io_handle.get_data_size()); ASSERT_EQ(0, memcmp(buf + user_offset, io_handle.get_buffer(), write_io_size - user_offset)); ASSERT_EQ(100, tmp_number); // callback process called io_handle.reset(); // ASSERT_EQ(10, tmp_number); // callback destructor called ASSERT_SUCC(THE_IO_DEVICE->close(fd)); } struct IOPerfDevice { IOPerfDevice() : device_id_(0), media_id_(0), async_channel_count_(0), sync_channel_count_(0), max_io_depth_(0), file_size_(0), file_path_(), fd_(0), device_handle_(nullptr) { memset(file_path_, 0, sizeof(file_path_)); } bool is_valid() const { return device_id_ > 0 && media_id_ >= 0 && async_channel_count_ > 0 && sync_channel_count_ > 0 && max_io_depth_ > 0 && file_size_ > 0 && strlen(file_path_) > 0; } TO_STRING_KV(K(device_id_), K(media_id_), K(async_channel_count_), K(sync_channel_count_), K(max_io_depth_), K(file_size_), K(file_path_), K(fd_), KP(device_handle_)); int32_t device_id_; int32_t media_id_; int32_t async_channel_count_; int32_t sync_channel_count_; int32_t max_io_depth_; int64_t file_size_; char file_path_[OB_MAX_FILE_NAME_LENGTH]; int32_t fd_; ObLocalDevice *device_handle_; }; struct IOPerfTenant { IOPerfTenant() : tenant_id_(0), config_() {} bool is_valid() const { return tenant_id_ > 0 && config_.is_valid(); } TO_STRING_KV(K(tenant_id_), K(config_)); int32_t tenant_id_; ObTenantIOConfig config_; }; enum class IOPerfMode { ROLLING, BATCH, ASYNC, UNKNOWN }; struct IOPerfLoad { IOPerfLoad() : tenant_id_(0), device_id_(0), mode_(ObIOMode::MAX_MODE), size_(0), depth_(0), iops_(0), thread_count_(0), is_sequence_(false), group_id_(0), start_delay_ts_(0), stop_delay_ts_(0), device_(nullptr), perf_mode_(IOPerfMode::UNKNOWN) {} TO_STRING_KV(K(tenant_id_), K(device_id_), "mode", ObIOMode::READ == mode_ ? "read" : ObIOMode::WRITE == mode_ ? "write" : "unknown", "group_id", group_id_, "io_size", size_, "io_depth", depth_, "target_iops", iops_, K(thread_count_), K(is_sequence_), K(start_delay_ts_), K(stop_delay_ts_), KP(device_), K(perf_mode_)); bool is_valid() const { return tenant_id_ > 0 && group_id_ >= 0 && device_id_ > 0 && mode_ < ObIOMode::MAX_MODE && size_ > 0 && depth_ > 0 && iops_ >= 0 && thread_count_ > 0 && start_delay_ts_ >= 0 && stop_delay_ts_ > start_delay_ts_ && size_ > 0 && (ObIOMode::WRITE == mode_ ? is_io_aligned(size_) : true) && perf_mode_ != IOPerfMode::UNKNOWN; } int32_t tenant_id_; int32_t device_id_; ObIOMode mode_; int32_t size_; int32_t depth_; int64_t iops_; int32_t thread_count_; bool is_sequence_; uint64_t group_id_; int64_t start_delay_ts_; int64_t stop_delay_ts_; IOPerfDevice *device_; IOPerfMode perf_mode_; }; struct IOPerfScheduler { IOPerfScheduler() : sender_count_(0), schedule_media_id_(0), io_greed_(0) {} int64_t sender_count_; int64_t schedule_media_id_; int64_t io_greed_; TO_STRING_KV(K(sender_count_), K(schedule_media_id_), K(io_greed_)); }; struct IOPerfResult { IOPerfResult() : succ_count_(0), fail_count_(0), sum_rt_(0), start_delay_ts_(0), stop_delay_ts_(0), disk_rt_(0) {} TO_STRING_KV(K(succ_count_), K(fail_count_), K(sum_rt_), K(start_delay_ts_), K(stop_delay_ts_)); int64_t succ_count_; int64_t fail_count_; int64_t sum_rt_; int64_t start_delay_ts_; int64_t stop_delay_ts_; int64_t disk_rt_; }; class IOPerfRunner : public ThreadPool { public: IOPerfRunner() : abs_ts_(0), last_offset_(0), write_buf_(nullptr), user_buf_(nullptr), io_count_(0), report_count_(0), total_io_count_(0) {} int init(const int64_t absolute_ts, const IOPerfLoad &load); void destroy(); virtual void run1() override; int do_batch_io(); int do_perf_batch(); int do_perf_rolling(); int wait_and_count(ObIOHandle &io_handle); int wait_handles(ObFixedQueue &handles); int print_result(); TO_STRING_KV(K(load_), K(result_), K(last_offset_), KP(write_buf_), KP(user_buf_), K(fd_)); public: int64_t abs_ts_; IOPerfLoad load_; IOPerfResult result_; int64_t last_offset_; const char *write_buf_; char *user_buf_; ObIOFd fd_; ObConcurrentFIFOAllocator allocator_; ObFixedQueue handle_queue_; int64_t io_count_; int64_t report_count_; int64_t total_io_count_; }; class IOConfModify : public ThreadPool { public: IOConfModify() : modify_init_ts_(0) {} int init(int64_t modify_init_ts, int64_t modify_delay_ts, const IOPerfTenant &curr_tenant); void destroy(); virtual void run1() override; int modify_tenant_io(const int64_t min_iops, const int64_t max_iops, const int64_t weight, IOPerfTenant &curr_tenant); TO_STRING_KV(K(load_), K(modify_delay_ts_), K(fd_), K(curr_tenant_)); public: int64_t modify_init_ts_; int64_t modify_delay_ts_; IOPerfTenant curr_tenant_; ObConcurrentFIFOAllocator allocator_; IOPerfLoad load_; ObIOFd fd_; }; class IOCallbackModifier : public ThreadPool { public: IOCallbackModifier() : modify_init_ts_(0) {} int init(int64_t modify_init_ts, int64_t modify_delay_ts, const IOPerfTenant &curr_tenant); void destroy(); virtual void run1() override; int modify_callback_num(const int64_t thread_num, IOPerfTenant &curr_tenant); TO_STRING_KV(K(load_), K(modify_delay_ts_), K(fd_), K(curr_tenant_)); public: int64_t modify_init_ts_; int64_t modify_delay_ts_; IOPerfTenant curr_tenant_; ObConcurrentFIFOAllocator allocator_; IOPerfLoad load_; ObIOFd fd_; }; class IOGroupModify : public ThreadPool { public: IOGroupModify() : modify_init_ts_(0) {} int init(int64_t modify_init_ts, int64_t modify_delay_ts, const IOPerfTenant &curr_tenant); void destroy(); virtual void run1() override; TO_STRING_KV(K(load_), K(modify_delay_ts_), K(fd_), K(curr_tenant_)); public: int64_t modify_init_ts_; int64_t modify_delay_ts_; IOPerfTenant curr_tenant_; ObConcurrentFIFOAllocator allocator_; IOPerfLoad load_; ObIOFd fd_; }; class IOTracerSwitch : public ThreadPool { public: IOTracerSwitch() {} int init(int64_t switch_init_ts, int64_t switch_delay_ts, const IOPerfTenant &curr_tenant); void destroy(); virtual void run1() override; int modify_tenant_io(IOPerfTenant &curr_tenant); TO_STRING_KV(K(load_), K(switch_init_ts_), K(switch_delay_ts_), K(curr_tenant_), K(load_)); public: int64_t switch_init_ts_; int64_t switch_delay_ts_; IOPerfTenant curr_tenant_; ObConcurrentFIFOAllocator allocator_; IOPerfLoad load_; }; #define GROUP_PERF_CONFIG_FILE "io_perf.conf" void write_group_perf_config(); int parse_group_perf_config(const char *config_file_path, IOPerfScheduler &scheduler_config, ObIArray &perf_devices, ObIArray &perf_tenants, ObIArray &perf_loads); int prepare_file(const char *file_path, const int64_t file_size, int32_t &fd) { int ret = OB_SUCCESS; fd = -1; bool need_create_perf_file = true; bool is_file_exist = false; int64_t exist_file_size = 0; if (OB_FAIL(FileDirectoryUtils::is_exists(file_path, is_file_exist))) { LOG_WARN("check file exist failed", K(ret)); } else if (is_file_exist) { if (OB_FAIL(FileDirectoryUtils::get_file_size(file_path, exist_file_size))) { LOG_WARN("get file size failed", K(ret)); } else if (exist_file_size == file_size) { if ((fd = ::open(file_path, O_DIRECT | O_RDWR)) < 0) { ret = OB_IO_ERROR; LOG_WARN("fail to open file", K(ret)); } else { need_create_perf_file = false; } } } if (need_create_perf_file) { char cmd[1024] = { 0 }; sprintf(cmd, "rm -rf %s", file_path); system(cmd); if ((fd = ::open(file_path, O_CREAT | O_TRUNC | O_DIRECT | O_RDWR, 0644)) < 0) { ret = OB_IO_ERROR; LOG_WARN("fail to create file", K(ret)); } else { if (fallocate(fd, 0, 0, file_size) < 0) { ret = OB_IO_ERROR; LOG_WARN("fail to allocate file", K(ret), K(fd), K(file_size)); } else { const int64_t buf_size = 1024L * 1024L; // 1MB const int64_t write_count = file_size / buf_size; sprintf(cmd, "dd if=/dev/zero of=%s bs=1M count=%ld", file_path, write_count); system(cmd); fsync(fd); LOG_INFO("prepare file finished", K(cmd), "file_size_mb", write_count); } } } return ret; } TEST_F(TestIOManager, tenant) { ObTenantIOConfig default_config = ObTenantIOConfig::default_instance(); default_config.unit_config_.max_iops_ = 20000L; int64_t current_ts = ObTimeUtility::fast_current_time(); IOPerfLoad load; load.group_id_ = 0; load.depth_ = 1; IOPerfDevice device; device.device_id_ = 1; strcpy(device.file_path_, "./perf_test"); device.file_size_ = 1024L * 1024L * 1024L; device.device_handle_ = static_cast(THE_IO_DEVICE); prepare_file(device.file_path_, device.file_size_, device.fd_); load.device_ = &device; load.device_id_ = 1; // unused load.iops_ = 0; load.is_sequence_ = false; load.mode_ = ObIOMode::READ; load.perf_mode_ = IOPerfMode::ROLLING; load.size_ = 8192; load.start_delay_ts_ = 0; load.stop_delay_ts_ = 3L * 1000L * 1000L; // 3s load.tenant_id_ = 1001; load.thread_count_ = 16; IOPerfRunner runner; ASSERT_SUCC(runner.init(current_ts, load)); usleep(2L * 1000L * 1000L); // 2s runner.wait(); runner.destroy(); } TEST_F(TestIOManager, perf) { // use multi thread to do some io stress, maybe use test_io_performance bool is_perf_config_exist = false; ASSERT_SUCC(FileDirectoryUtils::is_exists(GROUP_PERF_CONFIG_FILE, is_perf_config_exist)); if (!is_perf_config_exist) { write_group_perf_config(); } // parse configs IOPerfScheduler scheduler_config; ObArray perf_devices; ObArray perf_tenants; ObArray perf_loads; ASSERT_SUCC(parse_group_perf_config(GROUP_PERF_CONFIG_FILE, scheduler_config, perf_devices, perf_tenants, perf_loads)); ASSERT_TRUE(perf_devices.count() > 0); ASSERT_TRUE(perf_tenants.count() > 0); ASSERT_TRUE(perf_loads.count() > 0); ObIOManager::get_instance().destroy(); const int64_t memory_limit = 30L * 1024L * 1024L * 1024L; // 30GB const int64_t queue_depth = 100L; ASSERT_SUCC(ObIOManager::get_instance().init(memory_limit, queue_depth, scheduler_config.sender_count_, scheduler_config.schedule_media_id_)); ASSERT_SUCC(ObIOManager::get_instance().start()); // prepare devices and files char *device_buf = (char *)malloc(sizeof(ObLocalDevice) * perf_devices.count()); ASSERT_TRUE(nullptr != device_buf); for (int64_t i = 0; i < perf_devices.count(); ++i) { IOPerfDevice &curr_config = perf_devices.at(i); ASSERT_SUCC(prepare_file(curr_config.file_path_, curr_config.file_size_, curr_config.fd_)); ObLocalDevice *device = new (device_buf + sizeof(ObLocalDevice) * i) ObLocalDevice; ASSERT_SUCC(init_device(curr_config.media_id_, *device)); ASSERT_SUCC(OB_IO_MANAGER.add_device_channel(device, curr_config.async_channel_count_, curr_config.sync_channel_count_, curr_config.max_io_depth_)); curr_config.device_handle_ = device; } // prepare tenant io manager for (int64_t i = 0; i < perf_tenants.count(); ++i) { IOPerfTenant &curr_config = perf_tenants.at(i); LOG_INFO("wenqu: tenant config", K(curr_config), K(i)); ObRefHolder tenant_holder; ASSERT_SUCC(OB_IO_MANAGER.get_tenant_io_manager(curr_config.tenant_id_, tenant_holder)); ASSERT_SUCC(tenant_holder.get_ptr()->refresh_group_io_config()); } // prepare perf runners char *runner_buf = (char *)malloc(perf_loads.count() * sizeof(IOPerfRunner)); ObArray runners; const int64_t start_ts = ObTimeUtility::current_time() + 10000L; for (int64_t i = 0; i < perf_loads.count(); ++i) { IOPerfRunner *runner = new (runner_buf + i * sizeof(IOPerfRunner)) IOPerfRunner(); const IOPerfLoad &cur_load = perf_loads.at(i); ASSERT_SUCC(runner->init(start_ts, cur_load)); ASSERT_SUCC(runners.push_back(runner)); } // wait perf finished for (int64_t i = 0; i < runners.count(); ++i) { IOPerfRunner *runner = runners.at(i); runner->wait(); ASSERT_SUCC(runner->print_result()); runner->destroy(); } free(runner_buf); ObIOManager::get_instance().stop(); ObIOManager::get_instance().destroy(); for (int64_t i = 0; i < perf_devices.count(); ++i) { ObLocalDevice *device_handle = perf_devices.at(i).device_handle_; // ASSERT_SUCC(OB_IO_MANAGER.remove_device_channel(device_handle)); device_handle->destroy(); } free(device_buf); LOG_INFO("wenqu: perf finished"); } TEST_F(TestIOManager, alloc_memory) { // use multi thread to do some io stress, maybe use test_io_performance bool is_perf_config_exist = false; ASSERT_SUCC(FileDirectoryUtils::is_exists(GROUP_PERF_CONFIG_FILE, is_perf_config_exist)); if (!is_perf_config_exist) { write_group_perf_config(); } // parse configs IOPerfScheduler scheduler_config; ObArray perf_devices; ObArray perf_tenants; ObArray perf_loads; ASSERT_SUCC(parse_group_perf_config(GROUP_PERF_CONFIG_FILE, scheduler_config, perf_devices, perf_tenants, perf_loads)); ASSERT_TRUE(perf_devices.count() > 0); ASSERT_TRUE(perf_tenants.count() > 0); ASSERT_TRUE(perf_loads.count() > 0); ObIOManager::get_instance().destroy(); const int64_t memory_limit = 30L * 1024L * 1024L * 1024L; // 30GB const int64_t queue_depth = 100L; ASSERT_SUCC(ObIOManager::get_instance().init(memory_limit, queue_depth, scheduler_config.sender_count_, scheduler_config.schedule_media_id_)); ASSERT_SUCC(ObIOManager::get_instance().start()); // prepare devices and files char *device_buf = (char *)malloc(sizeof(ObLocalDevice) * perf_devices.count()); ASSERT_TRUE(nullptr != device_buf); for (int64_t i = 0; i < perf_devices.count(); ++i) { IOPerfDevice &curr_config = perf_devices.at(i); ASSERT_SUCC(prepare_file(curr_config.file_path_, curr_config.file_size_, curr_config.fd_)); ObLocalDevice *device = new (device_buf + sizeof(ObLocalDevice) * i) ObLocalDevice; ASSERT_SUCC(init_device(curr_config.media_id_, *device)); ASSERT_SUCC(OB_IO_MANAGER.add_device_channel(device, curr_config.async_channel_count_, curr_config.sync_channel_count_, curr_config.max_io_depth_)); curr_config.device_handle_ = device; } // prepare tenant io manager for (int64_t i = 0; i < perf_tenants.count(); ++i) { IOPerfTenant &curr_config = perf_tenants.at(i); curr_config.config_.memory_limit_ = 16L* 1024L * 1024L; //16MB ObRefHolder tenant_holder; ASSERT_SUCC(OB_IO_MANAGER.get_tenant_io_manager(curr_config.tenant_id_, tenant_holder)); ASSERT_SUCC(tenant_holder.get_ptr()->refresh_group_io_config()); } // prepare perf runners char *runner_buf = (char *)malloc(perf_loads.count() * sizeof(IOPerfRunner)); ObArray runners; const int64_t start_ts = ObTimeUtility::current_time() + 10000L; for (int64_t i = 0; i < perf_loads.count(); ++i) { IOPerfRunner *runner = new (runner_buf + i * sizeof(IOPerfRunner)) IOPerfRunner(); const IOPerfLoad &cur_load = perf_loads.at(i); ASSERT_SUCC(runner->init(start_ts, cur_load)); ASSERT_SUCC(runners.push_back(runner)); } // wait perf finished for (int64_t i = 0; i < runners.count(); ++i) { IOPerfRunner *runner = runners.at(i); runner->wait(); ASSERT_SUCC(runner->print_result()); runner->destroy(); } free(runner_buf); ObIOManager::get_instance().stop(); ObIOManager::get_instance().destroy(); for (int64_t i = 0; i < perf_devices.count(); ++i) { ObLocalDevice *device_handle = perf_devices.at(i).device_handle_; device_handle->destroy(); } free(device_buf);} TEST_F(TestIOManager, IOTracer) { // use multi thread to do modify group_io_config bool is_perf_config_exist = false; ASSERT_SUCC(FileDirectoryUtils::is_exists(GROUP_PERF_CONFIG_FILE, is_perf_config_exist)); if (!is_perf_config_exist) { write_group_perf_config(); } // parse configs IOPerfScheduler scheduler_config; ObArray perf_devices; ObArray perf_tenants; ObArray perf_loads; ASSERT_SUCC(parse_group_perf_config(GROUP_PERF_CONFIG_FILE, scheduler_config, perf_devices, perf_tenants, perf_loads)); ASSERT_TRUE(perf_devices.count() > 0); ASSERT_TRUE(perf_tenants.count() > 0); ASSERT_TRUE(perf_loads.count() > 0); ObIOManager::get_instance().destroy(); const int64_t memory_limit = 30L * 1024L * 1024L * 1024L; // 30GB const int64_t queue_depth = 100L; ASSERT_SUCC(ObIOManager::get_instance().init(memory_limit, queue_depth, scheduler_config.sender_count_, scheduler_config.schedule_media_id_)); ASSERT_SUCC(ObIOManager::get_instance().start()); // prepare devices and files char *device_buf = (char *)malloc(sizeof(ObLocalDevice) * perf_devices.count()); ASSERT_TRUE(nullptr != device_buf); for (int64_t i = 0; i < perf_devices.count(); ++i) { IOPerfDevice &curr_config = perf_devices.at(i); ASSERT_SUCC(prepare_file(curr_config.file_path_, curr_config.file_size_, curr_config.fd_)); ObLocalDevice *device = new (device_buf + sizeof(ObLocalDevice) * i) ObLocalDevice; ASSERT_SUCC(init_device(curr_config.media_id_, *device)); ASSERT_SUCC(OB_IO_MANAGER.add_device_channel(device, curr_config.async_channel_count_, curr_config.sync_channel_count_, curr_config.max_io_depth_)); curr_config.device_handle_ = device; } // prepare tenant io manager for (int64_t i = 0; i < perf_tenants.count(); ++i) { IOPerfTenant &curr_config = perf_tenants.at(i); LOG_INFO("wenqu: tenant config", K(curr_config), K(i)); ObRefHolder tenant_holder; ASSERT_SUCC(OB_IO_MANAGER.get_tenant_io_manager(curr_config.tenant_id_, tenant_holder)); ASSERT_SUCC(tenant_holder.get_ptr()->refresh_group_io_config()); } // prepare perf runners char *runner_buf = (char *)malloc(perf_loads.count() * sizeof(IOPerfRunner)); char *modifyer_buf = (char *)malloc(perf_loads.count() * sizeof(IOTracerSwitch)); ObArray runners; ObArray switches; const int64_t start_ts = ObTimeUtility::current_time() + 10000L; for (int64_t i = 0; i < perf_loads.count(); ++i) { IOPerfRunner *runner = new (runner_buf + i * sizeof(IOPerfRunner)) IOPerfRunner(); const IOPerfLoad &cur_load = perf_loads.at(i); ASSERT_SUCC(runner->init(start_ts, cur_load)); ASSERT_SUCC(runners.push_back(runner)); LOG_INFO("runner start now"); } //open tracer for (int64_t i = 0; i < perf_tenants.count(); ++i) { IOTracerSwitch *tracer_switch = new (modifyer_buf + i * sizeof(IOTracerSwitch)) IOTracerSwitch(); IOPerfTenant &curr_tenant = perf_tenants.at(i); int64_t switch_init_ts = start_ts; int64_t switch_delay_ts = 1000000L; //1s后打开开关 ASSERT_SUCC(tracer_switch->init(switch_init_ts, switch_delay_ts, curr_tenant)); ASSERT_SUCC(switches.push_back(tracer_switch)); } // wait perf finished for (int64_t i = 0; i < runners.count(); ++i) { IOPerfRunner *runner = runners.at(i); runner->wait(); ASSERT_SUCC(runner->print_result()); runner->destroy(); } free(runner_buf); free(modifyer_buf); ObIOManager::get_instance().stop(); ObIOManager::get_instance().destroy(); for (int64_t i = 0; i < perf_devices.count(); ++i) { ObLocalDevice *device_handle = perf_devices.at(i).device_handle_; // ASSERT_SUCC(OB_IO_MANAGER.remove_device_channel(device_handle)); device_handle->destroy(); } free(device_buf); LOG_INFO("wenqu: modify finished"); } TEST_F(TestIOManager, ModifyIOPS) { // use multi thread to do modify group_io_config bool is_perf_config_exist = false; ASSERT_SUCC(FileDirectoryUtils::is_exists(GROUP_PERF_CONFIG_FILE, is_perf_config_exist)); if (!is_perf_config_exist) { write_group_perf_config(); } // parse configs IOPerfScheduler scheduler_config; ObArray perf_devices; ObArray perf_tenants; ObArray perf_loads; ASSERT_SUCC(parse_group_perf_config(GROUP_PERF_CONFIG_FILE, scheduler_config, perf_devices, perf_tenants, perf_loads)); ASSERT_TRUE(perf_devices.count() > 0); ASSERT_TRUE(perf_tenants.count() > 0); ASSERT_TRUE(perf_loads.count() > 0); ObIOManager::get_instance().destroy(); const int64_t memory_limit = 30L * 1024L * 1024L * 1024L; // 30GB const int64_t queue_depth = 100L; ASSERT_SUCC(ObIOManager::get_instance().init(memory_limit, queue_depth, scheduler_config.sender_count_, scheduler_config.schedule_media_id_)); ASSERT_SUCC(ObIOManager::get_instance().start()); // prepare devices and files char *device_buf = (char *)malloc(sizeof(ObLocalDevice) * perf_devices.count()); ASSERT_TRUE(nullptr != device_buf); for (int64_t i = 0; i < perf_devices.count(); ++i) { IOPerfDevice &curr_config = perf_devices.at(i); ASSERT_SUCC(prepare_file(curr_config.file_path_, curr_config.file_size_, curr_config.fd_)); ObLocalDevice *device = new (device_buf + sizeof(ObLocalDevice) * i) ObLocalDevice; ASSERT_SUCC(init_device(curr_config.media_id_, *device)); ASSERT_SUCC(OB_IO_MANAGER.add_device_channel(device, curr_config.async_channel_count_, curr_config.sync_channel_count_, curr_config.max_io_depth_)); curr_config.device_handle_ = device; } // prepare tenant io manager for (int64_t i = 0; i < perf_tenants.count(); ++i) { IOPerfTenant &curr_config = perf_tenants.at(i); LOG_INFO("wenqu: tenant config", K(curr_config), K(i)); ObRefHolder tenant_holder; ASSERT_SUCC(OB_IO_MANAGER.get_tenant_io_manager(curr_config.tenant_id_, tenant_holder)); ASSERT_SUCC(tenant_holder.get_ptr()->refresh_group_io_config()); } // prepare perf runners char *runner_buf = (char *)malloc(perf_loads.count() * sizeof(IOPerfRunner)); char *modifyer_buf = (char *)malloc(perf_loads.count() * sizeof(IOConfModify)); ObArray runners; ObArray modifyers; const int64_t start_ts = ObTimeUtility::current_time() + 10000L; for (int64_t i = 0; i < perf_loads.count(); ++i) { IOPerfRunner *runner = new (runner_buf + i * sizeof(IOPerfRunner)) IOPerfRunner(); const IOPerfLoad &cur_load = perf_loads.at(i); ASSERT_SUCC(runner->init(start_ts, cur_load)); ASSERT_SUCC(runners.push_back(runner)); LOG_INFO("runner start now"); } //prepare modifyer for (int64_t i = 0; i < perf_tenants.count(); ++i) { IOConfModify *modifyer=new (modifyer_buf + i * sizeof(IOConfModify)) IOConfModify(); IOPerfTenant &curr_tenant = perf_tenants.at(i); int64_t modify_init_ts = start_ts; int64_t modify_delay_ts = 3000000L; //2s后开始修改 ASSERT_SUCC(modifyer->init(modify_init_ts, modify_delay_ts, curr_tenant)); ASSERT_SUCC(modifyers.push_back(modifyer)); } // wait perf finished for (int64_t i = 0; i < runners.count(); ++i) { IOPerfRunner *runner = runners.at(i); runner->wait(); ASSERT_SUCC(runner->print_result()); runner->destroy(); } free(runner_buf); free(modifyer_buf); ObIOManager::get_instance().stop(); ObIOManager::get_instance().destroy(); for (int64_t i = 0; i < perf_devices.count(); ++i) { ObLocalDevice *device_handle = perf_devices.at(i).device_handle_; // ASSERT_SUCC(OB_IO_MANAGER.remove_device_channel(device_handle)); device_handle->destroy(); } free(device_buf); LOG_INFO("wenqu: modify finished"); } TEST_F(TestIOManager, ModifyCallbackThread) { // use multi thread to do modify group_io_config bool is_perf_config_exist = false; ASSERT_SUCC(FileDirectoryUtils::is_exists(GROUP_PERF_CONFIG_FILE, is_perf_config_exist)); if (!is_perf_config_exist) { write_group_perf_config(); } // parse configs IOPerfScheduler scheduler_config; ObArray perf_devices; ObArray perf_tenants; ObArray perf_loads; ASSERT_SUCC(parse_group_perf_config(GROUP_PERF_CONFIG_FILE, scheduler_config, perf_devices, perf_tenants, perf_loads)); ASSERT_TRUE(perf_devices.count() > 0); ASSERT_TRUE(perf_tenants.count() > 0); ASSERT_TRUE(perf_loads.count() > 0); ObIOManager::get_instance().destroy(); const int64_t memory_limit = 30L * 1024L * 1024L * 1024L; // 30GB const int64_t queue_depth = 100L; ASSERT_SUCC(ObIOManager::get_instance().init(memory_limit, queue_depth, scheduler_config.sender_count_, scheduler_config.schedule_media_id_)); ASSERT_SUCC(ObIOManager::get_instance().start()); // prepare devices and files char *device_buf = (char *)malloc(sizeof(ObLocalDevice) * perf_devices.count()); ASSERT_TRUE(nullptr != device_buf); for (int64_t i = 0; i < perf_devices.count(); ++i) { IOPerfDevice &curr_config = perf_devices.at(i); ASSERT_SUCC(prepare_file(curr_config.file_path_, curr_config.file_size_, curr_config.fd_)); ObLocalDevice *device = new (device_buf + sizeof(ObLocalDevice) * i) ObLocalDevice; ASSERT_SUCC(init_device(curr_config.media_id_, *device)); ASSERT_SUCC(OB_IO_MANAGER.add_device_channel(device, curr_config.async_channel_count_, curr_config.sync_channel_count_, curr_config.max_io_depth_)); curr_config.device_handle_ = device; } // prepare tenant io manager for (int64_t i = 0; i < perf_tenants.count(); ++i) { IOPerfTenant &curr_config = perf_tenants.at(i); LOG_INFO("wenqu: tenant config", K(curr_config), K(i)); ObRefHolder tenant_holder; ASSERT_SUCC(OB_IO_MANAGER.get_tenant_io_manager(curr_config.tenant_id_, tenant_holder)); ASSERT_SUCC(tenant_holder.get_ptr()->refresh_group_io_config()); } // prepare perf runners char *runner_buf = (char *)malloc(perf_loads.count() * sizeof(IOPerfRunner)); char *modifier_buf = (char *)malloc(perf_loads.count() * sizeof(IOConfModify)); ObArray runners; ObArray modifiers; const int64_t start_ts = ObTimeUtility::current_time() + 10000L; for (int64_t i = 0; i < perf_loads.count(); ++i) { IOPerfRunner *runner = new (runner_buf + i * sizeof(IOPerfRunner)) IOPerfRunner(); const IOPerfLoad &cur_load = perf_loads.at(i); ASSERT_SUCC(runner->init(start_ts, cur_load)); ASSERT_SUCC(runners.push_back(runner)); LOG_INFO("runner start now"); } //prepare modifier for (int64_t i = 0; i < perf_tenants.count(); ++i) { IOCallbackModifier *modifier=new (modifier_buf + i * sizeof(IOCallbackModifier)) IOCallbackModifier(); IOPerfTenant &curr_tenant = perf_tenants.at(i); int64_t modify_init_ts = start_ts; int64_t modify_delay_ts = 2000000L; //2s后开始修改 ASSERT_SUCC(modifier->init(modify_init_ts, modify_delay_ts, curr_tenant)); ASSERT_SUCC(modifiers.push_back(modifier)); } // wait perf finished for (int64_t i = 0; i < runners.count(); ++i) { IOPerfRunner *runner = runners.at(i); runner->wait(); ASSERT_SUCC(runner->print_result()); runner->destroy(); } free(runner_buf); free(modifier_buf); ObIOManager::get_instance().stop(); ObIOManager::get_instance().destroy(); for (int64_t i = 0; i < perf_devices.count(); ++i) { ObLocalDevice *device_handle = perf_devices.at(i).device_handle_; device_handle->destroy(); } free(device_buf); LOG_INFO("modify callback thread finished"); } TEST_F(TestIOManager, ModifyGroupIO) { // use multi thread to do modify group_io_config bool is_perf_config_exist = false; ASSERT_SUCC(FileDirectoryUtils::is_exists(GROUP_PERF_CONFIG_FILE, is_perf_config_exist)); if (!is_perf_config_exist) { write_group_perf_config(); } // parse configs IOPerfScheduler scheduler_config; ObArray perf_devices; ObArray perf_tenants; ObArray perf_loads; ASSERT_SUCC(parse_group_perf_config(GROUP_PERF_CONFIG_FILE, scheduler_config, perf_devices, perf_tenants, perf_loads)); ASSERT_TRUE(perf_devices.count() > 0); ASSERT_TRUE(perf_tenants.count() > 0); ASSERT_TRUE(perf_loads.count() > 0); ObIOManager::get_instance().destroy(); const int64_t memory_limit = 30L * 1024L * 1024L * 1024L; // 30GB const int64_t queue_depth = 100L; ASSERT_SUCC(ObIOManager::get_instance().init(memory_limit, queue_depth, scheduler_config.sender_count_, scheduler_config.schedule_media_id_)); ASSERT_SUCC(ObIOManager::get_instance().start()); // prepare devices and files char *device_buf = (char *)malloc(sizeof(ObLocalDevice) * perf_devices.count()); ASSERT_TRUE(nullptr != device_buf); for (int64_t i = 0; i < perf_devices.count(); ++i) { IOPerfDevice &curr_config = perf_devices.at(i); ASSERT_SUCC(prepare_file(curr_config.file_path_, curr_config.file_size_, curr_config.fd_)); ObLocalDevice *device = new (device_buf + sizeof(ObLocalDevice) * i) ObLocalDevice; ASSERT_SUCC(init_device(curr_config.media_id_, *device)); ASSERT_SUCC(OB_IO_MANAGER.add_device_channel(device, curr_config.async_channel_count_, curr_config.sync_channel_count_, curr_config.max_io_depth_)); curr_config.device_handle_ = device; } // prepare tenant io manager for (int64_t i = 0; i < perf_tenants.count(); ++i) { IOPerfTenant &curr_config = perf_tenants.at(i); if (curr_config.tenant_id_ == 1002) { LOG_INFO("qilu: tenant config", K(curr_config), K(i)); ObRefHolder tenant_holder; ASSERT_SUCC(OB_IO_MANAGER.get_tenant_io_manager(curr_config.tenant_id_, tenant_holder)); ASSERT_SUCC(tenant_holder.get_ptr()->refresh_group_io_config()); } } // prepare perf runners char *runner_buf = (char *)malloc(perf_loads.count() * sizeof(IOPerfRunner)); char *modifyer_buf = (char *)malloc(perf_loads.count() * sizeof(IOGroupModify)); ObArray runners; ObArray modifyers; const int64_t start_ts = ObTimeUtility::current_time() + 10000L; for (int64_t i = 0; i < perf_loads.count(); ++i) { IOPerfRunner *runner = new (runner_buf + i * sizeof(IOPerfRunner)) IOPerfRunner(); const IOPerfLoad &cur_load = perf_loads.at(i); if (cur_load.tenant_id_ == 1002) { ASSERT_SUCC(runner->init(start_ts, cur_load)); ASSERT_SUCC(runners.push_back(runner)); LOG_INFO("runner start now"); } } //prepare modifyer for (int64_t i = 0; i < perf_tenants.count(); ++i) { IOPerfTenant &curr_tenant = perf_tenants.at(i); if (curr_tenant.tenant_id_ == 1002) { IOGroupModify *modifyer=new (modifyer_buf + i * sizeof(IOGroupModify)) IOGroupModify(); int64_t modify_init_ts = start_ts; int64_t modify_delay_ts = 3000000L; //3s后开始修改 ASSERT_SUCC(modifyer->init(modify_init_ts, modify_delay_ts, curr_tenant)); ASSERT_SUCC(modifyers.push_back(modifyer)); } } // wait perf finished for (int64_t i = 0; i < runners.count(); ++i) { IOPerfRunner *runner = runners.at(i); runner->wait(); ASSERT_SUCC(runner->print_result()); runner->destroy(); } free(runner_buf); free(modifyer_buf); ObIOManager::get_instance().stop(); ObIOManager::get_instance().destroy(); for (int64_t i = 0; i < perf_devices.count(); ++i) { ObLocalDevice *device_handle = perf_devices.at(i).device_handle_; // ASSERT_SUCC(OB_IO_MANAGER.remove_device_channel(device_handle)); device_handle->destroy(); } free(device_buf); LOG_INFO("qilu: modify group finished"); } TEST_F(TestIOManager, abnormal) { // simulate submit failure // simulate get_event failure // simulate device hang } #define LOG_FILE_PATH "./test_io_manager.log" int main(int argc, char **argv) { set_memory_limit(20L * 1024L * 1024L * 1024L); system("rm -rf " LOG_FILE_PATH "*"); oceanbase::common::ObLogger::get_logger().set_log_level("INFO"); oceanbase::common::ObLogger::get_logger().set_file_name(LOG_FILE_PATH, true); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } /**************** test io callback ******************/ TestIOCallback::~TestIOCallback() { if (nullptr != number_) { *number_ -= 90; number_ = nullptr; } if (nullptr != allocator_) { if (nullptr != help_buf_) { allocator_->free(help_buf_); help_buf_ = nullptr; } allocator_->free(this); LOG_INFO("success reset callback when out_rec_cnt = 0"); } } int TestIOCallback::alloc_data_buf(const char *io_data_buffer, const int64_t data_size) { int ret = OB_SUCCESS; if (OB_ISNULL(allocator_)) { ret = OB_INVALID_DATA; LOG_WARN("Invalid data, the allocator is NULL, ", K(ret)); } else if (OB_UNLIKELY(data_size <= 0)) { ret = OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "invalid data buffer size", K(ret), K(data_size)); } else if (OB_ISNULL(help_buf_ = static_cast(allocator_->alloc(data_size)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("Failed to allocate help buf", K(ret), K(data_size), KP(help_buf_)); } else { memset(help_buf_, 0, data_size); MEMCPY(help_buf_, io_data_buffer, data_size); } return OB_SUCCESS; } int TestIOCallback::inner_process(const char *data_buffer, const int64_t size) { int ret = OB_SUCCESS; if (OB_ISNULL(allocator_)) { // for test, ignore } else if (OB_FAIL(alloc_data_buf(data_buffer, size))) { LOG_WARN("Fail to allocate memory, ", K(ret), K(size)); } if (nullptr != number_) { *number_ += 100; } return OB_SUCCESS; } /**************** io perf ******************/ void write_group_perf_config() { int fd = -1; const char *file_name = GROUP_PERF_CONFIG_FILE; if (0 > (fd = ::open(file_name, O_RDWR | O_CREAT | O_TRUNC, 0644))) { LOG_WARN_RET(OB_ERR_SYS, "open perf config file failed", K(fd), K(file_name)); } else { const char *file_buf = "sender_count schedule_media_id io_greed\n" "8 0 0\n" "\n" "device_id media_id async_channel sync_channel max_io_depth file_size_gb file_path\n" "1 0 8 1 64 1 ./perf_test\n" "\n" "tenant_id min_iops max_iops weight group\n" "1 5000 100000 700 10001: testgroup1: 80, 100, 60; 10002: testgroup2: 10, 60, 30; 0: OTHER_GROUPS: 10, 100, 10;\n" "500 1000 50000 1000 12345: testgroup1: 50, 50, 50; 0: OTHER_GROUPS: 50, 50, 50;\n" "\n" "tenant_id device_id group io_mode io_size_byte io_depth perf_mode target_iops thread_count is_sequence start_s stop_s\n" "1 1 0 r 16384 10 rolling 0 16 0 0 8\n" "1 1 10001 r 16384 10 rolling 0 16 0 2 7\n" "1 1 10002 r 16384 10 rolling 0 16 0 0 6\n" "500 1 0 r 16384 100 rolling 0 16 0 0 10\n" "500 1 12345 r 16384 100 rolling 0 16 0 0 10\n" ; const int64_t file_len = strlen(file_buf); int write_ret = ::write(fd, file_buf, file_len); if (write_ret < file_len) { LOG_WARN_RET(OB_ERR_SYS, "write file content failed", K(write_ret), K(file_len)); } close(fd); } } int parse_group_perf_config(const char *config_file_path, IOPerfScheduler &scheduler_config, ObIArray &perf_devices, ObIArray &perf_tenants, ObIArray &perf_loads) { int ret = OB_SUCCESS; bool is_file_exist = false; FILE *file = nullptr; if (OB_FAIL(FileDirectoryUtils::is_exists(config_file_path, is_file_exist))) { LOG_WARN("failed to check file exists", K(ret)); } else if (!is_file_exist) { ret = OB_FILE_NOT_EXIST; LOG_WARN("file not exist!"); } else if (OB_UNLIKELY(nullptr == (file = ::fopen(config_file_path, "r")))) { ret = OB_IO_ERROR; LOG_WARN("failed to open iops data file, ", K(ret), K(errno), KERRMSG); } else { char curr_line[1024] = { 0 }; const char *scheduler_header = "sender_count schedule_media_id io_greed"; const char *device_header = "device_id media_id async_channel sync_channel max_io_depth file_size_gb file_path"; const char *tenant_header = "tenant_id min_iops max_iops weight group"; const char *load_header = "tenant_id device_id group io_mode io_size_byte io_depth perf_mode target_iops thread_count is_sequence start_s stop_s"; enum class PerfConfigType { SCHEDULER, DEVICE, TENANT, LOAD, MAX }; PerfConfigType config_type = PerfConfigType::MAX; while (OB_SUCC(ret)) { if (OB_UNLIKELY(nullptr == fgets(curr_line, sizeof(curr_line), file))) { ret = OB_ITER_END; } else if (' ' == curr_line[0] || '\0' == curr_line[0] || '#' == curr_line[0]) { // ignore empty line } else if (0 == atoi(curr_line)) { if (0 == strncmp(curr_line, scheduler_header, strlen(scheduler_header))) { config_type = PerfConfigType::SCHEDULER; } else if (0 == strncmp(curr_line, device_header, strlen(device_header))) { config_type = PerfConfigType::DEVICE; } else if (0 == strncmp(curr_line, tenant_header, strlen(tenant_header))) { config_type = PerfConfigType::TENANT; } else if (0 == strncmp(curr_line, load_header, strlen(load_header))) { config_type = PerfConfigType::LOAD; } } else if (PerfConfigType::SCHEDULER == config_type) { int scan_ret = sscanf(curr_line, "%ld%ld%ld\n", &scheduler_config.sender_count_, &scheduler_config.schedule_media_id_, &scheduler_config.io_greed_); if (OB_UNLIKELY(3 != scan_ret)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("scan config file failed", K(ret), K(scan_ret)); } LOG_INFO("qilu: parse scheduler config", K(ret), K(scheduler_config)); } else if (PerfConfigType::DEVICE == config_type) { IOPerfDevice item; int scan_ret = sscanf(curr_line, "%d%d%d%d%d%ld%s\n", &item.device_id_, &item.media_id_, &item.async_channel_count_, &item.sync_channel_count_, &item.max_io_depth_, &item.file_size_, item.file_path_); if (OB_UNLIKELY(7 != scan_ret)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("scan config file failed", K(ret), K(scan_ret)); } else { item.file_size_ *= 1024L * 1024L * 1024L; if (OB_UNLIKELY(!item.is_valid())) { ret = OB_INVALID_DATA; LOG_WARN("invalid data", K(ret), K(item)); } else if (OB_FAIL(perf_devices.push_back(item))) { LOG_WARN("add item failed", K(ret), K(item)); } LOG_INFO("qilu: parse device", K(ret), K(item)); } } else if (PerfConfigType::TENANT == config_type) { IOPerfTenant item; char group_config[1024] = { 0 }; int scan_ret = sscanf(curr_line, "%d%ld%ld%ld%[^\n]\n", &item.tenant_id_, &item.config_.unit_config_.min_iops_, &item.config_.unit_config_.max_iops_, &item.config_.unit_config_.weight_, group_config); if (OB_UNLIKELY(5 != scan_ret)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("scan config file failed", K(ret), K(scan_ret)); } else { item.config_.memory_limit_ = IO_MEMORY_LIMIT; item.config_.callback_thread_count_ = 0; // parse group config if (OB_FAIL(item.config_.parse_group_config(group_config))) { LOG_WARN("parse group config failed", K(ret), K(group_config)); } else if (OB_UNLIKELY(!item.is_valid())) { ret = OB_INVALID_DATA; LOG_WARN("invalid data", K(ret), K(item)); } else if (OB_FAIL(perf_tenants.push_back(item))) { LOG_WARN("add item failed", K(ret), K(item)); } LOG_INFO("qilu: parse tenant", K(ret), K(item), K(group_config), K(item.config_)); } } else if (PerfConfigType::LOAD == config_type) { IOPerfLoad item; char io_mode[16] = { 0 }; char perf_mode[16] = { 0 }; int scan_ret = sscanf(curr_line, "%d%d%ld%s%d%d%s%ld%d%d%ld%ld\n", &item.tenant_id_, &item.device_id_, &item.group_id_, io_mode, &item.size_, &item.depth_, perf_mode, &item.iops_, &item.thread_count_, (int *)&item.is_sequence_, &item.start_delay_ts_, &item.stop_delay_ts_); if (OB_UNLIKELY(12 != scan_ret)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("scan config file failed", K(ret), K(scan_ret)); } else { item.start_delay_ts_ *= 1000000L; item.stop_delay_ts_ *= 1000000L; switch (io_mode[0]) { case 'r' : item.mode_ = ObIOMode::READ; break; case 'w': item.mode_ = ObIOMode::WRITE; break; default: break; } const char *rolling_str = "rolling"; const char *batch_str = "batch"; const char *async_str = "async"; if (0 == strncmp(perf_mode, rolling_str, strlen(rolling_str))) { item.perf_mode_ = IOPerfMode::ROLLING; } else if (0 == strncmp(perf_mode, batch_str, strlen(batch_str))) { item.perf_mode_ = IOPerfMode::BATCH; } else if (0 == strncmp(perf_mode, async_str, strlen(async_str))) { item.perf_mode_ = IOPerfMode::ASYNC; } for (int64_t i = 0; OB_SUCC(ret) && i < perf_devices.count(); ++i) { if (item.device_id_ == perf_devices.at(i).device_id_) { item.device_ = &perf_devices.at(i); break; } } if (OB_UNLIKELY(!item.is_valid())) { ret = OB_INVALID_DATA; LOG_WARN("invalid data", K(ret), K(item)); } else if (OB_FAIL(perf_loads.push_back(item))) { LOG_WARN("add item failed", K(ret), K(item)); } LOG_INFO("qilu: parse load", K(ret), K(item), K(io_mode), K(item.group_id_)); } } else { ret = OB_ERR_UNEXPECTED; LOG_WARN("config type not match", K(ret), K(config_type)); } } if (OB_ITER_END == ret) { ret = OB_SUCCESS; } ::fclose(file); } return ret; } int IOPerfRunner::init(const int64_t absolute_ts, const IOPerfLoad &load) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!load.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(load)); } else if (OB_FAIL(allocator_.init(OB_MALLOC_BIG_BLOCK_SIZE, "perf runner", OB_SERVER_TENANT_ID, 1024L * 1024L * 1024L * 10L))) { LOG_WARN("init allocator failed", K(ret)); } else if (OB_FAIL(handle_queue_.init(1000L * 10000L, &allocator_))) { LOG_WARN("init handle queue failed", K(ret)); } else { abs_ts_ = absolute_ts; load_ = load; // prepare file fd_.first_id_ = ObIOFd::NORMAL_FILE_ID; fd_.second_id_ = load.device_->fd_; fd_.device_handle_ = load.device_->device_handle_; // prepare write buffer if (OB_SUCC(ret) && ObIOMode::WRITE == load_.mode_ && nullptr == write_buf_) { const int64_t buf_size = load_.size_ + DIO_READ_ALIGN_SIZE; void *tmp_buf = ob_malloc(buf_size, ObNewModIds::TEST); if (nullptr == tmp_buf) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("allocate memory failed", K(ret), K(buf_size)); } else { memset(tmp_buf, 'a', buf_size); write_buf_ = reinterpret_cast(tmp_buf); } } // prepare read buffer if (OB_SUCC(ret) && ObIOMode::READ == load_.mode_ && nullptr == user_buf_) { const int64_t buf_size = load_.size_; void *tmp_buf = ob_malloc(buf_size, ObNewModIds::TEST); if (nullptr == tmp_buf) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("allocate memory failed", K(ret), K(buf_size)); } else { memset(tmp_buf, '0', buf_size); user_buf_ = reinterpret_cast(tmp_buf); } } } if (OB_SUCC(ret)) { if (OB_FAIL(set_thread_count(load_.thread_count_ + 2))) { LOG_WARN("set thread count failed", K(ret), K(load_)); } else if (OB_FAIL(start())) { LOG_WARN("start thread pool failed", K(ret), K(load_)); } } if (OB_FAIL(ret)) { destroy(); } return ret; } void IOPerfRunner::destroy() { stop(); wait(); load_ = IOPerfLoad(); result_ = IOPerfResult(); last_offset_ = 0; if (nullptr != write_buf_) { ob_free((void *)write_buf_); write_buf_ = nullptr; } if (nullptr != user_buf_) { ob_free((void *)user_buf_); user_buf_ = nullptr; } fd_.reset(); } void IOPerfRunner::run1() { int ret = OB_SUCCESS; const int64_t thread_idx = get_thread_idx(); if (thread_idx < load_.thread_count_) { const int64_t current_ts = ObTimeUtility::current_time(); if (abs_ts_ + load_.start_delay_ts_ > current_ts) { usleep(abs_ts_ + load_.start_delay_ts_ - current_ts); } ATOMIC_CAS(&result_.start_delay_ts_, 0, ObTimeUtility::current_time()); if (IOPerfMode::ROLLING == load_.perf_mode_) { do_perf_rolling(); } else { do_perf_batch(); } } else if (thread_idx == load_.thread_count_) { const int64_t current_ts = ObTimeUtility::current_time(); if (abs_ts_ + load_.stop_delay_ts_ > current_ts) { usleep(abs_ts_ + load_.stop_delay_ts_ - current_ts); } stop(); usleep(100L * 1000L); wait_handles(handle_queue_); result_.stop_delay_ts_ = ObTimeUtility::current_time(); } else { while (!has_set_stop()) { usleep(10L * 1000L); wait_handles(handle_queue_); } } } int IOPerfRunner::do_perf_batch() { int ret = OB_SUCCESS; static const int64_t check_interval_ms = 50L; // 50ms int64_t io_count = 0; const int64_t check_count = load_.iops_ / load_.thread_count_ / (1000L / check_interval_ms); int64_t last_check_ts = ObTimeUtility::fast_current_time(); const bool need_control_io_speed = 0 != load_.iops_; LOG_INFO("perf start", K(load_.tenant_id_), K(load_.group_id_)); while (!has_set_stop()) { (void) do_batch_io(); if (need_control_io_speed) { io_count += load_.depth_; if (io_count > check_count) { const int64_t sleep_us = check_interval_ms * 1000L - (ObTimeUtility::fast_current_time() - last_check_ts); if (sleep_us > 0) { usleep(sleep_us); } io_count -= check_count; last_check_ts = ObTimeUtility::fast_current_time(); } } } return ret; } int IOPerfRunner::do_perf_rolling() { int ret = OB_SUCCESS; ObIOInfo info; info.tenant_id_ = load_.tenant_id_; info.flag_.set_group_id(load_.group_id_); info.flag_.set_mode(load_.mode_); info.flag_.set_wait_event(ObWaitEventIds::DB_FILE_DATA_READ); info.fd_ = fd_; info.size_ = load_.size_; info.timeout_us_ = DEFAULT_IO_WAIT_TIME_US; if (ObIOMode::WRITE == load_.mode_) { info.buf_ = reinterpret_cast(upper_align(reinterpret_cast(write_buf_), DIO_READ_ALIGN_SIZE)); } else if (ObIOMode::READ == load_.mode_) { info.user_data_buf_ = user_buf_; } ObArray handles; RLOCAL(int64_t, local_io_count); local_io_count = 0; const int64_t max_offset = load_.device_->file_size_ - load_.size_; for (int32_t i = 0; OB_SUCC(ret) && i < load_.depth_; ++i) { void *buf = nullptr; if (OB_ISNULL(buf = allocator_.alloc(sizeof(ObIOHandle)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("allocate memory failed", K(ret)); } else { ObIOHandle *cur_handle = new (buf) ObIOHandle; if (load_.is_sequence_) { last_offset_ += load_.size_; info.offset_ = last_offset_ % (max_offset); } else { info.offset_ = lower_align(ObRandom::rand(0, max_offset), DIO_READ_ALIGN_SIZE); } if (info.flag_.is_read()) { if (OB_FAIL(ObIOManager::get_instance().aio_read(info, *cur_handle))) { LOG_WARN("fail to aio read", K(ret), K(i)); } } else if (info.flag_.is_write()) { if (OB_FAIL(ObIOManager::get_instance().aio_write(info, *cur_handle))) { LOG_WARN("fail to aio write", K(ret), K(i)); } } if (OB_SUCC(ret)) { ++local_io_count; if (OB_FAIL(handles.push_back(cur_handle))) { LOG_WARN("push back handle failed", K(ret), KPC(cur_handle)); } } } } int64_t pos = 0; while (!has_set_stop() && OB_SUCC(ret)) { if (REACH_TIME_INTERVAL(1000L * 1000L)) { ATOMIC_FAA(&io_count_, local_io_count); ATOMIC_FAA(&total_io_count_, local_io_count); local_io_count = 0; ATOMIC_INC(&report_count_); if (load_.thread_count_ == ATOMIC_LOAD(&report_count_)) { // LOG_INFO("IO STATUS: perf io count", K(io_count_), K(load_.category_), K(total_io_count_)); report_count_ = 0; io_count_ = 0; } } pos = pos % handles.count(); ObIOHandle *cur_handle = handles[pos]; wait_and_count(*cur_handle); if (load_.is_sequence_) { last_offset_ += load_.size_; info.offset_ = last_offset_ % (max_offset); } else { info.offset_ = lower_align(ObRandom::rand(0, max_offset), DIO_READ_ALIGN_SIZE); } if (info.flag_.is_read()) { if (OB_FAIL(ObIOManager::get_instance().aio_read(info, *cur_handle))) { LOG_WARN("fail to aio read", K(ret)); } } else if (info.flag_.is_write()) { if (OB_FAIL(ObIOManager::get_instance().aio_write(info, *cur_handle))) { LOG_WARN("fail to aio write", K(ret)); } } if (OB_SUCC(ret)) { ++local_io_count; } ++pos; } for (int64_t i = 0; OB_SUCC(ret) && i < handles.count(); ++i) { int64_t tmp_pos = (pos + i) % handles.count(); ObIOHandle *cur_handle = handles[tmp_pos]; wait_and_count(*cur_handle); cur_handle->~ObIOHandle(); allocator_.free(cur_handle); } return ret; } int IOPerfRunner::do_batch_io() { int ret = OB_SUCCESS; ObIOInfo info; info.tenant_id_ = load_.tenant_id_; info.flag_.set_group_id(load_.group_id_); info.flag_.set_mode(load_.mode_); info.flag_.set_wait_event(ObWaitEventIds::DB_FILE_DATA_READ); info.fd_ = fd_; info.size_ = load_.size_; info.timeout_us_ = DEFAULT_IO_WAIT_TIME_US; if (ObIOMode::WRITE == load_.mode_) { info.buf_ = reinterpret_cast(upper_align(reinterpret_cast(write_buf_), DIO_READ_ALIGN_SIZE)); } else if (ObIOMode::READ == load_.mode_) { info.user_data_buf_ = user_buf_; } ObFixedQueue local_handles; if (OB_FAIL(local_handles.init(load_.depth_ * 2))) { LOG_WARN("init handles array failed", K(ret)); } ObFixedQueue &handles = IOPerfMode::ASYNC == load_.perf_mode_ ? handle_queue_ : local_handles; const int64_t max_offset = load_.device_->file_size_ - load_.size_; for (int32_t i = 0; OB_SUCC(ret) && i < load_.depth_; ++i) { void *buf = nullptr; if (OB_ISNULL(buf = allocator_.alloc(sizeof(ObIOHandle)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("allocate memory failed", K(ret)); } else { ObIOHandle *cur_handle = new (buf) ObIOHandle; if (load_.is_sequence_) { last_offset_ += load_.size_; info.offset_ = last_offset_ % (max_offset); } else { info.offset_ = lower_align(ObRandom::rand(0, max_offset), DIO_READ_ALIGN_SIZE); } if (info.flag_.is_read()) { if (OB_FAIL(ObIOManager::get_instance().aio_read(info, *cur_handle))) { LOG_WARN("fail to aio read", K(ret), K(i)); } } else if (info.flag_.is_write()) { if (OB_FAIL(ObIOManager::get_instance().aio_write(info, *cur_handle))) { LOG_WARN("fail to aio write", K(ret), K(i)); } } if (OB_FAIL(ret)) { } else if (OB_FAIL(handles.push(cur_handle))) { LOG_WARN("push back handle failed", K(ret), KPC(cur_handle)); } else if (local_handles.get_total() > 0) { wait_handles(local_handles); } } } return ret; } int IOPerfRunner::wait_and_count(ObIOHandle &io_handle) { int ret = OB_SUCCESS; if (OB_FAIL(io_handle.wait())) { LOG_WARN("fail to wait read io", K(ret)); ATOMIC_INC(&result_.fail_count_); } else { ATOMIC_INC(&result_.succ_count_); } const int64_t delay = max(0, io_handle.get_rt()); ATOMIC_FAA(&result_.sum_rt_, delay); return ret; } int IOPerfRunner::wait_handles(ObFixedQueue &handles) { int ret = OB_SUCCESS; ObIOHandle *cur_handle = nullptr; while (OB_SUCC(handles.pop(cur_handle))) { wait_and_count(*cur_handle); cur_handle->~ObIOHandle(); allocator_.free(cur_handle); } return ret; } int IOPerfRunner::print_result() { int ret = OB_SUCCESS; if (!(result_.start_delay_ts_ > 0 && result_.stop_delay_ts_ > 0 && result_.stop_delay_ts_ > result_.start_delay_ts_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("io perf not finished", K(ret), K(result_)); } else { LOG_INFO("io perf result", "perf_config", load_, "succ_cnt", result_.succ_count_, "fail_cnt", result_.fail_count_, "time_ms", (result_.stop_delay_ts_ - result_.start_delay_ts_) / 1000, "iops", result_.succ_count_ * 1000000L / max(1, result_.stop_delay_ts_ - result_.start_delay_ts_), "rt", result_.sum_rt_ / max(1, result_.succ_count_), "disk_rt", result_.disk_rt_ / max(1, result_.succ_count_)); } return ret; } int IOConfModify::init(int64_t modify_init_ts, int64_t modify_delay_ts, const IOPerfTenant &curr_tenant) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!curr_tenant.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(curr_tenant)); } else if (OB_FAIL(allocator_.init(OB_MALLOC_BIG_BLOCK_SIZE, "Modifier runner", OB_SERVER_TENANT_ID, 1024L * 1024L * 1024L * 10L))) { LOG_WARN("init allocator failed", K(ret)); } else { curr_tenant_ = curr_tenant; modify_init_ts_ = modify_init_ts; modify_delay_ts_ = modify_delay_ts; } if (OB_SUCC(ret)) { if (OB_FAIL(set_thread_count(load_.thread_count_ + 1))) { LOG_WARN("set thread count failed", K(ret), K(modify_init_ts_), K(curr_tenant_)); } else if (OB_FAIL(start())) { LOG_WARN("start thread failed", K(ret), K(modify_init_ts_), K(curr_tenant_)); } } if (OB_FAIL(ret)) { destroy(); } return ret; } void IOConfModify::destroy() { stop(); wait(); curr_tenant_ = IOPerfTenant(); } void IOConfModify::run1() { int ret = OB_SUCCESS; const int64_t thread_idx = get_thread_idx(); LOG_INFO("modify thread start"); const int64_t current_ts = ObTimeUtility::current_time(); if (modify_init_ts_ + modify_delay_ts_ > current_ts) { usleep(modify_init_ts_ + modify_delay_ts_ - current_ts); } int64_t min_test = 50000; int64_t max_test = 1000000; int64_t weight_test = 1000; if (OB_FAIL(modify_tenant_io(min_test, max_test, weight_test, curr_tenant_))) { LOG_WARN("modify config failed", K(ret), K(curr_tenant_)); } } int IOConfModify::modify_tenant_io( const int64_t min_iops, const int64_t max_iops, const int64_t weight, IOPerfTenant &curr_tenant) { int ret = OB_SUCCESS; //ObTenantIOConfig io_config; curr_tenant.config_.unit_config_.min_iops_ = min_iops; curr_tenant.config_.unit_config_.max_iops_ = max_iops; curr_tenant.config_.unit_config_.weight_ = weight; if (OB_FAIL(OB_IO_MANAGER.refresh_tenant_io_config(curr_tenant.tenant_id_, curr_tenant.config_))) { LOG_WARN("refresh tenant io config failed", K(ret), K(curr_tenant.tenant_id_), K(curr_tenant.config_)); } return ret; } int IOGroupModify::init(int64_t modify_init_ts, int64_t modify_delay_ts, const IOPerfTenant &curr_tenant) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!curr_tenant.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(curr_tenant)); } else if (OB_FAIL(allocator_.init(OB_MALLOC_BIG_BLOCK_SIZE, "group modifier", OB_SERVER_TENANT_ID, 1024L * 1024L * 1024L * 10L))) { LOG_WARN("init allocator failed", K(ret)); } else { curr_tenant_ = curr_tenant; modify_init_ts_ = modify_init_ts; modify_delay_ts_ = modify_delay_ts; } if (OB_SUCC(ret)) { if (OB_FAIL(set_thread_count(load_.thread_count_ + 1))) { LOG_WARN("set thread count failed", K(ret), K(modify_init_ts_), K(curr_tenant_)); } else if (OB_FAIL(start())) { LOG_WARN("start thread failed", K(ret), K(modify_init_ts_), K(curr_tenant_)); } } if (OB_FAIL(ret)) { destroy(); } return ret; } void IOGroupModify::destroy() { stop(); wait(); curr_tenant_ = IOPerfTenant(); } void IOGroupModify::run1() { int ret = OB_SUCCESS; const int64_t thread_idx = get_thread_idx(); LOG_INFO("modify thread start"); //change 1 int64_t current_ts = ObTimeUtility::current_time(); if (modify_init_ts_ + modify_delay_ts_ > current_ts) { usleep(modify_init_ts_ + modify_delay_ts_ - current_ts); } if (OB_FAIL(OB_IO_MANAGER.modify_group_io_config(curr_tenant_.tenant_id_, INT64_MAX, 100, 100, 100))) { LOG_WARN("fail to modify group config", K(ret)); } else if (OB_FAIL(OB_IO_MANAGER.modify_group_io_config(curr_tenant_.tenant_id_, 0, 0, 0, 0))) { LOG_WARN("fail to modify group config", K(ret)); } //change 2 usleep(3000L * 1000L); // sleep 3s if (OB_FAIL(OB_IO_MANAGER.modify_group_io_config(curr_tenant_.tenant_id_, INT64_MAX, 40, 40, 40))) { LOG_WARN("fail to modify group config", K(ret)); } else if (OB_FAIL(OB_IO_MANAGER.modify_group_io_config(curr_tenant_.tenant_id_, 0, 60, 60, 60))) { LOG_WARN("fail to modify group config", K(ret)); } } int IOTracerSwitch::init(int64_t switch_init_ts, int64_t switch_delay_ts, const IOPerfTenant &curr_tenant) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!curr_tenant.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(curr_tenant)); } else if (OB_FAIL(allocator_.init(OB_MALLOC_BIG_BLOCK_SIZE, "Switch runner", OB_SERVER_TENANT_ID, 1024L * 1024L * 1024L * 10L))) { LOG_WARN("init allocator failed", K(ret)); } else { curr_tenant_ = curr_tenant; switch_init_ts_ = switch_init_ts; switch_delay_ts_ = switch_delay_ts; } if (OB_SUCC(ret)) { if (OB_FAIL(set_thread_count(load_.thread_count_ + 1))) { LOG_WARN("set thread count failed", K(ret), K(load_)); } else if (OB_FAIL(start())) { LOG_WARN("start thread failed", K(ret)); } } if (OB_FAIL(ret)) { destroy(); } return ret; } void IOTracerSwitch::destroy() { stop(); wait(); curr_tenant_ = IOPerfTenant(); } void IOTracerSwitch::run1() { int ret = OB_SUCCESS; const int64_t thread_idx = get_thread_idx(); LOG_INFO("modify thread start"); const int64_t current_ts = ObTimeUtility::current_time(); if (switch_init_ts_ + switch_delay_ts_ > current_ts) { usleep(switch_init_ts_ + switch_delay_ts_ - current_ts); } if (OB_FAIL(modify_tenant_io(curr_tenant_))) { LOG_WARN("modify config failed", K(ret), K(curr_tenant_)); } } int IOTracerSwitch::modify_tenant_io(IOPerfTenant &curr_tenant) { int ret = OB_SUCCESS; ATOMIC_SET(&curr_tenant.config_.enable_io_tracer_, true); if (OB_FAIL(OB_IO_MANAGER.refresh_tenant_io_config(curr_tenant.tenant_id_, curr_tenant.config_))) { LOG_WARN("refresh tenant io config failed", K(ret), K(curr_tenant.tenant_id_), K(curr_tenant.config_)); } return ret; } int IOCallbackModifier::init(int64_t modify_init_ts, int64_t modify_delay_ts, const IOPerfTenant &curr_tenant) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!curr_tenant.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(curr_tenant)); } else if (OB_FAIL(allocator_.init(OB_MALLOC_BIG_BLOCK_SIZE, "Modifier runner", OB_SERVER_TENANT_ID, 1024L * 1024L * 1024L * 10L))) { LOG_WARN("init allocator failed", K(ret)); } else { curr_tenant_ = curr_tenant; modify_init_ts_ = modify_init_ts; modify_delay_ts_ = modify_delay_ts; } if (OB_SUCC(ret)) { if (OB_FAIL(set_thread_count(load_.thread_count_ + 1))) { LOG_WARN("set thread count failed", K(ret), K(modify_init_ts_), K(curr_tenant_)); } else if (OB_FAIL(start())) { LOG_WARN("start thread failed", K(ret), K(modify_init_ts_), K(curr_tenant_)); } } if (OB_FAIL(ret)) { destroy(); } return ret; } void IOCallbackModifier::destroy() { stop(); wait(); curr_tenant_ = IOPerfTenant(); } void IOCallbackModifier::run1() { int ret = OB_SUCCESS; const int64_t thread_idx = get_thread_idx(); LOG_INFO("modify thread start"); const int64_t current_ts = ObTimeUtility::current_time(); if (modify_init_ts_ + modify_delay_ts_ > current_ts) { usleep(modify_init_ts_ + modify_delay_ts_ - current_ts); } int64_t new_callback_num = 16; if (OB_FAIL(modify_callback_num(new_callback_num, curr_tenant_))) { LOG_WARN("modify config failed", K(ret), K(curr_tenant_)); } else { LOG_INFO("modify callback thread num success", K(curr_tenant_)); } } int IOCallbackModifier::modify_callback_num(const int64_t thread_num, IOPerfTenant &curr_tenant) { int ret = OB_SUCCESS; curr_tenant.config_.callback_thread_count_ = thread_num; if (OB_FAIL(OB_IO_MANAGER.refresh_tenant_io_config(curr_tenant.tenant_id_, curr_tenant.config_))) { LOG_WARN("refresh tenant io config failed", K(ret), K(curr_tenant.tenant_id_), K(curr_tenant.config_)); } return ret; }