hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FEAT MERGE] system resource optimization

Browse Source 
Co-authored-by: tushicheng <18829573815@163.com>
Co-authored-by: HaHaJeff <jeffzhouhhh@gmail.com>
Co-authored-by: dimstars <liangjinrongcm@gmail.com>
This commit is contained in:
obdev
2023-08-29 10:41:01 +00:00
committed by ob-robot
parent 7678a4a676
commit 81b8923b14
191 changed files with 2817 additions and 2464 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
mittest/simple_server/env/ob_simple_cluster_test_base.cpp vendored
View File

@ -161,7 +161,6 @@ int ObSimpleClusterTestBase::start()
GCONF.enable_perf_event = false;
GCONF.enable_sql_audit = true;
GCONF.enable_record_trace_log = false;
GMEMCONF.set_server_memory_limit(10 * 1024 * 1024 * 1024ul);
int32_t log_level;
bool change_log_level = false;

99
mittest/simple_server/test_observer_expand_shrink.cpp
View File

@ -210,105 +210,6 @@ TEST_F(ObserverExpandShink, resize_tenant_log_disk)
// EXPECT_NE(0, GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_);
// SERVER_LOG(INFO, "create_tenant_after_restart trace", KPC(GCTX.log_block_mgr_));
//}
TEST_F(ObserverExpandShink, test_hidden_sys_tenant)
{
omt::ObMultiTenant *omt = GCTX.omt_;
bool remove_tenant_succ = false;
int64_t log_disk_size_in_use = GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_;
share::TenantUnits units;
EXPECT_EQ(OB_SUCCESS, omt->get_tenant_units(units, false));
EXPECT_EQ(false, units.empty());
int64_t origin_sys_log_disk_size = 0;
int64_t hidden_sys_log_disk_size = 0;
for (int i = 0; i < units.count(); i++) {
if (OB_SYS_TENANT_ID == units[i].tenant_id_) {
origin_sys_log_disk_size = units[i].config_.log_disk_size();
EXPECT_EQ(OB_SUCCESS, omt->convert_real_to_hidden_sys_tenant());
}
}
EXPECT_EQ(OB_SUCCESS, omt->get_tenant_units(units, false));
EXPECT_EQ(false, units.empty());
ObUnitInfoGetter::ObTenantConfig sys_unit_config;
for (int i = 0; i < units.count(); i++) {
if (OB_SYS_TENANT_ID == units[i].tenant_id_) {
hidden_sys_log_disk_size = units[i].config_.log_disk_size();
sys_unit_config = units[i];
sys_unit_config.config_.resource_.log_disk_size_ = origin_sys_log_disk_size + 512*1024*1024;
}
}
CLOG_LOG(INFO, "runlin trace convert_hidden_to_real_sys_tenant", K(log_disk_size_in_use), KPC(GCTX.log_block_mgr_), K(origin_sys_log_disk_size), K(hidden_sys_log_disk_size));
// 类型转换后，sys租户的unit规格可能会发生变化（隐藏sys租户的规格会被重新生成，具体逻辑参见gen_sys_tenant_unit_config）
EXPECT_EQ(log_disk_size_in_use-origin_sys_log_disk_size+hidden_sys_log_disk_size,
GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_);
log_disk_size_in_use = GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_;
EXPECT_EQ(OB_SUCCESS, omt->convert_hidden_to_real_sys_tenant(sys_unit_config));
EXPECT_EQ(log_disk_size_in_use-hidden_sys_log_disk_size+sys_unit_config.config_.log_disk_size(),
GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_);
log_disk_size_in_use = GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_;
CLOG_LOG(INFO, "runlin trace after convert_hidden_to_real_sys_tenant", K(log_disk_size_in_use), KPC(GCTX.log_block_mgr_),
K(origin_sys_log_disk_size));
int64_t new_sys_log_disk_size = sys_unit_config.config_.log_disk_size()+512*1024*1024;
omt::ObTenant *tenant = nullptr;
EXPECT_EQ(OB_SUCCESS, omt->get_tenant(OB_SYS_TENANT_ID, tenant));
share::ObUnitInfoGetter::ObTenantConfig unit_config = tenant->get_unit();
unit_config.config_.resource_.log_disk_size_ = new_sys_log_disk_size;
// 扩容直接成功
EXPECT_EQ(OB_SUCCESS, omt->update_tenant_unit(unit_config));
unit_config = tenant->get_unit();
EXPECT_EQ(unit_config.config_.log_disk_size(), new_sys_log_disk_size);
EXPECT_EQ(log_disk_size_in_use-sys_unit_config.config_.log_disk_size()+new_sys_log_disk_size,
GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_);
CLOG_LOG(INFO, "runlin trace after convert_real_to_hidden_sys_tenant", K(log_disk_size_in_use), KPC(GCTX.log_block_mgr_),
K(origin_sys_log_disk_size));
}
// don't has any case after this.
TEST_F(ObserverExpandShink, paralle_set)
{
omt::ObTenantNodeBalancer::get_instance().refresh_interval_ = 1000 * 1000 * 1000;
sleep(3);
LOG_INFO("start to test parallel_set");
share::ObTenantSwitchGuard tguard;
ASSERT_EQ(OB_SUCCESS, tguard.switch_to(1));
ObLogService *log_service = MTL(ObLogService*);
palf::PalfOptions opts;
ASSERT_NE(nullptr, log_service);
EXPECT_EQ(OB_SUCCESS, log_service->get_palf_options(opts));
EXPECT_EQ(OB_INVALID_ARGUMENT, log_service->update_log_disk_usage_limit_size(1000));
opts.disk_options_.log_disk_utilization_limit_threshold_ = 10;
opts.disk_options_.log_disk_utilization_threshold_ = 11;
EXPECT_EQ(OB_INVALID_ARGUMENT, log_service->palf_env_->update_options(opts));
{
const int64_t new_log_disk_size = opts.disk_options_.log_disk_usage_limit_size_*50/100;
EXPECT_EQ(OB_SUCCESS, log_service->update_log_disk_usage_limit_size(new_log_disk_size));
sleep(1);
EXPECT_EQ(log_service->palf_env_->palf_env_impl_.disk_options_wrapper_.disk_opts_for_stopping_writing_.log_disk_usage_limit_size_,
new_log_disk_size);
}
{
const int64_t count1 = 10000;
const int64_t count2 = 9000;
std::atomic<int> new_log_disk_size(1*1024*1024*1024);
auto update_size = [&log_service, &new_log_disk_size](int64_t count)
{
for (int64_t i = 0; i < count; i++) {
log_service->update_log_disk_usage_limit_size(++new_log_disk_size);
}
};
std::thread t1(update_size, count1);
std::thread t2(update_size, count2);
t1.join();
t2.join();
EXPECT_EQ(OB_SUCCESS, log_service->get_palf_options(opts));
EXPECT_EQ(opts.disk_options_.log_disk_usage_limit_size_, 1*1024*1024*1024+count1+count2);
}
}
} // end unittest
} // end oceanbase

27
mittest/simple_server/test_transfer_task_operator.cpp
View File

@ -191,16 +191,16 @@ TEST_F(TestTransferTaskOperator, test_basic_func)
// ObTrasnferTaskComment
ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::EMPTY_COMMENT), ""));
ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::WAIT_FOR_MEMBER_LIST), "WAIT FOR MEMBER LIST TO BE SAME"));
ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::TASK_COMPLETED_AS_NO_VALID_PARTITION), "TASK COMPLETED AS NO VALID PARTITION"));
ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::TASK_CANCELED), "TASK CANCELED"));
ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::MAX_COMMENT), "UNKNOW"));
ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::WAIT_FOR_MEMBER_LIST), "Wait for member list to be same"));
ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::TASK_COMPLETED_AS_NO_VALID_PARTITION), "Task completed as no valid partition"));
ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::TASK_CANCELED), "Task canceled"));
ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::MAX_COMMENT), "Unknow"));
ASSERT_TRUE(ObTransferTaskComment::WAIT_FOR_MEMBER_LIST == str_to_transfer_task_comment("WAIT FOR MEMBER LIST TO BE SAME"));
ASSERT_TRUE(ObTransferTaskComment::TASK_COMPLETED_AS_NO_VALID_PARTITION == str_to_transfer_task_comment("TASK COMPLETED AS NO VALID PARTITION"));
ASSERT_TRUE(ObTransferTaskComment::TASK_CANCELED == str_to_transfer_task_comment("TASK CANCELED"));
ASSERT_TRUE(ObTransferTaskComment::WAIT_FOR_MEMBER_LIST == str_to_transfer_task_comment("Wait for member list to be same"));
ASSERT_TRUE(ObTransferTaskComment::TASK_COMPLETED_AS_NO_VALID_PARTITION == str_to_transfer_task_comment("Task completed as no valid partition"));
ASSERT_TRUE(ObTransferTaskComment::TASK_CANCELED == str_to_transfer_task_comment("Task canceled"));
ASSERT_TRUE(ObTransferTaskComment::EMPTY_COMMENT == str_to_transfer_task_comment(""));
ASSERT_TRUE(ObTransferTaskComment::MAX_COMMENT == str_to_transfer_task_comment("UNKNOW"));
ASSERT_TRUE(ObTransferTaskComment::MAX_COMMENT == str_to_transfer_task_comment("Unknow"));
ASSERT_TRUE(ObTransferTaskComment::MAX_COMMENT == str_to_transfer_task_comment("XXXXX"));
}
@ -218,6 +218,11 @@ TEST_F(TestTransferTaskOperator, test_operator)
ASSERT_EQ(OB_ENTRY_NOT_EXIST, ObTransferTaskOperator::get_all_task_status(sql_proxy, tenant_id_, task_status));
ASSERT_TRUE(task_status.empty());
// get_max_task_id_from_history when history is empty
ObTransferTaskID max_task_id;
ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get_max_task_id_from_history(sql_proxy, tenant_id_, max_task_id));
ASSERT_TRUE(!max_task_id.is_valid());
// insert
ObTransferTask other_task;
ObTransferTaskID other_task_id(222);
@ -235,7 +240,7 @@ TEST_F(TestTransferTaskOperator, test_operator)
ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(sql_proxy, tenant_id_, task_id_, false, task));
ASSERT_TRUE(task.get_task_id() == task_id_);
ASSERT_TRUE(task.get_tablet_list().empty());
ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(task.get_comment()), "TASK CANCELED"));
ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(task.get_comment()), "Task canceled"));
LOG_INFO("get from table", K(task));
task.reset();
ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(sql_proxy, tenant_id_, task_id_, true, task));
@ -375,6 +380,10 @@ TEST_F(TestTransferTaskOperator, test_operator)
ASSERT_TRUE(0 == compare(lock_conflict_part_list_str, history_lock_conflict_part_list_str));
ASSERT_EQ(OB_ENTRY_NOT_EXIST, ObTransferTaskOperator::get_history_task(sql_proxy,
tenant_id_, ObTransferTaskID(555), history_task, create_time, finish_time));
max_task_id.reset();
ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get_max_task_id_from_history(sql_proxy, tenant_id_, max_task_id));
ASSERT_TRUE(max_task_id == task_id_);
}
} // namespace share