hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Avoid stopping writing after restart due to log disk is shrinking.

Browse Source
This commit is contained in:
HaHaJeff
2023-07-13 09:48:23 +00:00
committed by ob-robot
parent b9780d0b60
commit 6810df627b
24 changed files with 1075 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files

151
mittest/simple_server/test_observer_expand_shrink.cpp
View File

@ -20,6 +20,10 @@
#include "logservice/ob_log_service.h"
#include "observer/ob_server_utils.h"
#include "env/ob_simple_server_restart_helper.h"
#include "observer/omt/ob_tenant.h"
#include "share/unit/ob_unit_config.h"
#undef protected
#undef private
const char *TEST_FILE_NAME = "test_observer_expand_shrink";
const char *BORN_CASE_NAME= "ObserverExpandShink";
@ -56,7 +60,7 @@ TEST_F(ObserverExpandShink, basic_func)
int64_t origin_server_in_use_size, origin_server_log_total_size;
EXPECT_EQ(OB_SUCCESS, GCTX.log_block_mgr_->get_disk_usage(origin_server_in_use_size, origin_server_log_total_size));
GCONF.log_disk_size = GCTX.log_block_mgr_->lower_align_(2 * origin_server_log_total_size);
sleep(6);
sleep(11);
int64_t new_server_in_use_size, new_server_log_total_size;
EXPECT_EQ(OB_SUCCESS, GCTX.log_block_mgr_->get_disk_usage(new_server_in_use_size, new_server_log_total_size));
EXPECT_EQ(new_server_log_total_size, 2 * origin_server_log_total_size);
@ -102,6 +106,96 @@ TEST_F(ObserverExpandShink, basic_func)
log_disk_percentage));
}
template<typename ... Args>
std::string string_format( const std::string& format, Args ... args )
{
int size_s = std::snprintf( nullptr, 0, format.c_str(), args ... ) + 1; // Extra space for '\0'
if( size_s <= 0 ){ throw std::runtime_error( "Error during formatting." ); }
auto size = static_cast<size_t>( size_s );
std::unique_ptr<char[]> buf( new char[ size ] );
std::snprintf( buf.get(), size, format.c_str(), args ... );
return std::string( buf.get(), buf.get() + size - 1 ); // We don't want the '\0' inside
}
TEST_F(ObserverExpandShink, resize_tenant_log_disk)
{
omt::ObTenantNodeBalancer::get_instance().refresh_interval_ = 1 * 1000 * 1000;
sleep(10);
GCONF.log_disk_size = 20 * 1024 * 1024 * 1024ul;
bool tenant_exist = true;
int ret = OB_SUCCESS;
delete_tenant("runlin");
while (true == tenant_exist && OB_SUCC(ret)) {
if (OB_FAIL(check_tenant_exist(tenant_exist, "runlin"))) {
SERVER_LOG(WARN, "check_tenant_exist failed", K(ret));
}
}
// 保证log_disk_size变为10G生效
sleep(3);
int64_t log_disk_origin_assigned = GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_;
bool bool_ret = true;
EXPECT_EQ(bool_ret, true);
LOG_INFO("runlin trace, before create one default tenant", KPC(GCTX.log_block_mgr_), K(log_disk_origin_assigned));
// 每个租户的日志盘大小为2G（默认值）
EXPECT_EQ(OB_SUCCESS, create_tenant("runlin1"));
LOG_INFO("runlin trace, after create one default tenant", KPC(GCTX.log_block_mgr_), K(log_disk_origin_assigned));
EXPECT_EQ(OB_SUCCESS, create_tenant("runlin2"));
EXPECT_EQ(GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_,
log_disk_origin_assigned + 4*1024*1024*1024ul);
LOG_INFO("runlin trace, after create two default tenant", KPC(GCTX.log_block_mgr_), K(log_disk_origin_assigned));
// 修改租户规格
int64_t affected_rows = 0;
std::string sql_str = "ALTER RESOURCE UNIT %s%s LOG_DISK_SIZE='%s'";
{
std::string alter_resource_failed_sql = string_format(sql_str, UNIT_BASE, "runlin1", "100G");
EXPECT_EQ(OB_MACHINE_RESOURCE_NOT_ENOUGH, exec_write_sql_sys(alter_resource_failed_sql.c_str(), affected_rows));
LOG_INFO("runlin trace, alter resource failed", KPC(GCTX.log_block_mgr_));
}
{
std::string alter_resource_failed_sql = string_format(sql_str, UNIT_BASE, "runlin1", "1G");
EXPECT_EQ(OB_RESOURCE_UNIT_VALUE_BELOW_LIMIT, exec_write_sql_sys(alter_resource_failed_sql.c_str(), affected_rows));
LOG_INFO("runlin trace, alter resource below limit", KPC(GCTX.log_block_mgr_));
}
{
// 扩容验证
std::string alter_resource_runlin1 = string_format(sql_str, UNIT_BASE, "runlin1", "6G");
std::string alter_resource_runlin2 = string_format(sql_str, UNIT_BASE, "runlin2", "6G");
EXPECT_EQ(OB_SUCCESS, exec_write_sql_sys(alter_resource_runlin1.c_str(), affected_rows));
EXPECT_EQ(OB_SUCCESS, exec_write_sql_sys(alter_resource_runlin2.c_str(), affected_rows));
sleep(3);
// 扩容操作直接执行成功
EXPECT_EQ(GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_,
log_disk_origin_assigned + 12*1024*1024*1024ul);
LOG_INFO("runlin trace, alter resource to 6G success", KPC(GCTX.log_block_mgr_));
}
{
// 缩容验证
// 暂停TenantNodeBalancer的运行
omt::ObTenantNodeBalancer::get_instance().refresh_interval_ = 10 * 1000 * 1000;
int64_t start_ts = ObTimeUtility::current_time();
sleep(2);
{
std::string alter_resource_runlin1 = string_format(sql_str, UNIT_BASE, "runlin1", "2G");
std::string alter_resource_runlin2 = string_format(sql_str, UNIT_BASE, "runlin2", "2G");
EXPECT_EQ(OB_SUCCESS, exec_write_sql_sys(alter_resource_runlin1.c_str(), affected_rows));
EXPECT_EQ(OB_SUCCESS, exec_write_sql_sys(alter_resource_runlin2.c_str(), affected_rows));
}
const int64_t cost_ts = ObTimeUtility::current_time() - start_ts;
// 假设sleep+执行sql的时间小于4s
EXPECT_LE(cost_ts, 4*1000*1000);
// 此时TenantNodeBalancer线程还未开始运行，ObServerLogBlockMgr的min_log_disk_size_for_all_tenants_不会发生变化
EXPECT_EQ(GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_,
log_disk_origin_assigned + 12*1024*1024*1024ul);
omt::ObTenantNodeBalancer::get_instance().refresh_interval_ = 1 * 1000 * 1000;
sleep(11);
LOG_INFO("runlin trace, alter resource to 2G success", KPC(GCTX.log_block_mgr_));
EXPECT_EQ(GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_,
log_disk_origin_assigned + 4*1024*1024*1024ul);
}
EXPECT_EQ(OB_SUCCESS, delete_tenant("runlin1"));
EXPECT_EQ(OB_SUCCESS, delete_tenant("runlin2"));
}
//class ObserverExpandShinkRestart: public ObSimpleClusterTestBase
//{
//public:
@ -114,56 +208,15 @@ TEST_F(ObserverExpandShink, basic_func)
// SERVER_LOG(INFO, "create_tenant_after_restart trace", KPC(GCTX.log_block_mgr_));
//}
TEST_F(ObserverExpandShink, direct_set_observer)
{
GCONF.log_disk_size = 1024 * 1024 * 1024ul * 3;
int64_t log_disk_size = 0;
int64_t log_disk_percentage = 0;
EXPECT_EQ(OB_SUCCESS, observer::ObServerUtils::get_log_disk_info_in_config(
log_disk_size,
log_disk_percentage));
GCONF.log_disk_size = 10000000 * share::ObUnitResource::UNIT_MIN_LOG_DISK_SIZE;
EXPECT_EQ(OB_SERVER_OUTOF_DISK_SPACE, observer::ObServerUtils::get_log_disk_info_in_config(
log_disk_size,
log_disk_percentage));
GCONF.log_disk_size = 3 * share::ObUnitResource::UNIT_MIN_LOG_DISK_SIZE;
EXPECT_EQ(false, GCTX.log_block_mgr_->check_space_is_enough_(share::ObUnitResource::UNIT_MIN_LOG_DISK_SIZE - 1));
{
GCONF.log_disk_size = 3 * share::ObUnitResource::UNIT_MIN_LOG_DISK_SIZE;
sleep(6);
}
EXPECT_EQ(OB_SUCCESS, create_tenant("tt2"));
EXPECT_EQ(false, GCTX.log_block_mgr_->check_space_is_enough_(2*share::ObUnitResource::UNIT_MIN_LOG_DISK_SIZE-1));
EXPECT_EQ(false, GCTX.log_block_mgr_->check_space_is_enough_(2*share::ObUnitResource::UNIT_MIN_LOG_DISK_SIZE));
int64_t affected_rows = 0;
std::string succ_sql_str = "ALTER RESOURCE UNIT sys_unit_config LOG_DISK_SIZE='2G'";
EXPECT_EQ(OB_SUCCESS, exec_write_sql_sys(succ_sql_str.c_str(), affected_rows));
sleep(2);
// tenant_node_balancer 1 s 运行一次
EXPECT_EQ(true, GCTX.log_block_mgr_->check_space_is_enough_(2*share::ObUnitResource::UNIT_MIN_LOG_DISK_SIZE));
bool tenant_exist = false;
EXPECT_EQ(OB_SUCCESS, check_tenant_exist(tenant_exist, "tt2"));
EXPECT_EQ(true, tenant_exist);
EXPECT_EQ(OB_SUCCESS, delete_tenant("tt2"));
int ret = OB_SUCCESS;
while (true == tenant_exist && OB_SUCC(ret)) {
if (OB_FAIL(check_tenant_exist(tenant_exist, "tt2"))) {
SERVER_LOG(WARN, "check_tenant_exist failed", K(ret));
}
}
}
TEST_F(ObserverExpandShink, test_hidden_sys_tenant)
{
omt::ObMultiTenant *omt = GCTX.omt_;
bool remove_tenant_succ = false;
int64_t log_disk_size_in_use = GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_;
EXPECT_EQ(log_disk_size_in_use, GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_);
share::TenantUnits units;
EXPECT_EQ(OB_SUCCESS, omt->get_tenant_units(units, false));
EXPECT_EQ(false, units.empty());
bool has_sys_tenant = false;
int64_t origin_sys_log_disk_size = 0;
int64_t hidden_sys_log_disk_size = 0;
for (int i = 0; i < units.count(); i++) {
@ -190,15 +243,20 @@ TEST_F(ObserverExpandShink, test_hidden_sys_tenant)
GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_);
log_disk_size_in_use = GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_;
EXPECT_EQ(OB_SUCCESS, omt->convert_hidden_to_real_sys_tenant(sys_unit_config));
has_sys_tenant = true;
EXPECT_EQ(log_disk_size_in_use-hidden_sys_log_disk_size+sys_unit_config.config_.log_disk_size(),
GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_);
log_disk_size_in_use = GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_;
CLOG_LOG(INFO, "runlin trace after convert_hidden_to_real_sys_tenant", K(log_disk_size_in_use), KPC(GCTX.log_block_mgr_),
K(origin_sys_log_disk_size));
int64_t new_sys_log_disk_size = 2 *1024*1024*1024l +512*1024*1024;
EXPECT_EQ(OB_SUCCESS, omt->update_tenant_log_disk_size(OB_SYS_TENANT_ID, new_sys_log_disk_size));
EXPECT_EQ(true, has_sys_tenant);
int64_t new_sys_log_disk_size = sys_unit_config.config_.log_disk_size()+512*1024*1024;
omt::ObTenant *tenant = nullptr;
EXPECT_EQ(OB_SUCCESS, omt->get_tenant(OB_SYS_TENANT_ID, tenant));
share::ObUnitInfoGetter::ObTenantConfig unit_config = tenant->get_unit();
unit_config.config_.resource_.log_disk_size_ = new_sys_log_disk_size;
// 扩容直接成功
EXPECT_EQ(OB_SUCCESS, omt->update_tenant_unit(unit_config));
unit_config = tenant->get_unit();
EXPECT_EQ(unit_config.config_.log_disk_size(), new_sys_log_disk_size);
EXPECT_EQ(log_disk_size_in_use-sys_unit_config.config_.log_disk_size()+new_sys_log_disk_size,
GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_);
CLOG_LOG(INFO, "runlin trace after convert_real_to_hidden_sys_tenant", K(log_disk_size_in_use), KPC(GCTX.log_block_mgr_),
@ -206,6 +264,7 @@ TEST_F(ObserverExpandShink, test_hidden_sys_tenant)
}
// don't has any case after this.
TEST_F(ObserverExpandShink, paralle_set)
{
omt::ObTenantNodeBalancer::get_instance().refresh_interval_ = 1000 * 1000 * 1000;