hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

support multi LogIOWorker

Browse Source
This commit is contained in:
HaHaJeff
2023-06-05 02:57:22 +00:00
committed by ob-robot
parent 305786a690
commit 24bf0c60d8
25 changed files with 941 additions and 611 deletions
Download Patch File Download Diff File Expand all files Collapse all files

148
unittest/logservice/test_palf_throttling.cpp
View File

@ -38,9 +38,13 @@ public:
virtual void SetUp();
virtual void TearDown();
protected:
bool g_need_purging_throttling;
NeedPurgingThrottlingFunc g_need_purging_throttling_func;
};
TestPalfThrottling::TestPalfThrottling() {}
TestPalfThrottling::TestPalfThrottling() : g_need_purging_throttling(false) {
g_need_purging_throttling_func = [this](){ return g_need_purging_throttling; };
}
TestPalfThrottling::~TestPalfThrottling()
{
@ -60,7 +64,8 @@ void TestPalfThrottling::TearDown()
//ObMallocAllocator::get_instance()->recycle_tenant_allocator(1001);
}
TEST(TestPalfThrottling, test_palf_options)
TEST_F(TestPalfThrottling, test_palf_options)
{
char buf[64] = {0};
memset(buf, 0, 64);
@ -76,11 +81,13 @@ TEST(TestPalfThrottling, test_palf_options)
wrapper.disk_opts_for_stopping_writing_.log_disk_usage_limit_size_ = total_disk_size;
wrapper.disk_opts_for_stopping_writing_.log_disk_utilization_threshold_ = 80;
wrapper.disk_opts_for_stopping_writing_.log_disk_utilization_limit_threshold_ = utilization_limit_threshold;
wrapper.disk_opts_for_stopping_writing_.log_writer_parallelism_ = 1;
int64_t unrecyclable_size = 0;
wrapper.set_cur_unrecyclable_log_disk_size(unrecyclable_size);
ASSERT_EQ(false, wrapper.need_throttling());
unrecyclable_size = total_disk_size * 70 /100;
wrapper.set_cur_unrecyclable_log_disk_size(unrecyclable_size);
PALF_LOG(INFO, "test_palf_options trace", K(wrapper));
ASSERT_EQ(true, wrapper.need_throttling());
//test PalfThrottleOptions
PalfThrottleOptions throttling_options;
@ -97,7 +104,7 @@ TEST(TestPalfThrottling, test_palf_options)
ASSERT_EQ(total_disk_size * (utilization_limit_threshold - throttling_percentage)/100, throttling_options.get_available_size_after_limit());
}
TEST(TestPalfThrottling, test_throttling_stat)
TEST_F(TestPalfThrottling, test_throttling_stat)
{
LogThrottlingStat stat;
ASSERT_EQ(false, stat.has_ever_throttled());
@ -125,7 +132,7 @@ TEST(TestPalfThrottling, test_throttling_stat)
ASSERT_EQ(0, stat.max_throttling_interval_);
}
TEST(TestPalfThrottling, test_log_write_throttle)
TEST_F(TestPalfThrottling, test_log_write_throttle)
{
int64_t total_disk_size = 1024 * 1024 * 1024L;
int64_t utilization_limit_threshold = 95;
@ -142,7 +149,7 @@ TEST(TestPalfThrottling, test_log_write_throttle)
palf_env_impl.disk_options_wrapper_.get_throttling_options(throttle_options);
LogWritingThrottle throttle;
ASSERT_EQ(false, throttle.need_throttling_());
ASSERT_EQ(false, throttle.need_throttling_not_guarded_by_lock_(g_need_purging_throttling_func));
ASSERT_EQ(false, throttle.need_writing_throttling_notified());
throttle.notify_need_writing_throttling(true);
ASSERT_EQ(true, throttle.need_writing_throttling_notified());
@ -150,13 +157,12 @@ TEST(TestPalfThrottling, test_log_write_throttle)
throttle.notify_need_writing_throttling(false);
ASSERT_EQ(false, throttle.need_writing_throttling_notified());
ASSERT_EQ(OB_INVALID_ARGUMENT, throttle.after_append_log(-1, 0));
ASSERT_EQ(OB_INVALID_ARGUMENT, throttle.after_append_log(0, -1));
ASSERT_EQ(OB_INVALID_ARGUMENT, throttle.after_append_log(-1));
//test throttling only after notified
PALF_LOG(INFO, "case 1: test no need throttling while notify_need_writing_throttling is false");
throttle.update_throttling_options(&palf_env_impl);
throttle.throttling(1024, &palf_env_impl);
throttle.throttling(1024, g_need_purging_throttling_func, &palf_env_impl);
PalfThrottleOptions invalid_throttle_options;
ASSERT_EQ(invalid_throttle_options, throttle.throttling_options_);
ASSERT_EQ(true, OB_INVALID_TIMESTAMP != throttle.last_update_ts_);
@ -164,7 +170,7 @@ TEST(TestPalfThrottling, test_log_write_throttle)
// test update interval 500ms
PALF_LOG(INFO, "case 2: test update interval");
throttle.throttling(1024, &palf_env_impl);
throttle.throttling(1024, g_need_purging_throttling_func, &palf_env_impl);
ASSERT_EQ(invalid_throttle_options, throttle.throttling_options_);
ASSERT_EQ(true, OB_INVALID_TIMESTAMP != throttle.last_update_ts_);
ASSERT_EQ(false, throttle.need_writing_throttling_notified_);
@ -174,8 +180,8 @@ TEST(TestPalfThrottling, test_log_write_throttle)
usleep(LogWritingThrottle::UPDATE_INTERVAL_US);
throttle.update_throttling_options(&palf_env_impl);
throttle.notify_need_writing_throttling(true);
throttle.throttling(1024, &palf_env_impl);
ASSERT_EQ(false, throttle.need_throttling_());
throttle.throttling(1024, g_need_purging_throttling_func, &palf_env_impl);
ASSERT_EQ(false, throttle.need_throttling_not_guarded_by_lock_(g_need_purging_throttling_func));
ASSERT_EQ(false, throttle.stat_.has_ever_throttled());
@ -185,9 +191,9 @@ TEST(TestPalfThrottling, test_log_write_throttle)
palf_env_impl.disk_options_wrapper_.disk_opts_for_stopping_writing_.log_disk_throttling_percentage_ = 100;
usleep(LogWritingThrottle::UPDATE_INTERVAL_US);
throttle.update_throttling_options(&palf_env_impl);
throttle.throttling(1024, &palf_env_impl);
throttle.throttling(1024, g_need_purging_throttling_func, &palf_env_impl);
palf_env_impl.disk_options_wrapper_.get_throttling_options(throttle_options);
ASSERT_EQ(false, throttle.need_throttling_());
ASSERT_EQ(false, throttle.need_throttling_not_guarded_by_lock_(g_need_purging_throttling_func));
ASSERT_EQ(false, throttle.stat_.has_ever_throttled());
//test need throttling after update
@ -197,109 +203,96 @@ TEST(TestPalfThrottling, test_log_write_throttle)
palf_env_impl.disk_options_wrapper_.set_cur_unrecyclable_log_disk_size(unrecyclable_size);
usleep(LogWritingThrottle::UPDATE_INTERVAL_US);
throttle.update_throttling_options(&palf_env_impl);
throttle.throttling(1024, &palf_env_impl);
throttle.throttling(1024, g_need_purging_throttling_func, &palf_env_impl);
palf_env_impl.disk_options_wrapper_.get_throttling_options(throttle_options);
ASSERT_EQ(throttle_options, throttle.throttling_options_);
ASSERT_EQ(true, throttle.need_throttling_());
ASSERT_EQ(true, throttle.need_throttling_not_guarded_by_lock_(g_need_purging_throttling_func));
ASSERT_EQ(true, throttle.stat_.has_ever_throttled());
ASSERT_EQ(1024, throttle.stat_.total_throttling_size_);
ASSERT_EQ(1, throttle.stat_.total_throttling_task_cnt_);
ASSERT_EQ(0, throttle.stat_.total_skipped_task_cnt_);
ASSERT_EQ(0, throttle.stat_.total_skipped_size_);
throttle.after_append_log(1024, 0);
throttle.after_append_log(1024);
ASSERT_EQ(1024, throttle.appended_log_size_cur_round_);
ASSERT_EQ(0, throttle.submitted_seq_);
ASSERT_EQ(0, throttle.handled_seq_);
//test no need throttling with flush meta task in queue
//.1 flush log task
g_need_purging_throttling = true;
PALF_LOG(INFO, "case 5: test no need throttling while flush meta task ", K(throttle));
throttle.inc_and_fetch_submitted_seq();
throttle.inc_and_fetch_submitted_seq();
throttle.inc_and_fetch_submitted_seq();
ASSERT_EQ(3, throttle.submitted_seq_);
throttle.throttling(1024, &palf_env_impl);
throttle.throttling(1024, g_need_purging_throttling_func, &palf_env_impl);
ASSERT_EQ(throttle_options, throttle.throttling_options_);
ASSERT_EQ(false, throttle.need_throttling_());
// meta task need purging throttling
ASSERT_EQ(false, throttle.need_throttling_not_guarded_by_lock_(g_need_purging_throttling_func));
ASSERT_EQ(1024, throttle.stat_.total_throttling_size_);
ASSERT_EQ(1, throttle.stat_.total_throttling_task_cnt_);
ASSERT_EQ(1, throttle.stat_.total_skipped_task_cnt_);
ASSERT_EQ(1024, throttle.stat_.total_skipped_size_);
throttle.after_append_log(1024, 0);
throttle.after_append_log(1024);
ASSERT_EQ(2048, throttle.appended_log_size_cur_round_);
ASSERT_EQ(3, throttle.submitted_seq_);
ASSERT_EQ(0, throttle.handled_seq_);
//.2 flush meta task
PALF_LOG(INFO, "case 6: test no need throttling and flush meta task ", K(throttle));
throttle.throttling(1024, &palf_env_impl);
throttle.throttling(1024, g_need_purging_throttling_func, &palf_env_impl);
ASSERT_EQ(throttle_options, throttle.throttling_options_);
ASSERT_EQ(true, throttle.need_throttling_with_options_());
ASSERT_EQ(false, throttle.need_throttling_());
ASSERT_EQ(true, throttle.need_throttling_with_options_not_guarded_by_lock_());
ASSERT_EQ(false, throttle.need_throttling_not_guarded_by_lock_(g_need_purging_throttling_func));
ASSERT_EQ(1024, throttle.stat_.total_throttling_size_);
ASSERT_EQ(1, throttle.stat_.total_throttling_task_cnt_);
ASSERT_EQ(2, throttle.stat_.total_skipped_task_cnt_);
ASSERT_EQ(2048, throttle.stat_.total_skipped_size_);
throttle.after_append_log(1024, 1);
throttle.after_append_log(1024);
ASSERT_EQ(3072, throttle.appended_log_size_cur_round_);
ASSERT_EQ(3, throttle.submitted_seq_);
ASSERT_EQ(1, throttle.handled_seq_);
throttle.throttling(1024, &palf_env_impl);
throttle.throttling(1024, g_need_purging_throttling_func, &palf_env_impl);
ASSERT_EQ(throttle_options, throttle.throttling_options_);
ASSERT_EQ(true, throttle.need_throttling_with_options_());
ASSERT_EQ(false, throttle.need_throttling_());
ASSERT_EQ(true, throttle.need_throttling_with_options_not_guarded_by_lock_());
ASSERT_EQ(false, throttle.need_throttling_not_guarded_by_lock_(g_need_purging_throttling_func));
ASSERT_EQ(1024, throttle.stat_.total_throttling_size_);
ASSERT_EQ(1, throttle.stat_.total_throttling_task_cnt_);
ASSERT_EQ(3, throttle.stat_.total_skipped_task_cnt_);
ASSERT_EQ(3072, throttle.stat_.total_skipped_size_);
throttle.after_append_log(1024, 3);
throttle.after_append_log(1024);
ASSERT_EQ(4096, throttle.appended_log_size_cur_round_);
ASSERT_EQ(3, throttle.submitted_seq_);
ASSERT_EQ(3, throttle.handled_seq_);
PALF_LOG(INFO, "case 7: need throttling after all flush meta task handled", K(throttle));
throttle.throttling(1024, &palf_env_impl);
g_need_purging_throttling = false;
throttle.throttling(1024, g_need_purging_throttling_func, &palf_env_impl);
ASSERT_EQ(throttle_options, throttle.throttling_options_);
ASSERT_EQ(true, throttle.decay_factor_ > 0.0);
ASSERT_EQ(true, throttle.need_throttling_());
ASSERT_EQ(true, throttle.need_throttling_not_guarded_by_lock_(g_need_purging_throttling_func));
ASSERT_EQ(2048, throttle.stat_.total_throttling_size_);
ASSERT_EQ(2, throttle.stat_.total_throttling_task_cnt_);
ASSERT_EQ(3, throttle.stat_.total_skipped_task_cnt_);
ASSERT_EQ(3072, throttle.stat_.total_skipped_size_);
throttle.after_append_log(1024, 3);
throttle.after_append_log(1024);
ASSERT_EQ(5120, throttle.appended_log_size_cur_round_);
ASSERT_EQ(3, throttle.submitted_seq_);
ASSERT_EQ(3, throttle.handled_seq_);
//test notify_need_writing_throttling(false) changed
PALF_LOG(INFO, "case 8: no need to throttle after notify_need_throttling(false)", K(throttle));
usleep(LogWritingThrottle::UPDATE_INTERVAL_US);
throttle.notify_need_writing_throttling(false);
throttle.update_throttling_options(&palf_env_impl);
throttle.throttling(1024, &palf_env_impl);
throttle.throttling(1024, g_need_purging_throttling_func, &palf_env_impl);
ASSERT_EQ(invalid_throttle_options, throttle.throttling_options_);
ASSERT_EQ(false, throttle.need_throttling_with_options_());
ASSERT_EQ(false, throttle.need_throttling_());
ASSERT_EQ(false, throttle.need_throttling_with_options_not_guarded_by_lock_());
ASSERT_EQ(false, throttle.need_throttling_not_guarded_by_lock_(g_need_purging_throttling_func));
ASSERT_EQ(2048, throttle.stat_.total_throttling_size_);
ASSERT_EQ(2, throttle.stat_.total_throttling_task_cnt_);
ASSERT_EQ(3, throttle.stat_.total_skipped_task_cnt_);
ASSERT_EQ(3072, throttle.stat_.total_skipped_size_);
ASSERT_EQ(true, OB_INVALID_TIMESTAMP != throttle.stat_.stop_ts_);
throttle.after_append_log(1024, 0);
throttle.after_append_log(1024);
ASSERT_EQ(1024, throttle.appended_log_size_cur_round_);
ASSERT_EQ(0, throttle.decay_factor_);
ASSERT_EQ(3, throttle.submitted_seq_);
ASSERT_EQ(3, throttle.handled_seq_);
//test need write throttling again
PALF_LOG(INFO, "case 9: need to throttle after notify_need_throttling(true)", K(throttle));
usleep(LogWritingThrottle::UPDATE_INTERVAL_US);
throttle.notify_need_writing_throttling(true);
throttle.update_throttling_options(&palf_env_impl);
throttle.throttling(1024, &palf_env_impl);
ASSERT_EQ(true, throttle.need_throttling_with_options_());
ASSERT_EQ(true, throttle.need_throttling_());
throttle.throttling(1024, g_need_purging_throttling_func, &palf_env_impl);
ASSERT_EQ(true, throttle.need_throttling_with_options_not_guarded_by_lock_());
ASSERT_EQ(true, throttle.need_throttling_not_guarded_by_lock_(g_need_purging_throttling_func));
ASSERT_EQ(throttle_options, throttle.throttling_options_);
ASSERT_EQ(true, throttle.decay_factor_ > 0.0);
ASSERT_EQ(1024, throttle.stat_.total_throttling_size_);
@ -308,10 +301,8 @@ TEST(TestPalfThrottling, test_log_write_throttle)
ASSERT_EQ(0, throttle.stat_.total_skipped_size_);
ASSERT_EQ(true, throttle.stat_.total_throttling_interval_ > 0);
ASSERT_EQ(true, OB_INVALID_TIMESTAMP == throttle.stat_.stop_ts_);
throttle.after_append_log(1024, 0);
throttle.after_append_log(1024);
ASSERT_EQ(1024, throttle.appended_log_size_cur_round_);
ASSERT_EQ(3, throttle.submitted_seq_);
ASSERT_EQ(3, throttle.handled_seq_);
double old_decay_factor = throttle.decay_factor_;
//
@ -320,9 +311,9 @@ TEST(TestPalfThrottling, test_log_write_throttle)
palf_env_impl.disk_options_wrapper_.disk_opts_for_stopping_writing_.log_disk_throttling_percentage_ = 55;
palf_env_impl.get_throttling_options(throttle_options);
throttle.update_throttling_options(&palf_env_impl);
throttle.throttling(1024, &palf_env_impl);
ASSERT_EQ(true, throttle.need_throttling_with_options_());
ASSERT_EQ(true, throttle.need_throttling_());
throttle.throttling(1024, g_need_purging_throttling_func, &palf_env_impl);
ASSERT_EQ(true, throttle.need_throttling_with_options_not_guarded_by_lock_());
ASSERT_EQ(true, throttle.need_throttling_not_guarded_by_lock_(g_need_purging_throttling_func));
ASSERT_EQ(throttle_options, throttle.throttling_options_);
ASSERT_EQ(true, throttle.decay_factor_ > 0.0);
ASSERT_EQ(true, throttle.decay_factor_ != old_decay_factor);
@ -331,10 +322,8 @@ TEST(TestPalfThrottling, test_log_write_throttle)
ASSERT_EQ(0, throttle.stat_.total_skipped_task_cnt_);
ASSERT_EQ(0, throttle.stat_.total_skipped_size_);
ASSERT_EQ(true, OB_INVALID_TIMESTAMP == throttle.stat_.stop_ts_);
throttle.after_append_log(1024, 0);
throttle.after_append_log(1024);
ASSERT_EQ(2048, throttle.appended_log_size_cur_round_);
ASSERT_EQ(3, throttle.submitted_seq_);
ASSERT_EQ(3, throttle.handled_seq_);
//test reset appended_log_size_cur_round_
PALF_LOG(INFO, "case 11: test reset appended_log_size_cur_round_ after unrecyclable_size changes", K(throttle));
@ -343,9 +332,9 @@ TEST(TestPalfThrottling, test_log_write_throttle)
unrecyclable_size = total_disk_size * 65/100;
palf_env_impl.disk_options_wrapper_.set_cur_unrecyclable_log_disk_size(unrecyclable_size);
throttle.update_throttling_options(&palf_env_impl);
throttle.throttling(1024, &palf_env_impl);
ASSERT_EQ(true, throttle.need_throttling_with_options_());
ASSERT_EQ(true, throttle.need_throttling_());
throttle.throttling(1024, g_need_purging_throttling_func, &palf_env_impl);
ASSERT_EQ(true, throttle.need_throttling_with_options_not_guarded_by_lock_());
ASSERT_EQ(true, throttle.need_throttling_not_guarded_by_lock_(g_need_purging_throttling_func));
palf_env_impl.get_throttling_options(throttle_options);
ASSERT_EQ(throttle_options, throttle.throttling_options_);
ASSERT_EQ(true, throttle.decay_factor_ > 0.0);
@ -356,12 +345,9 @@ TEST(TestPalfThrottling, test_log_write_throttle)
ASSERT_EQ(0, throttle.stat_.total_skipped_size_);
ASSERT_EQ(true, OB_INVALID_TIMESTAMP != throttle.stat_.start_ts_);
ASSERT_EQ(true, OB_INVALID_TIMESTAMP == throttle.stat_.stop_ts_);
throttle.after_append_log(1024, 0);
throttle.after_append_log(1024);
ASSERT_EQ(1024, throttle.appended_log_size_cur_round_);
ASSERT_EQ(3, throttle.submitted_seq_);
ASSERT_EQ(3, throttle.handled_seq_);
ASSERT_EQ(OB_ERR_UNEXPECTED, throttle.after_append_log(0, 4));
ASSERT_EQ(OB_ERR_UNEXPECTED, throttle.after_append_log(0, 2));
ASSERT_EQ(OB_SUCCESS, throttle.after_append_log(0));
//test stop write throttling when trigger percentage changed
PALF_LOG(INFO, "case 12: test stop write throttling when trigger percentage changed", K(throttle));
@ -369,9 +355,9 @@ TEST(TestPalfThrottling, test_log_write_throttle)
palf_env_impl.disk_options_wrapper_.disk_opts_for_stopping_writing_.log_disk_throttling_percentage_ = 80;
palf_env_impl.get_throttling_options(throttle_options);
throttle.update_throttling_options(&palf_env_impl);
throttle.throttling(1024, &palf_env_impl);
ASSERT_EQ(false, throttle.need_throttling_with_options_());
ASSERT_EQ(false, throttle.need_throttling_());
throttle.throttling(1024, g_need_purging_throttling_func, &palf_env_impl);
ASSERT_EQ(false, throttle.need_throttling_with_options_not_guarded_by_lock_());
ASSERT_EQ(false, throttle.need_throttling_not_guarded_by_lock_(g_need_purging_throttling_func));
ASSERT_EQ(invalid_throttle_options, throttle.throttling_options_);
ASSERT_EQ(0, throttle.decay_factor_);
ASSERT_EQ(3072, throttle.stat_.total_throttling_size_);
@ -380,17 +366,15 @@ TEST(TestPalfThrottling, test_log_write_throttle)
ASSERT_EQ(0, throttle.stat_.total_skipped_size_);
ASSERT_EQ(true, OB_INVALID_TIMESTAMP != throttle.stat_.stop_ts_);
ASSERT_EQ(0, throttle.appended_log_size_cur_round_);
ASSERT_EQ(3, throttle.submitted_seq_);
ASSERT_EQ(3, throttle.handled_seq_);
PALF_LOG(INFO, "case 12: test stop writing throttling when unrecyclable size fallbacks", K(throttle));
usleep(LogWritingThrottle::UPDATE_INTERVAL_US);
palf_env_impl.disk_options_wrapper_.disk_opts_for_stopping_writing_.log_disk_throttling_percentage_ = 60;
throttle.notify_need_writing_throttling(true);
throttle.update_throttling_options(&palf_env_impl);
throttle.throttling(1024, &palf_env_impl);
ASSERT_EQ(true, throttle.need_throttling_with_options_());
ASSERT_EQ(true, throttle.need_throttling_());
throttle.throttling(1024, g_need_purging_throttling_func, &palf_env_impl);
ASSERT_EQ(true, throttle.need_throttling_with_options_not_guarded_by_lock_());
ASSERT_EQ(true, throttle.need_throttling_not_guarded_by_lock_(g_need_purging_throttling_func));
palf_env_impl.get_throttling_options(throttle_options);
ASSERT_EQ(throttle_options, throttle.throttling_options_);
ASSERT_EQ(true, throttle.decay_factor_ > 0.0);
@ -400,17 +384,15 @@ TEST(TestPalfThrottling, test_log_write_throttle)
ASSERT_EQ(0, throttle.stat_.total_skipped_size_);
ASSERT_EQ(true, throttle.stat_.total_throttling_interval_ > 0);
ASSERT_EQ(true, OB_INVALID_TIMESTAMP == throttle.stat_.stop_ts_);
throttle.after_append_log(1024, 0);
throttle.after_append_log(1024);
ASSERT_EQ(1024, throttle.appended_log_size_cur_round_);
ASSERT_EQ(3, throttle.submitted_seq_);
ASSERT_EQ(3, throttle.handled_seq_);
usleep(LogWritingThrottle::UPDATE_INTERVAL_US);
unrecyclable_size = total_disk_size * 45/100;
palf_env_impl.disk_options_wrapper_.set_cur_unrecyclable_log_disk_size(unrecyclable_size);
throttle.update_throttling_options(&palf_env_impl);
throttle.throttling(1024, &palf_env_impl);
ASSERT_EQ(false, throttle.need_throttling_with_options_());
throttle.throttling(1024, g_need_purging_throttling_func, &palf_env_impl);
ASSERT_EQ(false, throttle.need_throttling_with_options_not_guarded_by_lock_());
}