hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

limit the print frequency for each ERROR log

Browse Source
This commit is contained in:
tushicheng
2024-01-09 11:13:23 +00:00
committed by ob-robot
parent 3f61aea79d
commit 4b58152040
4 changed files with 28 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
deps/oblib/src/lib/oblog/ob_log.cpp vendored
View File

@ -56,6 +56,7 @@ _RLOCAL(lib::ObRateLimiter*, ObLogger::tl_log_limiter_);
static const int64_t limiter_initial = 1000;
static const int64_t limiter_thereafter = 100;
ObSyslogSampleRateLimiter ObLogger::per_log_limiters_[];
ObSyslogSampleRateLimiter ObLogger::per_error_log_limiters_[];
_RLOCAL(int32_t, ObLogger::tl_type_);
_RLOCAL(uint64_t, ObLogger::curr_logging_seq_);
_RLOCAL(uint64_t, ObLogger::last_logging_seq_);
@ -1423,6 +1424,9 @@ int ObLogger::init(const ObBaseLogWriterCfg &log_cfg,
for (int i = 0; i < ARRAYSIZEOF(per_log_limiters_); i++) {
new (&per_log_limiters_[i])ObSyslogSampleRateLimiter(limiter_initial, limiter_thereafter);
}
for (int i = 0; i < ARRAYSIZEOF(per_error_log_limiters_); i++) {
new (&per_error_log_limiters_[i])ObSyslogSampleRateLimiter(limiter_initial, limiter_thereafter);
}
const int64_t limit = ObBaseLogWriterCfg::DEFAULT_MAX_BUFFER_ITEM_CNT * OB_MALLOC_BIG_BLOCK_SIZE / 8; // 256M
log_mem_limiter_ = new (buf) ObBlockAllocMgr(limit);
allocator_ = new (log_mem_limiter_ + 1) ObVSliceAlloc();
@ -1676,10 +1680,13 @@ int ObLogger::check_tl_log_limiter(const uint64_t location_hash_val,
if (!allow) { limiter_info = " REACH SYSLOG RATE LIMIT [bandwidth]"; }
}
if (allow) {
int64_t idx0 = (location_hash_val >> 32) % N_LIMITER;
int64_t idx1 = ((location_hash_val << 32) >> 32) % N_LIMITER;
bool r0 = OB_SUCCESS == per_log_limiters_[idx0].try_acquire(1, level, errcode);
bool r1 = OB_SUCCESS == per_log_limiters_[idx1].try_acquire(1, level, errcode);
int n_limiter = (OB_LOG_LEVEL_ERROR == level) ? N_ERROR_LIMITER : N_LIMITER;
ObSyslogSampleRateLimiter *log_sample_rate_limiters =
(OB_LOG_LEVEL_ERROR == level) ? per_error_log_limiters_ : per_log_limiters_;
int64_t idx0 = (location_hash_val >> 32) % n_limiter;
int64_t idx1 = ((location_hash_val << 32) >> 32) % n_limiter;
bool r0 = OB_SUCCESS == log_sample_rate_limiters[idx0].try_acquire(1, level, errcode);
bool r1 = OB_SUCCESS == log_sample_rate_limiters[idx1].try_acquire(1, level, errcode);
allow = r0 && r1;
if (!allow) { limiter_info = " REACH SYSLOG RATE LIMIT [frequency]"; }
}

2
deps/oblib/src/lib/oblog/ob_log.h vendored
View File

@ -753,7 +753,9 @@ private:
static ::oceanbase::lib::ObRateLimiter *default_log_limiter_;
RLOCAL_STATIC(lib::ObRateLimiter*, tl_log_limiter_);
static constexpr int N_LIMITER = 4096;
static constexpr int N_ERROR_LIMITER = 1024;
static ObSyslogSampleRateLimiter per_log_limiters_[N_LIMITER];
static ObSyslogSampleRateLimiter per_error_log_limiters_[N_ERROR_LIMITER];
RLOCAL_STATIC(int32_t, tl_type_);
//used for stat logging time and log dropped
RLOCAL_STATIC(uint64_t, curr_logging_seq_);

15
deps/oblib/src/lib/oblog/ob_syslog_rate_limiter.h vendored
View File

@ -53,10 +53,6 @@ public:
if (!in_acquire) {
if (0 == rate_) {
ret = common::OB_EAGAIN;
} else if (OB_LOG_LEVEL_ERROR == log_level
|| OB_LOG_LEVEL_DBA_WARN == log_level
|| OB_LOG_LEVEL_DBA_ERROR == log_level) {
// allow
} else {
in_acquire = 1;
ret = do_acquire(permits, log_level, errcode);
@ -117,9 +113,16 @@ public:
void reset_force_allows() override { };
int do_acquire(int64_t permits, int log_level, int errcode) override
{
UNUSED(log_level);
UNUSED(errcode);
return impl_.try_acquire(permits);
int ret = OB_SUCCESS;
if (OB_LOG_LEVEL_ERROR == log_level
|| OB_LOG_LEVEL_DBA_WARN == log_level
|| OB_LOG_LEVEL_DBA_ERROR == log_level) {
// allow
} else {
ret = impl_.try_acquire(permits);
}
return ret;
}
private:

6
deps/oblib/unittest/lib/utility/test_sample_rate_limiter.cpp vendored
View File

@ -34,6 +34,12 @@ TEST(TestSampleRateLimiter, Basic)
ASSERT_EQ(OB_SUCCESS, rl.try_acquire(1, OB_LOG_LEVEL_INFO));
ASSERT_EQ(OB_EAGAIN, rl.try_acquire(2, OB_LOG_LEVEL_INFO));
ASSERT_EQ(OB_SUCCESS, rl.try_acquire(1, OB_LOG_LEVEL_INFO));
sleep(1);
ASSERT_EQ(OB_SUCCESS, rl.try_acquire(1, OB_LOG_LEVEL_ERROR));
ASSERT_EQ(OB_EAGAIN, rl.try_acquire(2, OB_LOG_LEVEL_ERROR));
ASSERT_EQ(OB_SUCCESS, rl.try_acquire(1, OB_LOG_LEVEL_ERROR));
}
int main(int argc, char **argv)