limit the print frequency for each ERROR log

deps/oblib/src/lib/oblog/ob_log.cpp

@@ -56,6 +56,7 @@ _RLOCAL(lib::ObRateLimiter*, ObLogger::tl_log_limiter_);
 static const int64_t limiter_initial = 1000;
 static const int64_t limiter_thereafter = 100;
 ObSyslogSampleRateLimiter ObLogger::per_log_limiters_[];
+ObSyslogSampleRateLimiter ObLogger::per_error_log_limiters_[];
 _RLOCAL(int32_t, ObLogger::tl_type_);
 _RLOCAL(uint64_t, ObLogger::curr_logging_seq_);
 _RLOCAL(uint64_t, ObLogger::last_logging_seq_);
@@ -1423,6 +1424,9 @@ int ObLogger::init(const ObBaseLogWriterCfg &log_cfg,
     for (int i = 0; i < ARRAYSIZEOF(per_log_limiters_); i++) {
       new (&per_log_limiters_[i])ObSyslogSampleRateLimiter(limiter_initial, limiter_thereafter);
     }
+    for (int i = 0; i < ARRAYSIZEOF(per_error_log_limiters_); i++) {
+      new (&per_error_log_limiters_[i])ObSyslogSampleRateLimiter(limiter_initial, limiter_thereafter);
+    }
     const int64_t limit = ObBaseLogWriterCfg::DEFAULT_MAX_BUFFER_ITEM_CNT * OB_MALLOC_BIG_BLOCK_SIZE / 8; // 256M
     log_mem_limiter_ = new (buf) ObBlockAllocMgr(limit);
     allocator_ = new (log_mem_limiter_ + 1) ObVSliceAlloc();
@@ -1676,10 +1680,13 @@ int ObLogger::check_tl_log_limiter(const uint64_t location_hash_val,
         if (!allow) { limiter_info = " REACH SYSLOG RATE LIMIT [bandwidth]"; }
       }
       if (allow) {
-        int64_t idx0 = (location_hash_val >> 32) % N_LIMITER;
+        int n_limiter = (OB_LOG_LEVEL_ERROR == level) ? N_ERROR_LIMITER : N_LIMITER;
-        int64_t idx1 = ((location_hash_val << 32) >> 32) % N_LIMITER;
+        ObSyslogSampleRateLimiter *log_sample_rate_limiters =
-        bool r0 = OB_SUCCESS == per_log_limiters_[idx0].try_acquire(1, level, errcode);
+            (OB_LOG_LEVEL_ERROR == level) ? per_error_log_limiters_ : per_log_limiters_;
-        bool r1 = OB_SUCCESS == per_log_limiters_[idx1].try_acquire(1, level, errcode);
+        int64_t idx0 = (location_hash_val >> 32) % n_limiter;
+        int64_t idx1 = ((location_hash_val << 32) >> 32) % n_limiter;
+        bool r0 = OB_SUCCESS == log_sample_rate_limiters[idx0].try_acquire(1, level, errcode);
+        bool r1 = OB_SUCCESS == log_sample_rate_limiters[idx1].try_acquire(1, level, errcode);
         allow = r0 && r1;
         if (!allow) { limiter_info = " REACH SYSLOG RATE LIMIT [frequency]"; }
       }

deps/oblib/src/lib/oblog/ob_log.h

@@ -753,7 +753,9 @@ private:
   static ::oceanbase::lib::ObRateLimiter *default_log_limiter_;
   RLOCAL_STATIC(lib::ObRateLimiter*, tl_log_limiter_);
   static constexpr int N_LIMITER = 4096;
+  static constexpr int N_ERROR_LIMITER = 1024;
   static ObSyslogSampleRateLimiter per_log_limiters_[N_LIMITER];
+  static ObSyslogSampleRateLimiter per_error_log_limiters_[N_ERROR_LIMITER];
   RLOCAL_STATIC(int32_t, tl_type_);
   //used for stat logging time and log dropped
   RLOCAL_STATIC(uint64_t, curr_logging_seq_);

deps/oblib/src/lib/oblog/ob_syslog_rate_limiter.h

@@ -53,10 +53,6 @@ public:
     if (!in_acquire) {
       if (0 == rate_) {
         ret = common::OB_EAGAIN;
-      } else if (OB_LOG_LEVEL_ERROR == log_level
-                 || OB_LOG_LEVEL_DBA_WARN == log_level
-                 || OB_LOG_LEVEL_DBA_ERROR == log_level) {
-        // allow
       } else {
         in_acquire = 1;
         ret = do_acquire(permits, log_level, errcode);
@@ -117,9 +113,16 @@ public:
   void reset_force_allows() override { };
   int do_acquire(int64_t permits, int log_level, int errcode) override
   {
-    UNUSED(log_level);
     UNUSED(errcode);
-    return impl_.try_acquire(permits);
+    int ret = OB_SUCCESS;
+    if (OB_LOG_LEVEL_ERROR == log_level
+        || OB_LOG_LEVEL_DBA_WARN == log_level
+        || OB_LOG_LEVEL_DBA_ERROR == log_level) {
+      // allow
+    } else {
+      ret = impl_.try_acquire(permits);
+    }
+    return ret;
   }
 
 private:

deps/oblib/unittest/lib/utility/test_sample_rate_limiter.cpp

@@ -34,6 +34,12 @@ TEST(TestSampleRateLimiter, Basic)
   ASSERT_EQ(OB_SUCCESS, rl.try_acquire(1, OB_LOG_LEVEL_INFO));
   ASSERT_EQ(OB_EAGAIN, rl.try_acquire(2, OB_LOG_LEVEL_INFO));
   ASSERT_EQ(OB_SUCCESS, rl.try_acquire(1, OB_LOG_LEVEL_INFO));
+
+  sleep(1);
+  ASSERT_EQ(OB_SUCCESS, rl.try_acquire(1, OB_LOG_LEVEL_ERROR));
+  ASSERT_EQ(OB_EAGAIN, rl.try_acquire(2, OB_LOG_LEVEL_ERROR));
+  ASSERT_EQ(OB_SUCCESS, rl.try_acquire(1, OB_LOG_LEVEL_ERROR));
+
 }
 
 int main(int argc, char **argv)