prevent mini dag from being created when sstable count reaches up limit && optimize perf when getting tenant config

src/storage/access/ob_multiple_merge.h

@@ -28,6 +28,7 @@
 #include "ob_table_access_context.h"
 #include "storage/meta_mem/ob_tablet_handle.h"
 #include "storage/lob/ob_lob_data_reader.h"
+#include "storage/compaction/ob_tenant_tablet_scheduler.h"
 
 namespace oceanbase
 {
@@ -171,14 +172,7 @@ OB_INLINE int ObMultipleMerge::check_need_refresh_table(bool &need_refresh)
 OB_INLINE int ObMultipleMerge::update_and_report_tablet_stat()
 {
   int ret = OB_SUCCESS;
-  bool enable_adaptive_compaction = true;
+  if (MTL(ObTenantTabletScheduler *)->enable_adaptive_compaction()) {
-  {
-    omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID()));
-    if (tenant_config.is_valid()) {
-      enable_adaptive_compaction = tenant_config->_enable_adaptive_compaction;
-    }
-  }
-  if (enable_adaptive_compaction) {
     EVENT_ADD(ObStatEventIds::STORAGE_READ_ROW_COUNT, scan_cnt_);
     access_ctx_->table_store_stat_.access_row_cnt_ += row_stat_.filt_del_count_;
     if (NULL != access_ctx_->table_scan_stat_) {

src/storage/compaction/ob_partition_merge_policy.cpp

@@ -1326,15 +1326,21 @@ int ObAdaptiveMergePolicy::find_meta_major_tables(
     if (OB_FAIL(ret)) {
     } else if (tx_determ_table_cnt < 2) {
       ret = OB_NO_NEED_MERGE;
+      if (REACH_TENANT_TIME_INTERVAL(60 * 1000 * 1000/*60s*/)) {
         LOG_INFO("no enough table for meta merge", K(ret), K(result), K(table_store));
+      }
     } else if (inc_row_cnt < TRANS_STATE_DETERM_ROW_CNT_THRESHOLD
       || inc_row_cnt < INC_ROW_COUNT_PERCENTAGE_THRESHOLD * base_row_cnt) {
       ret = OB_NO_NEED_MERGE;
+      if (REACH_TENANT_TIME_INTERVAL(60 * 1000 * 1000/*60s*/)) {
         LOG_INFO("found sstable could merge is not enough", K(ret), K(inc_row_cnt), K(base_row_cnt));
+      }
     } else if (result.version_range_.snapshot_version_ < tablet.get_multi_version_start()) {
       ret = OB_NO_NEED_MERGE;
+      if (REACH_TENANT_TIME_INTERVAL(60 * 1000 * 1000/*60s*/)) {
         LOG_INFO("chosen snapshot is abandoned", K(ret), K(result), K(tablet.get_multi_version_start()));
       }
+    }
 
 #ifdef ERRSIM
   ret = OB_E(EventTable::EN_SCHEDULE_MEDIUM_COMPACTION) ret;

src/storage/compaction/ob_tablet_merge_task.cpp

@@ -240,6 +240,21 @@ int ObBasicTabletMergeDag::get_tablet_and_compat_mode()
     compat_mode_ = tmp_tablet_handle.get_obj()->get_tablet_meta().compat_mode_;
   }
 
+  if (OB_SUCC(ret) && is_mini_merge(merge_type_)) {
+    int64_t inc_sstable_cnt = tmp_tablet_handle.get_obj()->get_table_store().get_minor_sstables().count() + 1/*major table*/;
+    bool is_exist = false;
+    if (OB_FAIL(MTL(ObTenantDagScheduler *)->check_dag_exist(this, is_exist))) {
+      LOG_WARN("failed to check dag exist", K(ret), K_(param));
+    } else if (is_exist) {
+      ++inc_sstable_cnt;
+    }
+    if (OB_SUCC(ret) && inc_sstable_cnt >= MAX_SSTABLE_CNT_IN_STORAGE) {
+      ret = OB_EAGAIN;
+      LOG_WARN("Too many sstables in tablet, cannot schdule mini compaction, retry later",
+          K(ret), K(inc_sstable_cnt), K(tmp_tablet_handle.get_obj()));
+    }
+  }
+
   int tmp_ret = OB_SUCCESS;
   if (OB_SUCC(ret)
       && typeid(*this) != typeid(ObTxTableMergeDag)
@@ -1285,13 +1300,7 @@ int ObTabletMergeFinishTask::try_schedule_compaction_after_mini(
   int tmp_ret = OB_SUCCESS;
   const ObTabletID &tablet_id = ctx.param_.tablet_id_;
   ObLSID ls_id = ctx.param_.ls_id_;
-  bool enable_adaptive_compaction = true;
+  bool enable_adaptive_compaction = MTL(ObTenantTabletScheduler *)->enable_adaptive_compaction();
-  {
-    omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID()));
-    if (tenant_config.is_valid()) {
-      enable_adaptive_compaction = tenant_config->_enable_adaptive_compaction;
-    }
-  }
   // report tablet stat
   if (0 == ctx.get_merge_info().get_sstable_merge_info().macro_block_count_) {
     // empty mini compaction, no need to reprot stat

src/storage/compaction/ob_tenant_tablet_scheduler.cpp

@@ -166,7 +166,8 @@ ObTenantTabletScheduler::ObTenantTabletScheduler()
    merge_loop_task_(),
    medium_loop_task_(),
    sstable_gc_task_(),
-   fast_freeze_checker_()
+   fast_freeze_checker_(),
+   enable_adaptive_compaction_(false)
 {
   STATIC_ASSERT(static_cast<int64_t>(NO_MAJOR_MERGE_TYPE_CNT) == ARRAYSIZEOF(MERGE_TYPES), "merge type array len is mismatch");
 }
@@ -204,6 +205,7 @@ int ObTenantTabletScheduler::init()
     omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID()));
     if (tenant_config.is_valid()) {
       schedule_interval = tenant_config->ob_compaction_schedule_interval;
+      enable_adaptive_compaction_ = tenant_config->_enable_adaptive_compaction;
       fast_freeze_checker_.reload_config(tenant_config->_ob_enable_fast_freeze);
     }
   } // end of ObTenantConfigGuard
@@ -266,6 +268,7 @@ int ObTenantTabletScheduler::reload_tenant_config()
     omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID()));
     if (tenant_config.is_valid()) {
       merge_schedule_interval = tenant_config->ob_compaction_schedule_interval;
+      enable_adaptive_compaction_ = tenant_config->_enable_adaptive_compaction;
       fast_freeze_checker_.reload_config(tenant_config->_ob_enable_fast_freeze);
     }
   } // end of ObTenantConfigGuard
@@ -931,13 +934,6 @@ int ObTenantTabletScheduler::schedule_ls_medium_merge(
     bool is_leader = false;
     bool could_major_merge = false;
     const int64_t major_frozen_scn = get_frozen_version();
-    bool enable_adaptive_compaction = true;
-    {
-      omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID()));
-      if (tenant_config.is_valid()) {
-        enable_adaptive_compaction = tenant_config->_enable_adaptive_compaction;
-      }
-    }
 
     if (MTL(ObTenantTabletScheduler *)->could_major_merge_start()) {
       could_major_merge = true;
@@ -1001,13 +997,13 @@ int ObTenantTabletScheduler::schedule_ls_medium_merge(
           } else if (ObTimeUtility::fast_current_time() <
               tablet->get_medium_compaction_info_list().get_wait_check_medium_scn() + WAIT_MEDIUM_CHECK_THRESHOLD) {
             // need wait 10 mins before schedule meta major
-          } else if (enable_adaptive_compaction && OB_TMP_FAIL(schedule_tablet_meta_major_merge(ls_handle, tablet_handle))) {
+          } else if (enable_adaptive_compaction_ && OB_TMP_FAIL(schedule_tablet_meta_major_merge(ls_handle, tablet_handle))) {
             if (OB_SIZE_OVERFLOW != tmp_ret && OB_EAGAIN != tmp_ret) {
               LOG_WARN("failed to schedule tablet merge", K(tmp_ret), K(ls_id), K(tablet_id));
             }
           }
         } else if (could_major_merge
-          && (!tablet_merge_finish || enable_adaptive_compaction)
+          && (!tablet_merge_finish || enable_adaptive_compaction_)
           && OB_TMP_FAIL(func.schedule_next_medium_for_leader(
             tablet_merge_finish ? 0 : merge_version, schedule_stats_))) { // schedule another round
           LOG_WARN("failed to schedule next medium", K(tmp_ret), K(ls_id), K(tablet_id));

src/storage/compaction/ob_tenant_tablet_scheduler.h

@@ -103,6 +103,7 @@ public:
   void wait();
   bool is_stop() const { return is_stop_; }
   int reload_tenant_config();
+  bool enable_adaptive_compaction() const { return enable_adaptive_compaction_; }
 
   // major merge status control
   void stop_major_merge();
@@ -241,6 +242,7 @@ private:
   MediumLoopTask medium_loop_task_;
   SSTableGCTask sstable_gc_task_;
   ObFastFreezeChecker fast_freeze_checker_;
+  bool enable_adaptive_compaction_;
 };
 
 } // namespace storage

src/storage/tablet/ob_tablet.cpp

@@ -39,6 +39,7 @@
 #include "storage/blocksstable/ob_sstable.h"
 #include "storage/blocksstable/ob_sstable_sec_meta_iterator.h"
 #include "storage/compaction/ob_tenant_freeze_info_mgr.h"
+#include "storage/compaction/ob_tenant_tablet_scheduler.h"
 #include "storage/memtable/ob_memtable.h"
 #include "storage/meta_mem/ob_tenant_meta_mem_mgr.h"
 #include "storage/meta_mem/ob_meta_obj_struct.h"
@@ -1484,15 +1485,8 @@ int ObTablet::do_rowkey_exists(
         // ROWKEY IN_ROW_CACHE / NOT EXIST
       } else if (FALSE_IT(store_ctx.tablet_stat_.exist_row_read_table_cnt_ = check_table_cnt)) {
       } else if (FALSE_IT(store_ctx.tablet_stat_.exist_row_total_table_cnt_ = table_iter.count())) {
-      } else {
+      } else if (MTL(ObTenantTabletScheduler *)->enable_adaptive_compaction()) {
-        bool enable_adaptive_compaction = true;
+        if (OB_TMP_FAIL(MTL(ObTenantTabletStatMgr *)->report_stat(store_ctx.tablet_stat_))) {
-        {
-          omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID()));
-          if (tenant_config.is_valid()) {
-            enable_adaptive_compaction = tenant_config->_enable_adaptive_compaction;
-          }
-        }
-        if (enable_adaptive_compaction && OB_TMP_FAIL(MTL(ObTenantTabletStatMgr *)->report_stat(store_ctx.tablet_stat_))) {
           LOG_WARN("failed to report tablet stat", K(tmp_ret), K(stat));
         }
       }
@@ -1545,15 +1539,8 @@ int ObTablet::do_rowkeys_exist(ObTableStoreIterator &tables_iter, ObRowsInfo &ro
     int tmp_ret = OB_SUCCESS;
     if (0 == access_ctx.table_store_stat_.exist_row_.empty_read_cnt_) {
       // ROWKEY IN_ROW_CACHE / NOT EXIST
-    } else {
+    } else if (MTL(ObTenantTabletScheduler *)->enable_adaptive_compaction()) {
-      bool enable_adaptive_compaction = true;
+      if (OB_TMP_FAIL(MTL(ObTenantTabletStatMgr *)->report_stat(tablet_stat))) {
-      {
-        omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID()));
-        if (tenant_config.is_valid()) {
-          enable_adaptive_compaction = tenant_config->_enable_adaptive_compaction;
-        }
-      }
-      if (enable_adaptive_compaction && OB_TMP_FAIL(MTL(ObTenantTabletStatMgr *)->report_stat(tablet_stat))) {
         LOG_WARN("failed to report tablet stat", K(tmp_ret), K(tablet_stat));
       }
     }