add warning log when the number of sstable is at high level

src/storage/tablet/ob_table_store_util.cpp

@@ -134,7 +134,7 @@ int ObITableArray::copy(
 {
   int ret = OB_SUCCESS;
   if (OB_FAIL(init(allocator, other.count_))) {
-    LOG_ERROR("failed to init ObITableArray for copying", K(ret), K(other));
+    LOG_WARN("failed to init ObITableArray for copying", K(ret), K(other));
   } else {
     for (int64_t i = 0; OB_SUCC(ret) && i < count_; ++i) {
       if (OB_ISNULL(other[i])) {

src/storage/tablet/ob_tablet.cpp

@@ -2460,10 +2460,21 @@ int ObTablet::get_kept_multi_version_start(
     ls_min_reserved_snapshot = ls.get_min_reserved_snapshot();
   }
   if (OB_SUCC(ret)) {
+    const int64_t old_min_reserved_snapshot = min_reserved_snapshot;
     min_reserved_snapshot = common::min(
         ls_min_reserved_snapshot,
         common::min(min_reserved_snapshot, min_medium_snapshot));
     multi_version_start = MIN(MAX(min_reserved_snapshot, multi_version_start), tablet.get_snapshot_version());
+
+    const int64_t current_time = common::ObTimeUtility::fast_current_time() * 1000; // needs ns here.
+    if (current_time - multi_version_start > 120 * 60 * 1000 * 1000L /*2 hour*/) {
+      if (REACH_TENANT_TIME_INTERVAL(10 * 1000 * 1000L /*10s*/)) {
+        LOG_INFO("tablet multi version start not advance for a long time", K(ret),
+                "ls_id", tablet.get_tablet_meta().ls_id_, K(tablet_id),
+                K(multi_version_start), K(old_min_reserved_snapshot), K(min_medium_snapshot),
+                "ls_min_reserved_snapshot", ls.get_min_reserved_snapshot(), K(tablet));
+      }
+    }
   }
   LOG_DEBUG("get multi version start", "ls_id", tablet.get_tablet_meta().ls_id_, K(tablet_id),
       K(multi_version_start), K(min_reserved_snapshot), K(tablet.get_tablet_meta()), K(min_medium_snapshot),

src/storage/tablet/ob_tablet_table_store.cpp

@@ -1255,6 +1255,14 @@ int ObTabletTableStore::check_ready_for_read()
       is_ready_for_read_ = true;
     }
   }
+
+  if (OB_SUCC(ret) && get_table_count() > EMERGENCY_SSTABLE_CNT) {
+    int tmp_ret = OB_TOO_MANY_SSTABLE;
+    LOG_WARN("Emergency SSTable count, maybe frequency freeze occurs, or maybe multi_version_start not adavanced.",
+             K(tmp_ret),
+             "major table count: ", major_tables_.count(),
+             "minor table count: ", minor_tables_.count());
+  }
   return ret;
 }
 

src/storage/tablet/ob_tablet_table_store.h

@@ -236,6 +236,7 @@ private:
 public:
   static const int64_t TABLE_STORE_VERSION = 0x0100;
   static const int64_t MAX_SSTABLE_CNT = 128;
+  static const int64_t EMERGENCY_SSTABLE_CNT = 48;
 private:
   ObTablet *tablet_ptr_;
   ObSSTableArray major_tables_;

unittest/storage/test_parallel_minor_dag.cpp

@@ -4,10 +4,14 @@
 //
 
 #include <gtest/gtest.h>
+#include <gmock/gmock.h>
+#include <thread>
 
+#define USING_LOG_PREFIX STORAGE
 #define private public
 #define protected public
 
+#include "mtlenv/mock_tenant_module_env.h"
 #include "storage/compaction/ob_partition_merge_policy.h"
 #include "storage/ob_storage_struct.h"
 #include "storage/blocksstable/ob_sstable.h"
@@ -23,25 +27,18 @@ using namespace compaction;
 using namespace omt;
 using namespace share;
 
-namespace unittest
-{
-
 class TestParallelMinorDag : public ::testing::Test
 {
 public:
-  TestParallelMinorDag() : allocator_(ObModIds::TEST), tenant_base_(500) {}
+  TestParallelMinorDag()
+    : tenant_id_(1), allocator_(ObModIds::TEST), tenant_base_(tenant_id_)
+  {}
   virtual ~TestParallelMinorDag() {}
   int prepare_merge_result(const int64_t sstable_cnt, ObGetMergeTablesResult &result);
-
+  virtual void SetUp() override;
-  void SetUp()
+  virtual void TearDown() override;
-  {
+  static void SetUpTestCase();
-     ObTenantMetaMemMgr *t3m = OB_NEW(ObTenantMetaMemMgr, ObModIds::TEST, 500);
+  static void TearDownTestCase();
-     ASSERT_EQ(OB_SUCCESS, t3m->init());
-
-     tenant_base_.set(t3m);
-     ObTenantEnv::set_tenant(&tenant_base_);
-     ASSERT_EQ(OB_SUCCESS, tenant_base_.init());
-  }
 
   share::SCN get_start_log_ts(const int64_t idx);
   share::SCN get_end_log_ts(const int64_t idx);
@@ -56,11 +53,40 @@ public:
   static const int64_t TEST_COLUMN_CNT = 6;
   static const int64_t MAX_SSTABLE_CNT = 64;
 
+  const uint64_t tenant_id_;
   common::ObArenaAllocator allocator_;
   ObTenantBase tenant_base_;
   ObSSTable fake_sstables_[MAX_SSTABLE_CNT];
 };
 
+void TestParallelMinorDag::SetUpTestCase()
+{
+  EXPECT_EQ(OB_SUCCESS, MockTenantModuleEnv::get_instance().init());
+}
+
+void TestParallelMinorDag::TearDownTestCase()
+{
+  MockTenantModuleEnv::get_instance().destroy();
+}
+
+void TestParallelMinorDag::SetUp()
+{
+    ObTenantMetaMemMgr *t3m = OB_NEW(ObTenantMetaMemMgr, ObModIds::TEST, tenant_id_);
+    ASSERT_EQ(OB_SUCCESS, t3m->init());
+
+    tenant_base_.set(t3m);
+    ObTenantEnv::set_tenant(&tenant_base_);
+    ASSERT_EQ(OB_SUCCESS, tenant_base_.init());
+}
+
+void TestParallelMinorDag::TearDown()
+{
+  ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr *);
+  t3m->destroy();
+  ObTenantEnv::set_tenant(nullptr);
+}
+
+
 int TestParallelMinorDag::prepare_merge_result(
     const int64_t sstable_cnt,
     ObGetMergeTablesResult &result)
@@ -150,6 +176,8 @@ void TestParallelMinorDag::check_result(
   }
 }
 
+namespace unittest
+{
 TEST_F(TestParallelMinorDag, test_parallel_interval)
 {
   for (int64_t minor_compact_trigger = 2; minor_compact_trigger <= 16; ++minor_compact_trigger) {