[CP] [4377] add resourcer defensive after 4377

src/share/parameter/ob_parameter_seed.ipp

@@ -255,6 +255,11 @@ DEF_INT_WITH_CHECKER(_enable_defensive_check, OB_CLUSTER_PARAMETER, "1",
                      "1 means normal defensive check is enabled, "
                      "2 means more strict defensive check is enabled, such as check partition id validity",
                      ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
+
+DEF_BOOL(_delay_resource_recycle_after_correctness_issue, OB_CLUSTER_PARAMETER, "False",
+         "whether hinder the recycling of the log resources and the sstable resources under correctness issues",
+         ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
+
 //
 DEF_BOOL(_sql_insert_multi_values_split_opt, OB_CLUSTER_PARAMETER, "True",
          "True means that the split + batch optimization for inserting multiple rows of the insert values ​​statement can be done",

src/sql/das/ob_das_scan_op.cpp

@@ -18,6 +18,7 @@
 #include "sql/engine/px/ob_px_util.h"
 #include "sql/engine/ob_des_exec_context.h"
 #include "storage/access/ob_table_scan_iterator.h"
+#include "storage/concurrency_control/ob_data_validation_service.h"
 
 namespace oceanbase
 {
@@ -1201,6 +1202,7 @@ int ObLocalIndexLookupOp::check_lookup_row_cnt()
                       "data_table_tablet_id", tablet_id_ ,
                       KPC_(snapshot),
                       KPC_(tx_desc));
+      concurrency_control::ObDataValidationService::set_delay_resource_recycle(ls_id_);
       if (trans_info_array_.count() == scan_param_.key_ranges_.count()) {
         for (int64_t i = 0; i < trans_info_array_.count(); i++) {
           LOG_ERROR("dump TableLookup DAS Task trans_info and key_ranges", K(i),

src/storage/CMakeLists.txt

@@ -628,6 +628,7 @@ ob_set_subtarget(ob_storage compaction
 ob_set_subtarget(ob_storage concurrency_control
   concurrency_control/ob_multi_version_garbage_collector.cpp
   concurrency_control/ob_trans_stat_row.cpp
+  concurrency_control/ob_data_validation_service.cpp
 )
 
 ob_set_subtarget(ob_storage memtable

src/storage/access/ob_multiple_get_merge.cpp

@@ -13,6 +13,8 @@
 #include "ob_multiple_get_merge.h"
 #include "storage/ob_row_fuse.h"
 #include "storage/ob_storage_schema.h"
+#include "storage/tx_storage/ob_ls_service.h"
+#include "storage/concurrency_control/ob_data_validation_service.h"
 
 namespace oceanbase
 {
@@ -241,6 +243,7 @@ int ObMultipleGetMerge::inner_get_next_row(ObDatumRow &row)
                           K(rowkeys_->at(get_row_range_idx_ - 1)),
                           K(fuse_row),
                           KPC(access_ctx_->store_ctx_));
+            concurrency_control::ObDataValidationService::set_delay_resource_recycle(access_ctx_->ls_id_);
             dump_table_statistic_for_4377();
             dump_tx_statistic_for_4377(access_ctx_->store_ctx_);
           }

src/storage/access/ob_single_merge.cpp

@@ -18,6 +18,7 @@
 #include "storage/tablet/ob_tablet.h"
 #include "storage/tx/ob_defensive_check_mgr.h"
 #include "storage/column_store/ob_co_sstable_row_getter.h"
+#include "storage/concurrency_control/ob_data_validation_service.h"
 
 namespace oceanbase
 {
@@ -373,6 +374,7 @@ int ObSingleMerge::inner_get_next_row(ObDatumRow &row)
                     KPC(read_info),
                     KPC(access_ctx_->store_ctx_),
                     K(tables_));
+      concurrency_control::ObDataValidationService::set_delay_resource_recycle(access_ctx_->ls_id_);
       dump_table_statistic_for_4377();
       dump_tx_statistic_for_4377(access_ctx_->store_ctx_);
     }

src/storage/concurrency_control/ob_data_validation_service.cpp

@@ -0,0 +1,76 @@
+/**
+ * Copyright (c) 2024 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include "storage/concurrency_control/ob_data_validation_service.h"
+
+namespace oceanbase
+{
+namespace concurrency_control
+{
+
+bool ObDataValidationService::need_delay_resource_recycle(const ObLSID ls_id)
+{
+  int ret = OB_SUCCESS;
+  ObLSService *ls_service = MTL(ObLSService*);
+  ObLSHandle handle;
+  ObLS *ls = nullptr;
+  const bool need_delay_opt = GCONF._delay_resource_recycle_after_correctness_issue;
+  bool need_delay_ret = false;
+
+  if (OB_UNLIKELY(!ls_id.is_valid())) {
+    ret = OB_INVALID_ARGUMENT;
+    TRANS_LOG(WARN, "invalid argument", K(ret), K(ls_id));
+  } else if (OB_FAIL(ls_service->get_ls(ls_id, handle, ObLSGetMod::TXSTORAGE_MOD))) {
+    if (OB_LS_NOT_EXIST != ret) {
+      TRANS_LOG(DEBUG, "get log stream failed", K(ls_id), K(ret));
+    }
+  } else if (OB_ISNULL(ls = handle.get_ls())) {
+    ret = OB_ERR_UNEXPECTED;
+    TRANS_LOG(WARN, "get log stream failed", K(ls_id), K(ret));
+  } else {
+    need_delay_ret = ls->need_delay_resource_recycle() && need_delay_opt;
+
+    if (!need_delay_opt && ls->need_delay_resource_recycle()) {
+      ls->clear_delay_resource_recycle();
+    }
+  }
+
+  return need_delay_ret;
+}
+
+void ObDataValidationService::set_delay_resource_recycle(const ObLSID ls_id)
+{
+  int ret = OB_SUCCESS;
+  ObLSHandle handle;
+  ObLSService *ls_service = MTL(ObLSService*);
+  ObLS *ls = nullptr;
+  const bool need_delay_opt = GCONF._delay_resource_recycle_after_correctness_issue;
+
+  if (OB_LIKELY(!need_delay_opt)) {
+    // do nothing
+  } else if (OB_UNLIKELY(!ls_id.is_valid())) {
+    ret = OB_INVALID_ARGUMENT;
+    TRANS_LOG(WARN, "invalid argument", K(ret), K(ls_id));
+  } else if (OB_FAIL(ls_service->get_ls(ls_id, handle, ObLSGetMod::TXSTORAGE_MOD))) {
+    if (OB_LS_NOT_EXIST != ret) {
+      TRANS_LOG(DEBUG, "get log stream failed", K(ls_id), K(ret));
+    }
+  } else if (OB_ISNULL(ls = handle.get_ls())) {
+    ret = OB_ERR_UNEXPECTED;
+    TRANS_LOG(WARN, "get log stream failed", K(ls_id), K(ret));
+  } else {
+    ls->set_delay_resource_recycle();
+  }
+}
+
+} // namespace concurrency_control
+} // namespace oceanbase

src/storage/concurrency_control/ob_data_validation_service.h

@@ -0,0 +1,34 @@
+/**
+ * Copyright (c) 2024 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_STORAGE_CONCURRENCY_CONTROL_OB_DATA_VALIDATION_SERVICE
+#define OCEANBASE_STORAGE_CONCURRENCY_CONTROL_OB_DATA_VALIDATION_SERVICE
+
+#include "storage/ls/ob_ls.h"
+#include "storage/tx_storage/ob_ls_service.h"
+
+namespace oceanbase
+{
+namespace concurrency_control
+{
+
+class ObDataValidationService
+{
+public:
+  static bool need_delay_resource_recycle(const ObLSID ls_id);
+  static void set_delay_resource_recycle(const ObLSID ls_id);
+};
+
+} // namespace concurrency_control
+} // namespace oceanbase
+
+#endif // OCEANBASE_STORAGE_CONCURRENCY_CONTROL_OB_DATA_VALIDATION_SERVICE

src/storage/ls/ob_ls.cpp

@@ -106,7 +106,8 @@ ObLS::ObLS()
     switch_epoch_(0),
     ls_meta_(),
     rs_reporter_(nullptr),
-    startup_transfer_info_()
+    startup_transfer_info_(),
+    need_delay_resource_recycle_(false)
 {}
 
 ObLS::~ObLS()
@@ -211,6 +212,7 @@ int ObLS::init(const share::ObLSID &ls_id,
       LOG_WARN("register to service failed", K(ret));
     } else {
       election_priority_.set_ls_id(ls_id);
+      need_delay_resource_recycle_ = false;
       is_inited_ = true;
       LOG_INFO("ls init success", K(ls_id));
     }
@@ -740,6 +742,7 @@ void ObLS::destroy()
   tenant_id_ = OB_INVALID_TENANT_ID;
   startup_transfer_info_.reset();
   ls_transfer_status_.reset();
+  need_delay_resource_recycle_ = false;
 }
 
 int ObLS::offline_tx_(const int64_t start_ts)
@@ -2513,6 +2516,22 @@ int ObLS::set_ls_migration_gc(
   return ret;
 }
 
+bool ObLS::need_delay_resource_recycle() const
+{
+  LOG_INFO("need delay resource recycle", KPC(this));
+  return need_delay_resource_recycle_;
+}
 
+void ObLS::set_delay_resource_recycle()
+{
+  need_delay_resource_recycle_ = true;
+  LOG_INFO("set delay resource recycle", KPC(this));
+}
+
+void ObLS::clear_delay_resource_recycle()
+{
+  need_delay_resource_recycle_ = false;
+  LOG_INFO("clear delay resource recycle", KPC(this));
+}
 }
 }

src/storage/ls/ob_ls.h

@@ -388,7 +388,13 @@ public:
   int check_can_replay_clog(bool &can_replay);
   int check_ls_need_online(bool &need_online);
 
-  TO_STRING_KV(K_(running_state), K_(ls_meta), K_(switch_epoch), K_(log_handler), K_(restore_handler), K_(is_inited), K_(tablet_gc_handler), K_(startup_transfer_info));
+  // for delaying the resource recycle after correctness issue
+  bool need_delay_resource_recycle() const;
+  void set_delay_resource_recycle();
+  void clear_delay_resource_recycle();
+
+  TO_STRING_KV(K_(running_state), K_(ls_meta), K_(switch_epoch), K_(log_handler), K_(restore_handler),
+               K_(is_inited), K_(tablet_gc_handler), K_(startup_transfer_info), K_(need_delay_resource_recycle));
 private:
   void update_state_seq_();
   int ls_init_for_dup_table_();
@@ -1010,6 +1016,9 @@ private:
   ObLSTransferStatus ls_transfer_status_;
   // this is used for the meta lock, and will be removed later
   RWLock meta_rwlock_;
+
+  // for delaying the resource recycle after correctness issue
+  bool need_delay_resource_recycle_;
 };
 
 }

src/storage/ls/ob_ls_tablet_service.cpp

@@ -76,6 +76,7 @@
 #include "observer/ob_server_event_history_table_operator.h"
 #include "storage/high_availability/ob_storage_ha_utils.h"
 #include "storage/slog_ckpt/ob_tenant_checkpoint_slog_handler.h"
+#include "storage/concurrency_control/ob_data_validation_service.h"
 
 using namespace oceanbase::share;
 using namespace oceanbase::common;
@@ -3831,6 +3832,7 @@ int ObLSTabletService::check_old_row_legitimacy(
             "dml_flag", run_ctx.dml_flag_,
             "store_ctx", run_ctx.store_ctx_,
             "relative_table", run_ctx.relative_table_);
+        concurrency_control::ObDataValidationService::set_delay_resource_recycle(run_ctx.store_ctx_.ls_id_);
         LOG_ERROR("Dump data table info", K(ret), K(data_table));
         run_ctx.store_ctx_.force_print_trace_log();
       }

src/storage/tx_storage/ob_ls_service.cpp

@@ -1439,8 +1439,6 @@ int ObLSService::dump_ls_info()
   return ret;
 }
 
-
-
 } // storage
 } // oceanbase
 

tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result

@@ -269,6 +269,7 @@ _ctx_memory_limit
 _datafile_usage_lower_bound_percentage
 _datafile_usage_upper_bound_percentage
 _data_storage_io_timeout
+_delay_resource_recycle_after_correctness_issue
 _enable_adaptive_compaction
 _enable_adaptive_merge_schedule
 _enable_backtrace_function