From 2eeccaa3830413ac1cff7d7c647a878a51a3b871 Mon Sep 17 00:00:00 2001 From: Handora Date: Fri, 9 Feb 2024 10:14:08 +0000 Subject: [PATCH] [CP] [4377] add resourcer defensive after 4377 --- src/share/parameter/ob_parameter_seed.ipp | 5 ++ src/sql/das/ob_das_scan_op.cpp | 2 + src/storage/CMakeLists.txt | 1 + src/storage/access/ob_multiple_get_merge.cpp | 3 + src/storage/access/ob_single_merge.cpp | 2 + .../ob_data_validation_service.cpp | 76 +++++++++++++++++++ .../ob_data_validation_service.h | 34 +++++++++ src/storage/ls/ob_ls.cpp | 21 ++++- src/storage/ls/ob_ls.h | 11 ++- src/storage/ls/ob_ls_tablet_service.cpp | 2 + src/storage/tx_storage/ob_ls_service.cpp | 2 - .../all_virtual_sys_parameter_stat.result | 1 + 12 files changed, 156 insertions(+), 4 deletions(-) create mode 100644 src/storage/concurrency_control/ob_data_validation_service.cpp create mode 100644 src/storage/concurrency_control/ob_data_validation_service.h diff --git a/src/share/parameter/ob_parameter_seed.ipp b/src/share/parameter/ob_parameter_seed.ipp index c770c797b6..d1b61e5175 100644 --- a/src/share/parameter/ob_parameter_seed.ipp +++ b/src/share/parameter/ob_parameter_seed.ipp @@ -255,6 +255,11 @@ DEF_INT_WITH_CHECKER(_enable_defensive_check, OB_CLUSTER_PARAMETER, "1", "1 means normal defensive check is enabled, " "2 means more strict defensive check is enabled, such as check partition id validity", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); + +DEF_BOOL(_delay_resource_recycle_after_correctness_issue, OB_CLUSTER_PARAMETER, "False", + "whether hinder the recycling of the log resources and the sstable resources under correctness issues", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); + // DEF_BOOL(_sql_insert_multi_values_split_opt, OB_CLUSTER_PARAMETER, "True", "True means that the split + batch optimization for inserting multiple rows of the insert values ​​statement can be done", diff --git a/src/sql/das/ob_das_scan_op.cpp b/src/sql/das/ob_das_scan_op.cpp index dbcad94e80..d38e1f9b74 100644 --- a/src/sql/das/ob_das_scan_op.cpp +++ b/src/sql/das/ob_das_scan_op.cpp @@ -18,6 +18,7 @@ #include "sql/engine/px/ob_px_util.h" #include "sql/engine/ob_des_exec_context.h" #include "storage/access/ob_table_scan_iterator.h" +#include "storage/concurrency_control/ob_data_validation_service.h" namespace oceanbase { @@ -1201,6 +1202,7 @@ int ObLocalIndexLookupOp::check_lookup_row_cnt() "data_table_tablet_id", tablet_id_ , KPC_(snapshot), KPC_(tx_desc)); + concurrency_control::ObDataValidationService::set_delay_resource_recycle(ls_id_); if (trans_info_array_.count() == scan_param_.key_ranges_.count()) { for (int64_t i = 0; i < trans_info_array_.count(); i++) { LOG_ERROR("dump TableLookup DAS Task trans_info and key_ranges", K(i), diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 33de5d2fb6..c0f389f660 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -628,6 +628,7 @@ ob_set_subtarget(ob_storage compaction ob_set_subtarget(ob_storage concurrency_control concurrency_control/ob_multi_version_garbage_collector.cpp concurrency_control/ob_trans_stat_row.cpp + concurrency_control/ob_data_validation_service.cpp ) ob_set_subtarget(ob_storage memtable diff --git a/src/storage/access/ob_multiple_get_merge.cpp b/src/storage/access/ob_multiple_get_merge.cpp index 6ab0fdad84..06bcee166d 100644 --- a/src/storage/access/ob_multiple_get_merge.cpp +++ b/src/storage/access/ob_multiple_get_merge.cpp @@ -13,6 +13,8 @@ #include "ob_multiple_get_merge.h" #include "storage/ob_row_fuse.h" #include "storage/ob_storage_schema.h" +#include "storage/tx_storage/ob_ls_service.h" +#include "storage/concurrency_control/ob_data_validation_service.h" namespace oceanbase { @@ -241,6 +243,7 @@ int ObMultipleGetMerge::inner_get_next_row(ObDatumRow &row) K(rowkeys_->at(get_row_range_idx_ - 1)), K(fuse_row), KPC(access_ctx_->store_ctx_)); + concurrency_control::ObDataValidationService::set_delay_resource_recycle(access_ctx_->ls_id_); dump_table_statistic_for_4377(); dump_tx_statistic_for_4377(access_ctx_->store_ctx_); } diff --git a/src/storage/access/ob_single_merge.cpp b/src/storage/access/ob_single_merge.cpp index 9273d7a24d..367e419d28 100644 --- a/src/storage/access/ob_single_merge.cpp +++ b/src/storage/access/ob_single_merge.cpp @@ -18,6 +18,7 @@ #include "storage/tablet/ob_tablet.h" #include "storage/tx/ob_defensive_check_mgr.h" #include "storage/column_store/ob_co_sstable_row_getter.h" +#include "storage/concurrency_control/ob_data_validation_service.h" namespace oceanbase { @@ -373,6 +374,7 @@ int ObSingleMerge::inner_get_next_row(ObDatumRow &row) KPC(read_info), KPC(access_ctx_->store_ctx_), K(tables_)); + concurrency_control::ObDataValidationService::set_delay_resource_recycle(access_ctx_->ls_id_); dump_table_statistic_for_4377(); dump_tx_statistic_for_4377(access_ctx_->store_ctx_); } diff --git a/src/storage/concurrency_control/ob_data_validation_service.cpp b/src/storage/concurrency_control/ob_data_validation_service.cpp new file mode 100644 index 0000000000..8daa3ec6a1 --- /dev/null +++ b/src/storage/concurrency_control/ob_data_validation_service.cpp @@ -0,0 +1,76 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "storage/concurrency_control/ob_data_validation_service.h" + +namespace oceanbase +{ +namespace concurrency_control +{ + +bool ObDataValidationService::need_delay_resource_recycle(const ObLSID ls_id) +{ + int ret = OB_SUCCESS; + ObLSService *ls_service = MTL(ObLSService*); + ObLSHandle handle; + ObLS *ls = nullptr; + const bool need_delay_opt = GCONF._delay_resource_recycle_after_correctness_issue; + bool need_delay_ret = false; + + if (OB_UNLIKELY(!ls_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + TRANS_LOG(WARN, "invalid argument", K(ret), K(ls_id)); + } else if (OB_FAIL(ls_service->get_ls(ls_id, handle, ObLSGetMod::TXSTORAGE_MOD))) { + if (OB_LS_NOT_EXIST != ret) { + TRANS_LOG(DEBUG, "get log stream failed", K(ls_id), K(ret)); + } + } else if (OB_ISNULL(ls = handle.get_ls())) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "get log stream failed", K(ls_id), K(ret)); + } else { + need_delay_ret = ls->need_delay_resource_recycle() && need_delay_opt; + + if (!need_delay_opt && ls->need_delay_resource_recycle()) { + ls->clear_delay_resource_recycle(); + } + } + + return need_delay_ret; +} + +void ObDataValidationService::set_delay_resource_recycle(const ObLSID ls_id) +{ + int ret = OB_SUCCESS; + ObLSHandle handle; + ObLSService *ls_service = MTL(ObLSService*); + ObLS *ls = nullptr; + const bool need_delay_opt = GCONF._delay_resource_recycle_after_correctness_issue; + + if (OB_LIKELY(!need_delay_opt)) { + // do nothing + } else if (OB_UNLIKELY(!ls_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + TRANS_LOG(WARN, "invalid argument", K(ret), K(ls_id)); + } else if (OB_FAIL(ls_service->get_ls(ls_id, handle, ObLSGetMod::TXSTORAGE_MOD))) { + if (OB_LS_NOT_EXIST != ret) { + TRANS_LOG(DEBUG, "get log stream failed", K(ls_id), K(ret)); + } + } else if (OB_ISNULL(ls = handle.get_ls())) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "get log stream failed", K(ls_id), K(ret)); + } else { + ls->set_delay_resource_recycle(); + } +} + +} // namespace concurrency_control +} // namespace oceanbase diff --git a/src/storage/concurrency_control/ob_data_validation_service.h b/src/storage/concurrency_control/ob_data_validation_service.h new file mode 100644 index 0000000000..22226547f5 --- /dev/null +++ b/src/storage/concurrency_control/ob_data_validation_service.h @@ -0,0 +1,34 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_STORAGE_CONCURRENCY_CONTROL_OB_DATA_VALIDATION_SERVICE +#define OCEANBASE_STORAGE_CONCURRENCY_CONTROL_OB_DATA_VALIDATION_SERVICE + +#include "storage/ls/ob_ls.h" +#include "storage/tx_storage/ob_ls_service.h" + +namespace oceanbase +{ +namespace concurrency_control +{ + +class ObDataValidationService +{ +public: + static bool need_delay_resource_recycle(const ObLSID ls_id); + static void set_delay_resource_recycle(const ObLSID ls_id); +}; + +} // namespace concurrency_control +} // namespace oceanbase + +#endif // OCEANBASE_STORAGE_CONCURRENCY_CONTROL_OB_DATA_VALIDATION_SERVICE diff --git a/src/storage/ls/ob_ls.cpp b/src/storage/ls/ob_ls.cpp index 19ab70f84f..748a400645 100644 --- a/src/storage/ls/ob_ls.cpp +++ b/src/storage/ls/ob_ls.cpp @@ -106,7 +106,8 @@ ObLS::ObLS() switch_epoch_(0), ls_meta_(), rs_reporter_(nullptr), - startup_transfer_info_() + startup_transfer_info_(), + need_delay_resource_recycle_(false) {} ObLS::~ObLS() @@ -211,6 +212,7 @@ int ObLS::init(const share::ObLSID &ls_id, LOG_WARN("register to service failed", K(ret)); } else { election_priority_.set_ls_id(ls_id); + need_delay_resource_recycle_ = false; is_inited_ = true; LOG_INFO("ls init success", K(ls_id)); } @@ -740,6 +742,7 @@ void ObLS::destroy() tenant_id_ = OB_INVALID_TENANT_ID; startup_transfer_info_.reset(); ls_transfer_status_.reset(); + need_delay_resource_recycle_ = false; } int ObLS::offline_tx_(const int64_t start_ts) @@ -2513,6 +2516,22 @@ int ObLS::set_ls_migration_gc( return ret; } +bool ObLS::need_delay_resource_recycle() const +{ + LOG_INFO("need delay resource recycle", KPC(this)); + return need_delay_resource_recycle_; +} +void ObLS::set_delay_resource_recycle() +{ + need_delay_resource_recycle_ = true; + LOG_INFO("set delay resource recycle", KPC(this)); +} + +void ObLS::clear_delay_resource_recycle() +{ + need_delay_resource_recycle_ = false; + LOG_INFO("clear delay resource recycle", KPC(this)); +} } } diff --git a/src/storage/ls/ob_ls.h b/src/storage/ls/ob_ls.h index 258f11f61d..2873f31c63 100644 --- a/src/storage/ls/ob_ls.h +++ b/src/storage/ls/ob_ls.h @@ -388,7 +388,13 @@ public: int check_can_replay_clog(bool &can_replay); int check_ls_need_online(bool &need_online); - TO_STRING_KV(K_(running_state), K_(ls_meta), K_(switch_epoch), K_(log_handler), K_(restore_handler), K_(is_inited), K_(tablet_gc_handler), K_(startup_transfer_info)); + // for delaying the resource recycle after correctness issue + bool need_delay_resource_recycle() const; + void set_delay_resource_recycle(); + void clear_delay_resource_recycle(); + + TO_STRING_KV(K_(running_state), K_(ls_meta), K_(switch_epoch), K_(log_handler), K_(restore_handler), + K_(is_inited), K_(tablet_gc_handler), K_(startup_transfer_info), K_(need_delay_resource_recycle)); private: void update_state_seq_(); int ls_init_for_dup_table_(); @@ -1010,6 +1016,9 @@ private: ObLSTransferStatus ls_transfer_status_; // this is used for the meta lock, and will be removed later RWLock meta_rwlock_; + + // for delaying the resource recycle after correctness issue + bool need_delay_resource_recycle_; }; } diff --git a/src/storage/ls/ob_ls_tablet_service.cpp b/src/storage/ls/ob_ls_tablet_service.cpp index 06a4ce1f03..6882352f56 100644 --- a/src/storage/ls/ob_ls_tablet_service.cpp +++ b/src/storage/ls/ob_ls_tablet_service.cpp @@ -76,6 +76,7 @@ #include "observer/ob_server_event_history_table_operator.h" #include "storage/high_availability/ob_storage_ha_utils.h" #include "storage/slog_ckpt/ob_tenant_checkpoint_slog_handler.h" +#include "storage/concurrency_control/ob_data_validation_service.h" using namespace oceanbase::share; using namespace oceanbase::common; @@ -3831,6 +3832,7 @@ int ObLSTabletService::check_old_row_legitimacy( "dml_flag", run_ctx.dml_flag_, "store_ctx", run_ctx.store_ctx_, "relative_table", run_ctx.relative_table_); + concurrency_control::ObDataValidationService::set_delay_resource_recycle(run_ctx.store_ctx_.ls_id_); LOG_ERROR("Dump data table info", K(ret), K(data_table)); run_ctx.store_ctx_.force_print_trace_log(); } diff --git a/src/storage/tx_storage/ob_ls_service.cpp b/src/storage/tx_storage/ob_ls_service.cpp index edaddac56b..a165c0c374 100644 --- a/src/storage/tx_storage/ob_ls_service.cpp +++ b/src/storage/tx_storage/ob_ls_service.cpp @@ -1439,8 +1439,6 @@ int ObLSService::dump_ls_info() return ret; } - - } // storage } // oceanbase diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result index cfd1fba9a3..fe4a58f080 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result @@ -269,6 +269,7 @@ _ctx_memory_limit _datafile_usage_lower_bound_percentage _datafile_usage_upper_bound_percentage _data_storage_io_timeout +_delay_resource_recycle_after_correctness_issue _enable_adaptive_compaction _enable_adaptive_merge_schedule _enable_backtrace_function