From aeeef83dac8d653e792fda8505bd1cb6b05f74e9 Mon Sep 17 00:00:00 2001 From: obdev Date: Wed, 10 Apr 2024 12:46:00 +0000 Subject: [PATCH] [FEAT MERGE] transfer_diagnose 422 patch 431 --- src/observer/CMakeLists.txt | 2 + src/observer/ob_server.cpp | 20 + src/observer/omt/ob_multi_tenant.cpp | 3 +- ..._all_virtual_storage_ha_error_diagnose.cpp | 204 +++ ...ob_all_virtual_storage_ha_error_diagnose.h | 71 + ...b_all_virtual_storage_ha_perf_diagnose.cpp | 225 ++++ .../ob_all_virtual_storage_ha_perf_diagnose.h | 74 + .../ob_virtual_table_iterator_factory.cpp | 22 +- src/share/CMakeLists.txt | 2 + .../ob_inner_table_schema.12401_12450.cpp | 516 +++++++ .../ob_inner_table_schema.451_500.cpp | 532 ++++++++ .../ob_inner_table_schema.50451_50500.cpp | 270 ++++ .../ob_inner_table_schema.60451_60500.cpp | 180 +++ src/share/inner_table/ob_inner_table_schema.h | 40 +- .../inner_table/ob_inner_table_schema.lob.cpp | 2 +- .../ob_inner_table_schema_constants.h | 16 + .../inner_table/ob_inner_table_schema_def.py | 112 +- src/share/inner_table/table_id_to_name | 4 + src/share/ob_storage_ha_diagnose_operator.cpp | 369 +++++ src/share/ob_storage_ha_diagnose_operator.h | 91 ++ src/share/ob_storage_ha_diagnose_struct.cpp | 1195 +++++++++++++++++ src/share/ob_storage_ha_diagnose_struct.h | 395 ++++++ src/share/parameter/ob_parameter_seed.ipp | 7 + src/share/rc/ob_tenant_base.h | 2 + src/storage/CMakeLists.txt | 2 + .../high_availability/ob_finish_transfer.cpp | 147 +- .../high_availability/ob_finish_transfer.h | 19 +- .../ob_storage_ha_diagnose_mgr.cpp | 747 +++++++++++ .../ob_storage_ha_diagnose_mgr.h | 158 +++ .../ob_storage_ha_diagnose_service.cpp | 690 ++++++++++ .../ob_storage_ha_diagnose_service.h | 104 ++ .../high_availability/ob_storage_ha_utils.cpp | 446 ++++++ .../high_availability/ob_storage_ha_utils.h | 107 +- .../ob_tablet_backfill_tx.cpp | 119 +- .../high_availability/ob_tablet_backfill_tx.h | 15 + .../ob_transfer_backfill_tx.cpp | 27 + .../ob_transfer_backfill_tx.h | 10 + .../high_availability/ob_transfer_handler.cpp | 391 +++++- .../high_availability/ob_transfer_handler.h | 44 + .../high_availability/ob_transfer_struct.cpp | 416 ++++++ .../high_availability/ob_transfer_struct.h | 79 ++ .../ob_tablet_finish_transfer_mds_helper.cpp | 118 +- .../ob_tablet_start_transfer_mds_helper.cpp | 148 +- .../r/mysql/information_schema.result | 6 + .../all_virtual_sys_parameter_stat.result | 1 + .../r/mysql/desc_virtual_table_in_sys.result | 43 + .../r/mysql/inner_table_overall.result | 4 + 47 files changed, 8140 insertions(+), 55 deletions(-) create mode 100644 src/observer/virtual_table/ob_all_virtual_storage_ha_error_diagnose.cpp create mode 100644 src/observer/virtual_table/ob_all_virtual_storage_ha_error_diagnose.h create mode 100644 src/observer/virtual_table/ob_all_virtual_storage_ha_perf_diagnose.cpp create mode 100644 src/observer/virtual_table/ob_all_virtual_storage_ha_perf_diagnose.h create mode 100644 src/share/ob_storage_ha_diagnose_operator.cpp create mode 100644 src/share/ob_storage_ha_diagnose_operator.h create mode 100644 src/share/ob_storage_ha_diagnose_struct.cpp create mode 100644 src/share/ob_storage_ha_diagnose_struct.h create mode 100644 src/storage/high_availability/ob_storage_ha_diagnose_mgr.cpp create mode 100644 src/storage/high_availability/ob_storage_ha_diagnose_mgr.h create mode 100644 src/storage/high_availability/ob_storage_ha_diagnose_service.cpp create mode 100644 src/storage/high_availability/ob_storage_ha_diagnose_service.h diff --git a/src/observer/CMakeLists.txt b/src/observer/CMakeLists.txt index 054d7737d..0ce7d5666 100644 --- a/src/observer/CMakeLists.txt +++ b/src/observer/CMakeLists.txt @@ -408,6 +408,8 @@ ob_set_subtarget(ob_server virtual_table virtual_table/ob_all_virtual_checkpoint_diagnose_info.cpp virtual_table/ob_all_virtual_checkpoint_diagnose_memtable_info.cpp virtual_table/ob_all_virtual_nic_info.cpp + virtual_table/ob_all_virtual_storage_ha_error_diagnose.cpp + virtual_table/ob_all_virtual_storage_ha_perf_diagnose.cpp ) ob_server_add_target(ob_server) diff --git a/src/observer/ob_server.cpp b/src/observer/ob_server.cpp index 27971b318..bab86ceb3 100644 --- a/src/observer/ob_server.cpp +++ b/src/observer/ob_server.cpp @@ -110,6 +110,7 @@ #include "observer/ob_startup_accel_task_handler.h" #include "share/detect/ob_detect_manager.h" #include "observer/table/ttl/ob_table_ttl_task.h" +#include "storage/high_availability/ob_storage_ha_diagnose_service.h" #ifdef OB_BUILD_ARBITRATION #include "logservice/arbserver/palf_env_lite_mgr.h" #include "logservice/arbserver/ob_arb_srv_network_frame.h" @@ -506,6 +507,8 @@ int ObServer::init(const ObServerOptions &opts, const ObPLogWriterCfg &log_cfg) LOG_WARN("init ObDetectManagerThread failed", KR(ret)); } else if (OB_FAIL(wr_service_.init())) { LOG_WARN("failed to init wr service", K(ret)); + } else if (OB_FAIL(ObStorageHADiagService::instance().init(GCTX.sql_proxy_))) { + LOG_WARN("init storage ha diagnose service failed", K(ret)); } else { GDS.set_rpc_proxy(&rs_rpc_proxy_); } @@ -771,6 +774,9 @@ void ObServer::destroy() palf::election::GLOBAL_REPORT_TIMER.destroy(); FLOG_INFO("global election report timer destroyed"); + ObStorageHADiagService::instance().destroy(); + FLOG_INFO("storage ha diagnose destroy"); + FLOG_INFO("begin to destroy virtual tenant manager"); ObVirtualTenantManager::get_instance().destroy(); FLOG_INFO("virtual tenant manager destroyed"); @@ -964,6 +970,12 @@ int ObServer::start() } #endif + if (FAILEDx(ObStorageHADiagService::instance().start())) { + LOG_ERROR("fail to start storage ha diagnose service", KR(ret)); + } else { + FLOG_INFO("success to start storage ha diagnose service"); + } + if (FAILEDx(unix_domain_listener_.start())) { LOG_ERROR("fail to start unix domain listener", KR(ret)); } else { @@ -1446,6 +1458,10 @@ int ObServer::stop() FLOG_INFO("rpc network shutdowned"); } + FLOG_INFO("begin to stop storage ha diagnose service"); + ObStorageHADiagService::instance().stop(); + FLOG_INFO("storage ha diagnose service stopped"); + FLOG_INFO("begin to shutdown high priority rpc"); if (OB_FAIL(net_frame_.high_prio_rpc_shutdown())) { FLOG_WARN("fail to shutdown high priority rpc", KR(ret)); @@ -1759,6 +1775,10 @@ int ObServer::wait() ObClockGenerator::get_instance().wait(); FLOG_INFO("wait clock generator success"); + FLOG_INFO("begin to wait storage ha diagnose"); + ObStorageHADiagService::instance().wait(); + FLOG_INFO("wait storage ha diagnose success"); + gctx_.status_ = SS_STOPPED; FLOG_INFO("[OBSERVER_NOTICE] wait observer end", KR(ret)); if (OB_SUCCESS != fail_ret) { diff --git a/src/observer/omt/ob_multi_tenant.cpp b/src/observer/omt/ob_multi_tenant.cpp index 1d51d5ae6..be85f1f37 100644 --- a/src/observer/omt/ob_multi_tenant.cpp +++ b/src/observer/omt/ob_multi_tenant.cpp @@ -150,6 +150,7 @@ #include "observer/table/ttl/ob_ttl_service.h" #include "sql/dtl/ob_dtl_interm_result_manager.h" #include "storage/tablet/ob_tablet_memtable_mgr.h" +#include "storage/high_availability/ob_storage_ha_diagnose_mgr.h" #ifdef ERRSIM #include "share/errsim_module/ob_tenant_errsim_module_mgr.h" #include "share/errsim_module/ob_tenant_errsim_event_mgr.h" @@ -559,7 +560,7 @@ int ObMultiTenant::init(ObAddr myaddr, MTL_BIND2(mtl_new_default, rootserver::ObHeartbeatService::mtl_init, nullptr, rootserver::ObHeartbeatService::mtl_stop, rootserver::ObHeartbeatService::mtl_wait, mtl_destroy_default); MTL_BIND2(mtl_new_default, table::ObTTLService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); MTL_BIND2(mtl_new_default, ObEmptyReadBucket::mtl_init, nullptr, nullptr, nullptr, ObEmptyReadBucket::mtl_destroy); - + MTL_BIND2(mtl_new_default, storage::ObStorageHADiagMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default); #ifdef ERRSIM MTL_BIND2(mtl_new_default, ObTenantErrsimModuleMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default); MTL_BIND2(mtl_new_default, ObTenantErrsimEventMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default); diff --git a/src/observer/virtual_table/ob_all_virtual_storage_ha_error_diagnose.cpp b/src/observer/virtual_table/ob_all_virtual_storage_ha_error_diagnose.cpp new file mode 100644 index 000000000..d085dc315 --- /dev/null +++ b/src/observer/virtual_table/ob_all_virtual_storage_ha_error_diagnose.cpp @@ -0,0 +1,204 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "ob_all_virtual_storage_ha_error_diagnose.h" + +namespace oceanbase +{ +using namespace storage; +using namespace common; +using namespace share; +namespace observer +{ +ObAllVirtualStorageHAErrorDiagnose::ObAllVirtualStorageHAErrorDiagnose() + : info_(nullptr), + iter_() +{ + memset(ip_buf_, 0, sizeof(ip_buf_)); + memset(info_str_, 0, sizeof(info_str_)); + memset(task_id_, 0, sizeof(task_id_)); +} + +ObAllVirtualStorageHAErrorDiagnose::~ObAllVirtualStorageHAErrorDiagnose() +{ + reset(); +} + +int ObAllVirtualStorageHAErrorDiagnose::get_info_from_type_(ObStorageHADiagInfo *&info, ObTransferErrorDiagInfo &transfer_err_diag) +{ + int ret = OB_SUCCESS; + info = nullptr; + ObStorageHADiagTaskKey key; + if (OB_FAIL(iter_.get_cur_key(key))) { + if (ret != OB_ITER_END) { + SERVER_LOG(WARN, "failed to get cur key", K(ret), K(iter_)); + } + } else { + switch(key.module_) { + case ObStorageHADiagModule::TRANSFER_ERROR_DIAGNOSE: { + info = &transfer_err_diag; + break; + } + case ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE: + ret = OB_NOT_SUPPORTED; + SERVER_LOG(WARN, "not supported perf diagnose", K(ret), K(key.module_)); + break; + default: { + ret = OB_INVALID_ARGUMENT; + SERVER_LOG(WARN, "invalid module", K(ret), K(key.module_)); + break; + } + } + } + + return ret; +} + +int ObAllVirtualStorageHAErrorDiagnose::inner_get_next_row(ObNewRow *&row) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(execute(row))) { + SERVER_LOG(WARN, "fail to execute", K(ret)); + } + return ret; +} + +void ObAllVirtualStorageHAErrorDiagnose::release_last_tenant() +{ + memset(ip_buf_, 0, sizeof(ip_buf_)); + memset(info_str_, 0, sizeof(info_str_)); + memset(task_id_, 0, sizeof(task_id_)); + info_ = nullptr; + iter_.reset(); +} + +bool ObAllVirtualStorageHAErrorDiagnose::is_need_process(uint64_t tenant_id) +{ + if (!is_virtual_tenant_id(tenant_id) + && (is_sys_tenant(effective_tenant_id_) || tenant_id == effective_tenant_id_)) { + return true; + } + return false; +} + +int ObAllVirtualStorageHAErrorDiagnose::process_curr_tenant(common::ObNewRow *&row) +{ + int ret = OB_SUCCESS; + const int64_t col_count = output_column_ids_.count(); + ObObj *cells = cur_row_.cells_; + ObTransferErrorDiagInfo transfer_err_diag; + if (!iter_.is_opened() && OB_FAIL(iter_.open(ObStorageHADiagType::ERROR_DIAGNOSE))) { + SERVER_LOG(WARN, "Fail to open storage ha diagnose iter", K(ret), K(iter_)); + } else if (OB_ISNULL(cur_row_.cells_)) { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(ERROR, "cur row cell is NULL", K(ret)); + } else if (OB_FAIL(get_info_from_type_(info_, transfer_err_diag))) { + if (OB_ITER_END != ret) { + SERVER_LOG(WARN, "fail to get info"); + } + } else if (OB_FAIL(iter_.get_next_info(*info_))) { + if (OB_ITER_END != ret) { + STORAGE_LOG(WARN, "Fail to get next diagnose info", K(ret)); + } + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < col_count; ++i) { + uint64_t col_id = output_column_ids_.at(i); + switch (col_id) { + case TENANT_ID: + cells[i].set_int(MTL_ID()); + break; + case LS_ID: + cells[i].set_int(info_->ls_id_.id()); + break; + case MODULE: { + cells[i].set_varchar(info_->get_module_str()); + cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + break; + } + case TYPE: { + cells[i].set_varchar(info_->get_type_str()); + cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + break; + } + case TASK_ID: { + if (OB_FAIL(info_->get_task_id(task_id_, sizeof(task_id_)))) { + SERVER_LOG(WARN, "failed to get task id", K(ret)); + } + cells[i].set_varchar(task_id_); + cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + break; + } + case SVR_IP: + //svr_ip + if (ObServerConfig::get_instance().self_addr_.ip_to_string(ip_buf_, sizeof(ip_buf_))) { + cells[i].set_varchar(ip_buf_); + cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + } else { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "fail to execute ip_to_string", K(ret)); + } + break; + case SVR_PORT: + //svr_port + cells[i].set_int(ObServerConfig::get_instance().self_addr_.get_port()); + break; + case RETRY_ID: + cells[i].set_int(info_->retry_id_); + break; + case CREATE_TIME: + cells[i].set_timestamp(info_->timestamp_); + break; + case RESULT_CODE: + cells[i].set_int(info_->result_code_); + break; + case RESULT_MSG: { + cells[i].set_varchar(info_->get_transfer_error_diagnose_msg()); + cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + break; + } + case INFO: { + if (OB_FAIL(info_->get_info(info_str_, sizeof(info_str_)))) { + SERVER_LOG(WARN, "failed to get info str", K(ret)); + } + cells[i].set_lob_value(ObLongTextType, info_str_, strlen(info_str_) + 1); + cells[i].set_collation_type(ObCharset::get_default_collation( + ObCharset::get_default_charset())); + break; + } + default: + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "invalid column id", K(ret), K(col_id)); + } + } + } + + if (OB_SUCC(ret)) { + row = &cur_row_; + } + + return ret; +} + +void ObAllVirtualStorageHAErrorDiagnose::reset() +{ + omt::ObMultiTenantOperator::reset(); + ObVirtualTableScannerIterator::reset(); + memset(ip_buf_, 0, sizeof(ip_buf_)); + memset(info_str_, 0, sizeof(info_str_)); + memset(task_id_, 0, sizeof(task_id_)); + info_ = nullptr; + iter_.reset(); +} + + +} /* namespace observer */ +} /* namespace oceanbase */ diff --git a/src/observer/virtual_table/ob_all_virtual_storage_ha_error_diagnose.h b/src/observer/virtual_table/ob_all_virtual_storage_ha_error_diagnose.h new file mode 100644 index 000000000..f87d4bfa9 --- /dev/null +++ b/src/observer/virtual_table/ob_all_virtual_storage_ha_error_diagnose.h @@ -0,0 +1,71 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OB_ALL_VIRTUAL_STORAGE_HA_ERROR_DIAGNOSE_H_ +#define OB_ALL_VIRTUAL_STORAGE_HA_ERROR_DIAGNOSE_H_ +#include "share/ob_virtual_table_scanner_iterator.h" +#include "share/ob_storage_ha_diagnose_struct.h" +#include "storage/high_availability/ob_storage_ha_diagnose_mgr.h" +#include "observer/omt/ob_multi_tenant_operator.h" + +namespace oceanbase +{ +namespace observer +{ + +class ObAllVirtualStorageHAErrorDiagnose : public common::ObVirtualTableScannerIterator, + public omt::ObMultiTenantOperator +{ +public: + enum COLUMN_ID_LIST + { + TENANT_ID = common::OB_APP_MIN_COLUMN_ID, + LS_ID, + MODULE, + TYPE, + TASK_ID, + SVR_IP, + SVR_PORT, + RETRY_ID, + CREATE_TIME, + RESULT_CODE, + RESULT_MSG, + INFO, + }; + ObAllVirtualStorageHAErrorDiagnose(); + virtual ~ObAllVirtualStorageHAErrorDiagnose(); + int init(); + virtual int inner_get_next_row(common::ObNewRow *&row); + virtual void reset(); +protected: + int get_info_from_type_(ObStorageHADiagInfo *&info, ObTransferErrorDiagInfo &transfer_err_diag); + +private: + // Filter out tenants that need to be processed + virtual bool is_need_process(uint64_t tenant_id) override; + // Process tenants for the current iteration + virtual int process_curr_tenant(common::ObNewRow *&row) override; + // Release the resources of the previous tenant + virtual void release_last_tenant() override; + +private: + char ip_buf_[common::OB_IP_STR_BUFF]; + ObStorageHADiagInfo *info_; + ObStorageHADiagIterator iter_; + char info_str_[common::OB_DIAGNOSE_INFO_LENGTH]; + char task_id_[common::OB_MAX_TRACE_ID_BUFFER_SIZE]; + DISALLOW_COPY_AND_ASSIGN(ObAllVirtualStorageHAErrorDiagnose); +}; + +} /* namespace observer */ +} /* namespace oceanbase */ +#endif diff --git a/src/observer/virtual_table/ob_all_virtual_storage_ha_perf_diagnose.cpp b/src/observer/virtual_table/ob_all_virtual_storage_ha_perf_diagnose.cpp new file mode 100644 index 000000000..861527ac6 --- /dev/null +++ b/src/observer/virtual_table/ob_all_virtual_storage_ha_perf_diagnose.cpp @@ -0,0 +1,225 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "ob_all_virtual_storage_ha_perf_diagnose.h" + +namespace oceanbase +{ +using namespace storage; +using namespace common; +using namespace share; +namespace observer +{ +ObAllVirtualStorageHAPerfDiagnose::ObAllVirtualStorageHAPerfDiagnose() + : info_(nullptr), + iter_() +{ + memset(ip_buf_, 0, sizeof(ip_buf_)); + memset(info_str_, 0, sizeof(info_str_)); + memset(task_id_, 0, sizeof(task_id_)); +} + +ObAllVirtualStorageHAPerfDiagnose::~ObAllVirtualStorageHAPerfDiagnose() +{ + reset(); +} + +int ObAllVirtualStorageHAPerfDiagnose::get_info_from_type_(ObStorageHADiagInfo *&info, ObTransferPerfDiagInfo &transfer_diagnose_info) +{ + int ret = OB_SUCCESS; + info = nullptr; + ObStorageHADiagTaskKey key; + if (OB_FAIL(iter_.get_cur_key(key))) { + if (ret != OB_ITER_END) { + SERVER_LOG(WARN, "failed to get cur key", K(ret), K(iter_)); + } + } else { + switch(key.module_) { + case ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE: { + info = &transfer_diagnose_info; + break; + } + case ObStorageHADiagModule::TRANSFER_ERROR_DIAGNOSE: + ret = OB_NOT_SUPPORTED; + SERVER_LOG(WARN, "not supported perf diagnose", K(ret), K(key.module_)); + break; + default: { + ret = OB_INVALID_ARGUMENT; + SERVER_LOG(WARN, "invalid module", K(ret), K(key.module_)); + break; + } + } + } + + return ret; +} + +int ObAllVirtualStorageHAPerfDiagnose::inner_get_next_row(common::ObNewRow *&row) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(NULL == allocator_)) { + ret = OB_NOT_INIT; + SERVER_LOG(WARN, "allocator is NULL", K(ret)); + } else if (OB_FAIL(execute(row))) { + SERVER_LOG(WARN, "fail to execute", K(ret)); + } + return ret; +} + +void ObAllVirtualStorageHAPerfDiagnose::release_last_tenant() +{ + memset(ip_buf_, 0, sizeof(ip_buf_)); + memset(info_str_, 0, sizeof(info_str_)); + memset(task_id_, 0, sizeof(task_id_)); + info_ = nullptr; + iter_.reset(); +} + +bool ObAllVirtualStorageHAPerfDiagnose::is_need_process(uint64_t tenant_id) +{ + if (!is_virtual_tenant_id(tenant_id) + && (is_sys_tenant(effective_tenant_id_) || tenant_id == effective_tenant_id_)) { + return true; + } + return false; +} + +int ObAllVirtualStorageHAPerfDiagnose::process_curr_tenant(common::ObNewRow *&row) +{ + int ret = OB_SUCCESS; + const int64_t col_count = output_column_ids_.count(); + ObObj *cells = cur_row_.cells_; + ObArenaAllocator alloc; + ObTransferPerfDiagInfo transfer_diagnose_info; + ObTransferPerfDiagInfo * transfer_perf_info = nullptr; + if (OB_FAIL(transfer_diagnose_info.init(&alloc, MTL_ID()))) { + SERVER_LOG(WARN, "fail to init info", K(ret)); + } else if (!iter_.is_opened() && OB_FAIL(iter_.open(ObStorageHADiagType::PERF_DIAGNOSE))) { + SERVER_LOG(WARN, "Fail to open storage ha diagnose iter", K(ret), K(iter_)); + } else if (OB_ISNULL(cur_row_.cells_)) { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(ERROR, "cur row cell is NULL", K(ret)); + } else if (OB_FAIL(get_info_from_type_(info_, transfer_diagnose_info))) { + if (OB_ITER_END != ret) { + SERVER_LOG(WARN, "fail to get info"); + } + } else if (OB_FAIL(iter_.get_next_info(*info_))) { + if (OB_ITER_END != ret) { + STORAGE_LOG(WARN, "Fail to get next diagnose info", K(ret)); + } + } else if (ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE != info_->module_) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected info module", K(ret), K(info_->module_)); + } else { + transfer_perf_info = static_cast(info_); + for (int64_t i = 0; OB_SUCC(ret) && i < col_count; ++i) { + uint64_t col_id = output_column_ids_.at(i); + switch (col_id) { + case TENANT_ID: + cells[i].set_int(MTL_ID()); + break; + case LS_ID: + cells[i].set_int(info_->ls_id_.id()); + break; + case MODULE: { + cells[i].set_varchar(info_->get_module_str()); + cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + break; + } + case TYPE: { + cells[i].set_varchar(info_->get_type_str()); + cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + break; + } + case TASK_ID: { + if (OB_FAIL(info_->get_task_id(task_id_, sizeof(task_id_)))) { + SERVER_LOG(WARN, "failed to get task id", K(ret)); + } + cells[i].set_varchar(task_id_); + cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + break; + } + case SVR_IP: + //svr_ip + if (ObServerConfig::get_instance().self_addr_.ip_to_string(ip_buf_, sizeof(ip_buf_))) { + cells[i].set_varchar(ip_buf_); + cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + } else { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "fail to execute ip_to_string", K(ret)); + } + break; + case SVR_PORT: + //svr_port + cells[i].set_int(ObServerConfig::get_instance().self_addr_.get_port()); + break; + case RETRY_ID: + cells[i].set_int(info_->retry_id_); + break; + case START_TIMESTAMP: + cells[i].set_timestamp(info_->timestamp_); + break; + case END_TIMESTAMP: { + cells[i].set_timestamp(transfer_perf_info->end_timestamp_); + break; + } + case TABLET_ID: { + cells[i].set_int(info_->tablet_id_.id()); + break; + } + case TABLET_COUNT: { + cells[i].set_int(transfer_perf_info->tablet_count_); + break; + } + case RESULT_CODE: + cells[i].set_int(info_->result_code_); + break; + case RESULT_MSG: { + cells[i].set_varchar(info_->get_transfer_error_diagnose_msg()); + cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + break; + } + case INFO: { + if (OB_FAIL(info_->get_info(info_str_, sizeof(info_str_)))) { + SERVER_LOG(WARN, "failed to get info str", K(ret)); + } + cells[i].set_lob_value(ObLongTextType, info_str_, strlen(info_str_) + 1); + cells[i].set_collation_type(ObCharset::get_default_collation( + ObCharset::get_default_charset())); + break; + } + default: + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "invalid column id", K(ret), K(col_id)); + } + } + } + if (OB_SUCC(ret)) { + row = &cur_row_; + } + + return ret; +} + +void ObAllVirtualStorageHAPerfDiagnose::reset() +{ + omt::ObMultiTenantOperator::reset(); + ObVirtualTableScannerIterator::reset(); + memset(ip_buf_, 0, sizeof(ip_buf_)); + memset(info_str_, 0, sizeof(info_str_)); + memset(task_id_, 0, sizeof(task_id_)); + info_ = nullptr; + iter_.reset(); +} + +} /* namespace observer */ +} /* namespace oceanbase */ diff --git a/src/observer/virtual_table/ob_all_virtual_storage_ha_perf_diagnose.h b/src/observer/virtual_table/ob_all_virtual_storage_ha_perf_diagnose.h new file mode 100644 index 000000000..e9cc07aef --- /dev/null +++ b/src/observer/virtual_table/ob_all_virtual_storage_ha_perf_diagnose.h @@ -0,0 +1,74 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OB_ALL_VIRTUAL_STORAGE_HA_PERF_DIAGNOSE_H_ +#define OB_ALL_VIRTUAL_STORAGE_HA_PERF_DIAGNOSE_H_ +#include "share/ob_virtual_table_scanner_iterator.h" +#include "share/ob_storage_ha_diagnose_struct.h" +#include "storage/high_availability/ob_storage_ha_diagnose_mgr.h" +#include "observer/omt/ob_multi_tenant_operator.h" + +namespace oceanbase +{ +namespace observer +{ + +class ObAllVirtualStorageHAPerfDiagnose : public common::ObVirtualTableScannerIterator, + public omt::ObMultiTenantOperator +{ +public: + enum COLUMN_ID_LIST + { + TENANT_ID = common::OB_APP_MIN_COLUMN_ID, + LS_ID, + MODULE, + TYPE, + TASK_ID, + SVR_IP, + SVR_PORT, + RETRY_ID, + START_TIMESTAMP, + END_TIMESTAMP, + TABLET_ID, + TABLET_COUNT, + RESULT_CODE, + RESULT_MSG, + INFO, + }; + ObAllVirtualStorageHAPerfDiagnose(); + virtual ~ObAllVirtualStorageHAPerfDiagnose(); + virtual int inner_get_next_row(common::ObNewRow *&row); + virtual void reset(); +protected: + int get_info_from_type_(ObStorageHADiagInfo *&info, ObTransferPerfDiagInfo &transfer_diagnose_info); + +private: + // 过滤得到需要处理的租户 + virtual bool is_need_process(uint64_t tenant_id) override; + // 处理当前迭代的租户 + virtual int process_curr_tenant(common::ObNewRow *&row) override; + // 释放上一个租户的资源 + virtual void release_last_tenant() override; + static constexpr int64_t OB_STORAGE_HA_DIAGNOSE_INFO_LENGTH = (1 << 12);// 4k + +private: + char ip_buf_[common::OB_IP_STR_BUFF]; + ObStorageHADiagInfo *info_; + ObStorageHADiagIterator iter_; + char info_str_[OB_STORAGE_HA_DIAGNOSE_INFO_LENGTH]; + char task_id_[common::OB_MAX_TRACE_ID_BUFFER_SIZE]; + DISALLOW_COPY_AND_ASSIGN(ObAllVirtualStorageHAPerfDiagnose); +}; + +} /* namespace observer */ +} /* namespace oceanbase */ +#endif diff --git a/src/observer/virtual_table/ob_virtual_table_iterator_factory.cpp b/src/observer/virtual_table/ob_virtual_table_iterator_factory.cpp index 57cddb637..d23fe3da9 100644 --- a/src/observer/virtual_table/ob_virtual_table_iterator_factory.cpp +++ b/src/observer/virtual_table/ob_virtual_table_iterator_factory.cpp @@ -219,7 +219,9 @@ #include "observer/virtual_table/ob_all_virtual_activity_metrics.h" #include "observer/virtual_table/ob_all_virtual_checkpoint_diagnose_info.h" #include "observer/virtual_table/ob_all_virtual_checkpoint_diagnose_memtable_info.h" - +#include "observer/virtual_table/ob_all_virtual_storage_ha_error_diagnose.h" +#include "observer/virtual_table/ob_all_virtual_storage_ha_perf_diagnose.h" +#include "observer/virtual_table/ob_tenant_show_restore_preview.h" #include "observer/virtual_table/ob_all_virtual_kv_connection.h" #include "observer/virtual_table/ob_tenant_show_restore_preview.h" #include "observer/virtual_table/ob_all_virtual_nic_info.h" @@ -2658,6 +2660,24 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, } break; } + case OB_ALL_VIRTUAL_STORAGE_HA_ERROR_DIAGNOSE_TID: { + ObAllVirtualStorageHAErrorDiagnose *storage_ha_error_diagnose = NULL; + if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualStorageHAErrorDiagnose, storage_ha_error_diagnose))) { + SERVER_LOG(ERROR, "ObAllVirtualStorageHAErrorDiagnose construct failed", K(ret)); + } else { + vt_iter = static_cast(storage_ha_error_diagnose); + } + break; + } + case OB_ALL_VIRTUAL_STORAGE_HA_PERF_DIAGNOSE_TID: { + ObAllVirtualStorageHAPerfDiagnose *storage_ha_perf_diagnose = NULL; + if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualStorageHAPerfDiagnose, storage_ha_perf_diagnose))) { + SERVER_LOG(ERROR, "ObAllVirtualStorageHAPerfDiagnose construct failed", K(ret)); + } else { + vt_iter = static_cast(storage_ha_perf_diagnose); + } + break; + } END_CREATE_VT_ITER_SWITCH_LAMBDA #define AGENT_VIRTUAL_TABLE_CREATE_ITER diff --git a/src/share/CMakeLists.txt b/src/share/CMakeLists.txt index be0355a96..96cebd0b7 100644 --- a/src/share/CMakeLists.txt +++ b/src/share/CMakeLists.txt @@ -184,6 +184,8 @@ ob_set_subtarget(ob_share common ob_cluster_event_history_table_operator.cpp scn.cpp ob_throttling_utils.cpp + ob_storage_ha_diagnose_struct.cpp + ob_storage_ha_diagnose_operator.cpp ) ob_set_subtarget(ob_share common_mixed diff --git a/src/share/inner_table/ob_inner_table_schema.12401_12450.cpp b/src/share/inner_table/ob_inner_table_schema.12401_12450.cpp index 176e266aa..901cfa8c2 100644 --- a/src/share/inner_table/ob_inner_table_schema.12401_12450.cpp +++ b/src/share/inner_table/ob_inner_table_schema.12401_12450.cpp @@ -9593,6 +9593,522 @@ int ObInnerTableSchema::all_virtual_column_group_mapping_history_schema(ObTableS return ret; } +int ObInnerTableSchema::all_virtual_storage_ha_error_diagnose_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_STORAGE_HA_ERROR_DIAGNOSE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(VIRTUAL_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_STORAGE_HA_ERROR_DIAGNOSE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ls_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("module", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MODULE_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("type", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_SYS_TASK_TYPE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_TRACE_ID_BUFFER_SIZE, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("svr_ip", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 1, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_IP_ADDR_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("svr_port", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 2, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("retry_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("create_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("result_code", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("result_msg", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_DIAGNOSE_INFO_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("info", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObLongTextType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + if (OB_SUCC(ret)) { + table_schema.get_part_option().set_part_num(1); + table_schema.set_part_level(PARTITION_LEVEL_ONE); + table_schema.get_part_option().set_part_func_type(PARTITION_FUNC_TYPE_LIST_COLUMNS); + if (OB_FAIL(table_schema.get_part_option().set_part_expr("svr_ip, svr_port"))) { + LOG_WARN("set_part_expr failed", K(ret)); + } else if (OB_FAIL(table_schema.mock_list_partition_array())) { + LOG_WARN("mock list partition array failed", K(ret)); + } + } + table_schema.set_index_using_type(USING_HASH); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::all_virtual_storage_ha_perf_diagnose_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_STORAGE_HA_PERF_DIAGNOSE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(VIRTUAL_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_STORAGE_HA_PERF_DIAGNOSE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ls_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("module", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MODULE_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("type", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_SYS_TASK_TYPE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_TRACE_ID_BUFFER_SIZE, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("svr_ip", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 1, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_IP_ADDR_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("svr_port", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 2, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("retry_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("start_timestamp", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("end_timestamp", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tablet_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tablet_count", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("result_code", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("result_msg", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_DIAGNOSE_INFO_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("info", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObLongTextType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + if (OB_SUCC(ret)) { + table_schema.get_part_option().set_part_num(1); + table_schema.set_part_level(PARTITION_LEVEL_ONE); + table_schema.get_part_option().set_part_func_type(PARTITION_FUNC_TYPE_LIST_COLUMNS); + if (OB_FAIL(table_schema.get_part_option().set_part_expr("svr_ip, svr_port"))) { + LOG_WARN("set_part_expr failed", K(ret)); + } else if (OB_FAIL(table_schema.mock_list_partition_array())) { + LOG_WARN("mock list partition array failed", K(ret)); + } + } + table_schema.set_index_using_type(USING_HASH); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::all_virtual_clone_job_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.451_500.cpp b/src/share/inner_table/ob_inner_table_schema.451_500.cpp index 32c1c5dd7..95e06c141 100644 --- a/src/share/inner_table/ob_inner_table_schema.451_500.cpp +++ b/src/share/inner_table/ob_inner_table_schema.451_500.cpp @@ -9947,6 +9947,538 @@ int ObInnerTableSchema::all_import_table_task_history_schema(ObTableSchema &tabl return ret; } +int ObInnerTableSchema::all_storage_ha_error_diagnose_history_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("gmt_create", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("svr_ip", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_IP_ADDR_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("svr_port", //column_name + ++column_id, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ls_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("module", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MODULE_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("type", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_SYS_TASK_TYPE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_TRACE_ID_BUFFER_SIZE, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("retry_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("create_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("result_code", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("result_msg", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_DIAGNOSE_INFO_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("info", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObLongTextType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_TID); + table_schema.set_aux_lob_meta_tid(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_META_TID); + table_schema.set_aux_lob_piece_tid(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::all_storage_ha_perf_diagnose_history_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("gmt_create", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("svr_ip", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_IP_ADDR_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("svr_port", //column_name + ++column_id, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ls_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("module", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MODULE_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("type", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_SYS_TASK_TYPE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_TRACE_ID_BUFFER_SIZE, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("retry_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("start_timestamp", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("end_timestamp", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tablet_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tablet_count", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("result_code", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("result_msg", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_DIAGNOSE_INFO_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("info", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObLongTextType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_TID); + table_schema.set_aux_lob_meta_tid(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_META_TID); + table_schema.set_aux_lob_piece_tid(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::all_clone_job_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.50451_50500.cpp b/src/share/inner_table/ob_inner_table_schema.50451_50500.cpp index 633bf294f..46fdcb979 100644 --- a/src/share/inner_table/ob_inner_table_schema.50451_50500.cpp +++ b/src/share/inner_table/ob_inner_table_schema.50451_50500.cpp @@ -3940,6 +3940,276 @@ int ObInnerTableSchema::all_import_table_task_history_aux_lob_meta_schema(ObTabl return ret; } +int ObInnerTableSchema::all_storage_ha_error_diagnose_history_aux_lob_meta_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_META_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(AUX_LOB_META); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_META_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 16, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("seq_id", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 8192, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("binary_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("char_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("piece_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_data", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 262144, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_META_TID); + table_schema.set_data_table_id(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::all_storage_ha_perf_diagnose_history_aux_lob_meta_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_META_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(AUX_LOB_META); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_META_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 16, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("seq_id", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 8192, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("binary_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("char_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("piece_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_data", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 262144, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_META_TID); + table_schema.set_data_table_id(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::all_clone_job_aux_lob_meta_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.60451_60500.cpp b/src/share/inner_table/ob_inner_table_schema.60451_60500.cpp index a9b082a25..a198c02ef 100644 --- a/src/share/inner_table/ob_inner_table_schema.60451_60500.cpp +++ b/src/share/inner_table/ob_inner_table_schema.60451_60500.cpp @@ -2635,6 +2635,186 @@ int ObInnerTableSchema::all_import_table_task_history_aux_lob_piece_schema(ObTab return ret; } +int ObInnerTableSchema::all_storage_ha_error_diagnose_history_aux_lob_piece_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(AUX_LOB_PIECE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("piece_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("data_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_data", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 32, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TID); + table_schema.set_data_table_id(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::all_storage_ha_perf_diagnose_history_aux_lob_piece_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(AUX_LOB_PIECE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("piece_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("data_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_data", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 32, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TID); + table_schema.set_data_table_id(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::all_clone_job_aux_lob_piece_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.h b/src/share/inner_table/ob_inner_table_schema.h index 39b45750a..b7f1c26ca 100644 --- a/src/share/inner_table/ob_inner_table_schema.h +++ b/src/share/inner_table/ob_inner_table_schema.h @@ -567,6 +567,8 @@ public: static int all_import_table_job_history_schema(share::schema::ObTableSchema &table_schema); static int all_import_table_task_schema(share::schema::ObTableSchema &table_schema); static int all_import_table_task_history_schema(share::schema::ObTableSchema &table_schema); + static int all_storage_ha_error_diagnose_history_schema(share::schema::ObTableSchema &table_schema); + static int all_storage_ha_perf_diagnose_history_schema(share::schema::ObTableSchema &table_schema); static int all_clone_job_schema(share::schema::ObTableSchema &table_schema); static int all_clone_job_history_schema(share::schema::ObTableSchema &table_schema); static int all_ncomp_dll_schema(share::schema::ObTableSchema &table_schema); @@ -996,6 +998,8 @@ public: static int all_virtual_column_group_mapping_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_column_group_history_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_column_group_mapping_history_schema(share::schema::ObTableSchema &table_schema); + static int all_virtual_storage_ha_error_diagnose_schema(share::schema::ObTableSchema &table_schema); + static int all_virtual_storage_ha_perf_diagnose_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_clone_job_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_clone_job_history_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_checkpoint_diagnose_memtable_info_schema(share::schema::ObTableSchema &table_schema); @@ -2401,6 +2405,8 @@ public: static int all_import_table_job_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_import_table_task_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_import_table_task_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); + static int all_storage_ha_error_diagnose_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); + static int all_storage_ha_perf_diagnose_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_clone_job_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_clone_job_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_ncomp_dll_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); @@ -2686,6 +2692,8 @@ public: static int all_import_table_job_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_import_table_task_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_import_table_task_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); + static int all_storage_ha_error_diagnose_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); + static int all_storage_ha_perf_diagnose_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_clone_job_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_clone_job_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_ncomp_dll_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); @@ -3180,6 +3188,8 @@ const schema_create_func sys_table_schema_creators [] = { ObInnerTableSchema::all_import_table_job_history_schema, ObInnerTableSchema::all_import_table_task_schema, ObInnerTableSchema::all_import_table_task_history_schema, + ObInnerTableSchema::all_storage_ha_error_diagnose_history_schema, + ObInnerTableSchema::all_storage_ha_perf_diagnose_history_schema, ObInnerTableSchema::all_clone_job_schema, ObInnerTableSchema::all_clone_job_history_schema, ObInnerTableSchema::all_ncomp_dll_schema, @@ -3612,6 +3622,8 @@ const schema_create_func virtual_table_schema_creators [] = { ObInnerTableSchema::all_virtual_column_group_mapping_schema, ObInnerTableSchema::all_virtual_column_group_history_schema, ObInnerTableSchema::all_virtual_column_group_mapping_history_schema, + ObInnerTableSchema::all_virtual_storage_ha_error_diagnose_schema, + ObInnerTableSchema::all_virtual_storage_ha_perf_diagnose_schema, ObInnerTableSchema::all_virtual_clone_job_schema, ObInnerTableSchema::all_virtual_clone_job_history_schema, ObInnerTableSchema::all_virtual_checkpoint_diagnose_memtable_info_schema, @@ -9683,7 +9695,9 @@ const uint64_t cluster_distributed_vtables [] = { OB_ALL_VIRTUAL_HA_DIAGNOSE_TID, OB_ALL_VIRTUAL_IO_SCHEDULER_TID, OB_ALL_VIRTUAL_TX_DATA_TID, - OB_ALL_VIRTUAL_STORAGE_LEAK_INFO_TID, }; + OB_ALL_VIRTUAL_STORAGE_LEAK_INFO_TID, + OB_ALL_VIRTUAL_STORAGE_HA_ERROR_DIAGNOSE_TID, + OB_ALL_VIRTUAL_STORAGE_HA_PERF_DIAGNOSE_TID, }; const uint64_t tenant_distributed_vtables [] = { OB_ALL_VIRTUAL_PROCESSLIST_TID, @@ -12318,6 +12332,22 @@ LOBMapping const lob_aux_table_mappings [] = { ObInnerTableSchema::all_import_table_task_history_aux_lob_piece_schema }, + { + OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_TID, + OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_META_TID, + OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TID, + ObInnerTableSchema::all_storage_ha_error_diagnose_history_aux_lob_meta_schema, + ObInnerTableSchema::all_storage_ha_error_diagnose_history_aux_lob_piece_schema + }, + + { + OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_TID, + OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_META_TID, + OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TID, + ObInnerTableSchema::all_storage_ha_perf_diagnose_history_aux_lob_meta_schema, + ObInnerTableSchema::all_storage_ha_perf_diagnose_history_aux_lob_piece_schema + }, + { OB_ALL_CLONE_JOB_TID, OB_ALL_CLONE_JOB_AUX_LOB_META_TID, @@ -12435,12 +12465,12 @@ static inline int get_sys_table_lob_aux_schema(const uint64_t tid, } const int64_t OB_CORE_TABLE_COUNT = 4; -const int64_t OB_SYS_TABLE_COUNT = 282; -const int64_t OB_VIRTUAL_TABLE_COUNT = 786; +const int64_t OB_SYS_TABLE_COUNT = 284; +const int64_t OB_VIRTUAL_TABLE_COUNT = 788; const int64_t OB_SYS_VIEW_COUNT = 854; -const int64_t OB_SYS_TENANT_TABLE_COUNT = 1927; +const int64_t OB_SYS_TENANT_TABLE_COUNT = 1931; const int64_t OB_CORE_SCHEMA_VERSION = 1; -const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 1930; +const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 1934; } // end namespace share } // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.lob.cpp b/src/share/inner_table/ob_inner_table_schema.lob.cpp index 9ceb4f1b7..2b18ac031 100644 --- a/src/share/inner_table/ob_inner_table_schema.lob.cpp +++ b/src/share/inner_table/ob_inner_table_schema.lob.cpp @@ -21,7 +21,7 @@ inner_lob_map_t inner_lob_map; bool lob_mapping_init() { int ret = OB_SUCCESS; - if (OB_FAIL(inner_lob_map.create(285, ObModIds::OB_INNER_LOB_HASH_SET))) { + if (OB_FAIL(inner_lob_map.create(287, ObModIds::OB_INNER_LOB_HASH_SET))) { SERVER_LOG(WARN, "fail to create inner lob map", K(ret)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < ARRAYSIZEOF(lob_aux_table_mappings); ++i) { diff --git a/src/share/inner_table/ob_inner_table_schema_constants.h b/src/share/inner_table/ob_inner_table_schema_constants.h index 6ab79661f..283fa4f54 100644 --- a/src/share/inner_table/ob_inner_table_schema_constants.h +++ b/src/share/inner_table/ob_inner_table_schema_constants.h @@ -303,6 +303,8 @@ const uint64_t OB_ALL_IMPORT_TABLE_JOB_TID = 477; // "__all_import_table_job" const uint64_t OB_ALL_IMPORT_TABLE_JOB_HISTORY_TID = 478; // "__all_import_table_job_history" const uint64_t OB_ALL_IMPORT_TABLE_TASK_TID = 479; // "__all_import_table_task" const uint64_t OB_ALL_IMPORT_TABLE_TASK_HISTORY_TID = 480; // "__all_import_table_task_history" +const uint64_t OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_TID = 483; // "__all_storage_ha_error_diagnose_history" +const uint64_t OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_TID = 484; // "__all_storage_ha_perf_diagnose_history" const uint64_t OB_ALL_CLONE_JOB_TID = 485; // "__all_clone_job" const uint64_t OB_ALL_CLONE_JOB_HISTORY_TID = 486; // "__all_clone_job_history" const uint64_t OB_ALL_NCOMP_DLL_TID = 493; // "__all_ncomp_dll" @@ -732,6 +734,8 @@ const uint64_t OB_ALL_VIRTUAL_DATA_ACTIVITY_METRICS_TID = 12429; // "__all_virtu const uint64_t OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_TID = 12430; // "__all_virtual_column_group_mapping" const uint64_t OB_ALL_VIRTUAL_COLUMN_GROUP_HISTORY_TID = 12431; // "__all_virtual_column_group_history" const uint64_t OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_HISTORY_TID = 12432; // "__all_virtual_column_group_mapping_history" +const uint64_t OB_ALL_VIRTUAL_STORAGE_HA_ERROR_DIAGNOSE_TID = 12433; // "__all_virtual_storage_ha_error_diagnose" +const uint64_t OB_ALL_VIRTUAL_STORAGE_HA_PERF_DIAGNOSE_TID = 12434; // "__all_virtual_storage_ha_perf_diagnose" const uint64_t OB_ALL_VIRTUAL_CLONE_JOB_TID = 12435; // "__all_virtual_clone_job" const uint64_t OB_ALL_VIRTUAL_CLONE_JOB_HISTORY_TID = 12436; // "__all_virtual_clone_job_history" const uint64_t OB_ALL_VIRTUAL_CHECKPOINT_DIAGNOSE_MEMTABLE_INFO_TID = 12437; // "__all_virtual_checkpoint_diagnose_memtable_info" @@ -2137,6 +2141,8 @@ const uint64_t OB_ALL_IMPORT_TABLE_JOB_AUX_LOB_META_TID = 50477; // "__all_impor const uint64_t OB_ALL_IMPORT_TABLE_JOB_HISTORY_AUX_LOB_META_TID = 50478; // "__all_import_table_job_history_aux_lob_meta" const uint64_t OB_ALL_IMPORT_TABLE_TASK_AUX_LOB_META_TID = 50479; // "__all_import_table_task_aux_lob_meta" const uint64_t OB_ALL_IMPORT_TABLE_TASK_HISTORY_AUX_LOB_META_TID = 50480; // "__all_import_table_task_history_aux_lob_meta" +const uint64_t OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_META_TID = 50483; // "__all_storage_ha_error_diagnose_history_aux_lob_meta" +const uint64_t OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_META_TID = 50484; // "__all_storage_ha_perf_diagnose_history_aux_lob_meta" const uint64_t OB_ALL_CLONE_JOB_AUX_LOB_META_TID = 50485; // "__all_clone_job_aux_lob_meta" const uint64_t OB_ALL_CLONE_JOB_HISTORY_AUX_LOB_META_TID = 50486; // "__all_clone_job_history_aux_lob_meta" const uint64_t OB_ALL_NCOMP_DLL_AUX_LOB_META_TID = 50493; // "__all_ncomp_dll_aux_lob_meta" @@ -2422,6 +2428,8 @@ const uint64_t OB_ALL_IMPORT_TABLE_JOB_AUX_LOB_PIECE_TID = 60477; // "__all_impo const uint64_t OB_ALL_IMPORT_TABLE_JOB_HISTORY_AUX_LOB_PIECE_TID = 60478; // "__all_import_table_job_history_aux_lob_piece" const uint64_t OB_ALL_IMPORT_TABLE_TASK_AUX_LOB_PIECE_TID = 60479; // "__all_import_table_task_aux_lob_piece" const uint64_t OB_ALL_IMPORT_TABLE_TASK_HISTORY_AUX_LOB_PIECE_TID = 60480; // "__all_import_table_task_history_aux_lob_piece" +const uint64_t OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TID = 60483; // "__all_storage_ha_error_diagnose_history_aux_lob_piece" +const uint64_t OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TID = 60484; // "__all_storage_ha_perf_diagnose_history_aux_lob_piece" const uint64_t OB_ALL_CLONE_JOB_AUX_LOB_PIECE_TID = 60485; // "__all_clone_job_aux_lob_piece" const uint64_t OB_ALL_CLONE_JOB_HISTORY_AUX_LOB_PIECE_TID = 60486; // "__all_clone_job_history_aux_lob_piece" const uint64_t OB_ALL_NCOMP_DLL_AUX_LOB_PIECE_TID = 60493; // "__all_ncomp_dll_aux_lob_piece" @@ -2903,6 +2911,8 @@ const char *const OB_ALL_IMPORT_TABLE_JOB_TNAME = "__all_import_table_job"; const char *const OB_ALL_IMPORT_TABLE_JOB_HISTORY_TNAME = "__all_import_table_job_history"; const char *const OB_ALL_IMPORT_TABLE_TASK_TNAME = "__all_import_table_task"; const char *const OB_ALL_IMPORT_TABLE_TASK_HISTORY_TNAME = "__all_import_table_task_history"; +const char *const OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_TNAME = "__all_storage_ha_error_diagnose_history"; +const char *const OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_TNAME = "__all_storage_ha_perf_diagnose_history"; const char *const OB_ALL_CLONE_JOB_TNAME = "__all_clone_job"; const char *const OB_ALL_CLONE_JOB_HISTORY_TNAME = "__all_clone_job_history"; const char *const OB_ALL_NCOMP_DLL_TNAME = "__all_ncomp_dll"; @@ -3332,6 +3342,8 @@ const char *const OB_ALL_VIRTUAL_DATA_ACTIVITY_METRICS_TNAME = "__all_virtual_da const char *const OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_TNAME = "__all_virtual_column_group_mapping"; const char *const OB_ALL_VIRTUAL_COLUMN_GROUP_HISTORY_TNAME = "__all_virtual_column_group_history"; const char *const OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_HISTORY_TNAME = "__all_virtual_column_group_mapping_history"; +const char *const OB_ALL_VIRTUAL_STORAGE_HA_ERROR_DIAGNOSE_TNAME = "__all_virtual_storage_ha_error_diagnose"; +const char *const OB_ALL_VIRTUAL_STORAGE_HA_PERF_DIAGNOSE_TNAME = "__all_virtual_storage_ha_perf_diagnose"; const char *const OB_ALL_VIRTUAL_CLONE_JOB_TNAME = "__all_virtual_clone_job"; const char *const OB_ALL_VIRTUAL_CLONE_JOB_HISTORY_TNAME = "__all_virtual_clone_job_history"; const char *const OB_ALL_VIRTUAL_CHECKPOINT_DIAGNOSE_MEMTABLE_INFO_TNAME = "__all_virtual_checkpoint_diagnose_memtable_info"; @@ -4737,6 +4749,8 @@ const char *const OB_ALL_IMPORT_TABLE_JOB_AUX_LOB_META_TNAME = "__all_import_tab const char *const OB_ALL_IMPORT_TABLE_JOB_HISTORY_AUX_LOB_META_TNAME = "__all_import_table_job_history_aux_lob_meta"; const char *const OB_ALL_IMPORT_TABLE_TASK_AUX_LOB_META_TNAME = "__all_import_table_task_aux_lob_meta"; const char *const OB_ALL_IMPORT_TABLE_TASK_HISTORY_AUX_LOB_META_TNAME = "__all_import_table_task_history_aux_lob_meta"; +const char *const OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_META_TNAME = "__all_storage_ha_error_diagnose_history_aux_lob_meta"; +const char *const OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_META_TNAME = "__all_storage_ha_perf_diagnose_history_aux_lob_meta"; const char *const OB_ALL_CLONE_JOB_AUX_LOB_META_TNAME = "__all_clone_job_aux_lob_meta"; const char *const OB_ALL_CLONE_JOB_HISTORY_AUX_LOB_META_TNAME = "__all_clone_job_history_aux_lob_meta"; const char *const OB_ALL_NCOMP_DLL_AUX_LOB_META_TNAME = "__all_ncomp_dll_aux_lob_meta"; @@ -5022,6 +5036,8 @@ const char *const OB_ALL_IMPORT_TABLE_JOB_AUX_LOB_PIECE_TNAME = "__all_import_ta const char *const OB_ALL_IMPORT_TABLE_JOB_HISTORY_AUX_LOB_PIECE_TNAME = "__all_import_table_job_history_aux_lob_piece"; const char *const OB_ALL_IMPORT_TABLE_TASK_AUX_LOB_PIECE_TNAME = "__all_import_table_task_aux_lob_piece"; const char *const OB_ALL_IMPORT_TABLE_TASK_HISTORY_AUX_LOB_PIECE_TNAME = "__all_import_table_task_history_aux_lob_piece"; +const char *const OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TNAME = "__all_storage_ha_error_diagnose_history_aux_lob_piece"; +const char *const OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TNAME = "__all_storage_ha_perf_diagnose_history_aux_lob_piece"; const char *const OB_ALL_CLONE_JOB_AUX_LOB_PIECE_TNAME = "__all_clone_job_aux_lob_piece"; const char *const OB_ALL_CLONE_JOB_HISTORY_AUX_LOB_PIECE_TNAME = "__all_clone_job_history_aux_lob_piece"; const char *const OB_ALL_NCOMP_DLL_AUX_LOB_PIECE_TNAME = "__all_ncomp_dll_aux_lob_piece"; diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index 456c50253..af9221acd 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -6725,8 +6725,60 @@ def_table_schema( # 481 : __all_import_stmt_exec_history # 482 : __all_tablet_reorganize_history -# 483 : __all_storage_ha_error_diagnose_history -# 484 : __all_storage_ha_perf_diagnose_history +def_table_schema( + owner = 'zhixing.yh', + table_name = '__all_storage_ha_error_diagnose_history', + table_id = '483', + table_type = 'SYSTEM_TABLE', + in_tenant_space = False, + gm_columns = [], + rowkey_columns = [ + ('gmt_create', 'timestamp', 'false'), + ('svr_ip', 'varchar:MAX_IP_ADDR_LENGTH'), + ('svr_port', 'int') + ], + normal_columns = [ + ('tenant_id', 'int'), + ('ls_id', 'int'), + ('module', 'varchar:OB_MODULE_NAME_LENGTH'), + ('type', 'varchar:OB_SYS_TASK_TYPE_LENGTH'), + ('task_id', 'varchar:OB_MAX_TRACE_ID_BUFFER_SIZE'), + ('retry_id', 'int'), + ('create_time', 'timestamp'), + ('result_code', 'int'), + ('result_msg', 'varchar:OB_DIAGNOSE_INFO_LENGTH'), + ('info', 'longtext') + ], +) + +def_table_schema( + owner = 'zhixing.yh', + table_name = '__all_storage_ha_perf_diagnose_history', + table_id = '484', + table_type = 'SYSTEM_TABLE', + in_tenant_space = False, + gm_columns = [], + rowkey_columns = [ + ('gmt_create', 'timestamp', 'false'), + ('svr_ip', 'varchar:MAX_IP_ADDR_LENGTH'), + ('svr_port', 'int') + ], + normal_columns = [ + ('tenant_id', 'int'), + ('ls_id', 'int'), + ('module', 'varchar:OB_MODULE_NAME_LENGTH'), + ('type', 'varchar:OB_SYS_TASK_TYPE_LENGTH'), + ('task_id', 'varchar:OB_MAX_TRACE_ID_BUFFER_SIZE'), + ('retry_id', 'int'), + ('start_timestamp', 'timestamp'), + ('end_timestamp', 'timestamp'), + ('tablet_id', 'int'), + ('tablet_count', 'int'), + ('result_code', 'int'), + ('result_msg', 'varchar:OB_DIAGNOSE_INFO_LENGTH'), + ('info', 'longtext'), + ], +) def_table_schema( owner = 'chensen.cs', @@ -13737,8 +13789,60 @@ def_table_schema(**gen_iterate_virtual_table_def( keywords = all_def_keywords['__all_column_group_mapping_history'], in_tenant_space = True)) -# 12433: __all_virtual_storage_ha_error_diagnose -# 12434: __all_virtual_storage_ha_perf_diagnose +def_table_schema( + owner = 'zhixing.yh', + table_name = '__all_virtual_storage_ha_error_diagnose', + table_id = '12433', + table_type = 'VIRTUAL_TABLE', + in_tenant_space = False, + gm_columns = [], + rowkey_columns = [], + normal_columns = [ + ('tenant_id', 'int'), + ('ls_id', 'int'), + ('module', 'varchar:OB_MODULE_NAME_LENGTH'), + ('type', 'varchar:OB_SYS_TASK_TYPE_LENGTH'), + ('task_id', 'varchar:OB_MAX_TRACE_ID_BUFFER_SIZE'), + ('svr_ip', 'varchar:MAX_IP_ADDR_LENGTH'), + ('svr_port', 'int'), + ('retry_id', 'int'), + ('create_time', 'timestamp'), + ('result_code', 'int'), + ('result_msg', 'varchar:OB_DIAGNOSE_INFO_LENGTH'), + ('info', 'longtext') + ], + partition_columns = ['svr_ip', 'svr_port'], + vtable_route_policy = 'distributed', +) + +def_table_schema( + owner = 'zhixing.yh', + table_name = '__all_virtual_storage_ha_perf_diagnose', + table_id = '12434', + table_type = 'VIRTUAL_TABLE', + in_tenant_space = False, + gm_columns = [], + rowkey_columns = [], + normal_columns = [ + ('tenant_id', 'int'), + ('ls_id', 'int'), + ('module', 'varchar:OB_MODULE_NAME_LENGTH'), + ('type', 'varchar:OB_SYS_TASK_TYPE_LENGTH'), + ('task_id', 'varchar:OB_MAX_TRACE_ID_BUFFER_SIZE'), + ('svr_ip', 'varchar:MAX_IP_ADDR_LENGTH'), + ('svr_port', 'int'), + ('retry_id', 'int'), + ('start_timestamp', 'timestamp'), + ('end_timestamp', 'timestamp'), + ('tablet_id', 'int'), + ('tablet_count', 'int'), + ('result_code', 'int'), + ('result_msg', 'varchar:OB_DIAGNOSE_INFO_LENGTH'), + ('info', 'longtext'), + ], + partition_columns = ['svr_ip', 'svr_port'], + vtable_route_policy = 'distributed', +) def_table_schema(**gen_iterate_private_virtual_table_def( table_id = '12435', diff --git a/src/share/inner_table/table_id_to_name b/src/share/inner_table/table_id_to_name index e130dbc61..4b86d4283 100644 --- a/src/share/inner_table/table_id_to_name +++ b/src/share/inner_table/table_id_to_name @@ -329,6 +329,8 @@ # 478: __all_import_table_job_history # 479: __all_import_table_task # 480: __all_import_table_task_history +# 483: __all_storage_ha_error_diagnose_history +# 484: __all_storage_ha_perf_diagnose_history # 485: __all_clone_job # 486: __all_clone_job_history # 493: __all_ncomp_dll @@ -1049,6 +1051,8 @@ # 12432: __all_virtual_column_group_mapping_history # 12432: __all_column_group_mapping # BASE_TABLE_NAME # 12432: __all_column_group_mapping_history # BASE_TABLE_NAME1 +# 12433: __all_virtual_storage_ha_error_diagnose +# 12434: __all_virtual_storage_ha_perf_diagnose # 12435: __all_virtual_clone_job # 12435: __all_clone_job # BASE_TABLE_NAME # 12436: __all_virtual_clone_job_history diff --git a/src/share/ob_storage_ha_diagnose_operator.cpp b/src/share/ob_storage_ha_diagnose_operator.cpp new file mode 100644 index 000000000..383faf4a8 --- /dev/null +++ b/src/share/ob_storage_ha_diagnose_operator.cpp @@ -0,0 +1,369 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SHARE + +#include "lib/mysqlclient/ob_mysql_proxy.h" // ObISqlClient, SMART_VAR +#include "share/inner_table/ob_inner_table_schema.h" +#include "share/ob_storage_ha_diagnose_operator.h" +#include "share/ob_dml_sql_splicer.h" // ObDMLSqlSplicer +#include "share/config/ob_server_config.h" + +namespace oceanbase +{ + +namespace share +{ + +ObStorageHADiagOperator::ObStorageHADiagOperator() + : is_inited_(false), + last_event_ts_(0) {} + +int ObStorageHADiagOperator::init() +{ + int ret = OB_SUCCESS; + if (is_inited_) { + ret = OB_INIT_TWICE; + LOG_WARN("ObStorageHADiagOperator is already init", K(ret)); + } else { + last_event_ts_ = 0; + is_inited_ = true; + } + return ret; +} + +void ObStorageHADiagOperator::reset() +{ + is_inited_ = false; + last_event_ts_ = 0; +} + +int ObStorageHADiagOperator::get_batch_row_keys( + common::ObISQLClient &sql_proxy, + const uint64_t tenant_id, + const ObStorageHADiagModule &module, + const int64_t last_end_timestamp, + ObIArray ×tamp_array) const +{ + int ret = OB_SUCCESS; + ObSqlString sql; + const char* table_name = nullptr; + timestamp_array.reset(); + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || ObStorageHADiagModule::MAX_MODULE == module)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(module)); + } else if (OB_FAIL(get_table_name_(module, table_name))) { + LOG_WARN("fail to get table name", KR(ret), K(module)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + sqlclient::ObMySQLResult *result = nullptr; + if (OB_FAIL(sql.assign_fmt( + "select time_to_usec(gmt_create) as create_ts from %s where gmt_create > usec_to_time(%ld) order by gmt_create limit 100", + table_name, last_end_timestamp))) { + LOG_WARN("assign_fmt failed", K(ret), K(last_end_timestamp)); + } else if (OB_FAIL(sql_proxy.read(res, sql.ptr()))) { + LOG_WARN("fail to execute sql", K(sql), K(ret), K(tenant_id)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result is NULL", K(sql), K(ret)); + } else { + while (OB_SUCC(ret)) { + if (OB_FAIL(result->next())) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + break; + } else { + LOG_WARN("fail to get next result", K(sql), K(ret)); + } + } else if (OB_ISNULL(result)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result is NULL", K(sql),K(ret)); + } else { + int64_t timestamp = 0; + EXTRACT_INT_FIELD_MYSQL(*result, "create_ts", timestamp, int64_t); + if (OB_FAIL(ret)) { + LOG_WARN("extract mysql failed", K(ret), K(sql)); + } else if (OB_FAIL(timestamp_array.push_back(timestamp))) { + LOG_WARN("failed to push to array", K(ret), K(timestamp_array), K(timestamp)); + } + } + } + } + } + } + return ret; +} + +int ObStorageHADiagOperator::do_batch_delete( + common::ObISQLClient &sql_proxy, + const uint64_t tenant_id, + const ObStorageHADiagModule &module, + const ObIArray ×tamp_array, + const int64_t delete_timestamp, + int64_t &delete_index) const +{ + int ret = OB_SUCCESS; + const char* table_name = nullptr; + ObSqlString sql; + int64_t affected_rows = 0; + delete_index = 0; + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || ObStorageHADiagModule::MAX_MODULE == module)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(module)); + } else if (OB_FAIL(get_table_name_(module, table_name))) { + LOG_WARN("fail to get table name", KR(ret), K(module)); + } else if (OB_FAIL(sql.assign_fmt("delete from %s where ", + table_name))) { + LOG_WARN("assign_fmt failed", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < timestamp_array.count(); i++) { + int64_t timestamp = timestamp_array.at(i); + if (timestamp <= delete_timestamp) { + if (i > 0 && OB_FAIL(sql.append_fmt(" or "))) { + LOG_WARN("append_fmt failed", K(ret), K(i)); + } else if (OB_FAIL(sql.append_fmt("gmt_create = usec_to_time(%ld)", timestamp))) { + LOG_WARN("append_fmt failed", K(ret), K(i), K(timestamp)); + } + } else if (0 == i) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected timestamp", K(ret), K(i), K(timestamp), K(delete_timestamp)); + } else { + delete_index = i - 1; + break; + } + } + if (OB_FAIL(ret)) { + //do nothing + LOG_WARN("fail to splice sql", K(ret), K(sql)); + } else if (OB_FAIL(sql_proxy.write(tenant_id, sql.ptr(), affected_rows))) { + LOG_WARN("fail to execute sql", K(sql), K(ret), K(tenant_id)); + } + } + return ret; +} + +//TODO(zhixing.yh) batch insert row +int ObStorageHADiagOperator::insert_row( + common::ObISQLClient &sql_proxy, + const uint64_t tenant_id, + const uint64_t report_tenant_id, + const ObStorageHADiagInfo &info, + const ObStorageHADiagModule &module) +{ + int ret = OB_SUCCESS; + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) + || !info.is_valid() + || ObStorageHADiagModule::MAX_MODULE == module)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(info)); + } else { + ObSqlString sql; + ObDMLSqlSplicer dml_splicer; + int64_t affected_rows = 0; + const char* table_name = nullptr; + if (OB_FAIL(fill_dml_(tenant_id, report_tenant_id, info, dml_splicer))) { + LOG_WARN("fail to fill dml splicer", KR(ret), K(tenant_id), K(info)); + } else if (OB_FAIL(get_table_name_(module, table_name))) { + LOG_WARN("fail to get table name", KR(ret), K(tenant_id), K(info)); + } else if (OB_FAIL(dml_splicer.splice_insert_sql(table_name, sql))) { + LOG_WARN("fail to splice insert sql", KR(ret), K(module)); + } else if (OB_FAIL(sql_proxy.write(tenant_id, sql.ptr(), affected_rows))) { + LOG_WARN("fail to execute sql", K(sql), K(ret), K(tenant_id)); + } else { + LOG_DEBUG("insert storage ha diagnose info history success", + K(tenant_id), K(affected_rows), K(info)); + } + } + return ret; +} + +int ObStorageHADiagOperator::get_table_name_(const ObStorageHADiagModule &module, const char *&table_name) const +{ + int ret = OB_SUCCESS; + switch(module) { + case ObStorageHADiagModule::TRANSFER_ERROR_DIAGNOSE: + table_name = OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_TNAME; + break; + case ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE: + table_name = OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_TNAME; + break; + default: + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid module", K(ret), K(module)); + break; + } + return ret; +} + +int ObStorageHADiagOperator::gen_event_ts_(int64_t &event_ts) +{ + int ret = OB_SUCCESS; + event_ts = 0; + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else { + const int64_t now = ObTimeUtility::current_time(); + event_ts = (last_event_ts_ >= now ? last_event_ts_ + 1 : now); + last_event_ts_ = event_ts; + } + return ret; +} + +int ObStorageHADiagOperator::fill_dml_( + const uint64_t tenant_id, + const uint64_t report_tenant_id, + const ObStorageHADiagInfo &input_info, + ObDMLSqlSplicer &dml) +{ + int ret = OB_SUCCESS; + char diagnose_info[common::OB_DIAGNOSE_INFO_LENGTH] = { 0 }; + char task_id[common::OB_MAX_TRACE_ID_BUFFER_SIZE] = { 0 }; + char ip_buf[common::OB_IP_STR_BUFF] = { 0 }; + int64_t timestamp = 0; + if (OB_FAIL(gen_event_ts_(timestamp))) { + LOG_WARN("failed to gen event ts", K(ret), K(timestamp), K(last_event_ts_)); + } else if (OB_FAIL(dml.add_gmt_create(timestamp))) { + LOG_WARN("failed to add gmt_create", K(ret), K(input_info)); + } else if (!ObServerConfig::get_instance().self_addr_.ip_to_string(ip_buf, sizeof(ip_buf))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to set svr_ip", K(ret)); + } else if (OB_FAIL(dml.add_column("svr_ip", ip_buf))) { + LOG_WARN("failed to add svr_ip", K(ret)); + } else if (OB_FAIL(dml.add_column("svr_port", ObServerConfig::get_instance().self_addr_.get_port()))) { + LOG_WARN("failed to add svr_port", K(ret)); + } else if (OB_FAIL(dml.add_column("tenant_id", report_tenant_id))) { + LOG_WARN("failed to add tenant id", K(ret), K(report_tenant_id)); + } else if (OB_FAIL(dml.add_column("ls_id", input_info.ls_id_.id()))) { + LOG_WARN("failed to add ls id", K(ret), K(input_info)); + } else if (OB_FAIL(dml.add_column("module", input_info.get_module_str()))) { + LOG_WARN("failed to add module", K(ret), K(input_info)); + } else if (OB_FAIL(dml.add_column("type", input_info.get_type_str()))) { + LOG_WARN("failed to add type", K(ret), K(input_info)); + } else if (OB_FAIL(input_info.get_task_id(task_id, sizeof(task_id)))) { + LOG_WARN("failed to get task id", K(ret), K(input_info)); + } else if (OB_FAIL(dml.add_column("task_id", task_id))) { + LOG_WARN("failed to add task id", K(ret), K(input_info)); + } else if (OB_FAIL(dml.add_column("retry_id", input_info.retry_id_))) { + LOG_WARN("failed to add retry id", K(ret), K(input_info)); + } else if (OB_FAIL(dml.add_column("result_code", input_info.result_code_))) { + LOG_WARN("failed to add result code", K(ret), K(input_info)); + } else if (OB_FAIL(dml.add_column("result_msg", input_info.get_transfer_error_diagnose_msg()))) { + LOG_WARN("failed to add result msg", K(ret), K(input_info)); + } else if (OB_FAIL(input_info.get_info(diagnose_info, sizeof(diagnose_info)))) { + LOG_WARN("failed to get diagnose info", K(ret), K(input_info)); + } else if (OB_FAIL(dml.add_column("info", diagnose_info))) { + LOG_WARN("failed to add info", K(ret), K(diagnose_info)); + } else { + switch (input_info.module_) { + case ObStorageHADiagModule::TRANSFER_ERROR_DIAGNOSE: { + if (OB_FAIL(dml.add_time_column("create_time", input_info.timestamp_))) { + LOG_WARN("failed to add create_time", K(ret), K(input_info)); + } + break; + } + case ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE: { + const ObTransferPerfDiagInfo &transfer_perf_info = static_cast(input_info); + if (OB_FAIL(dml.add_time_column("start_timestamp", input_info.timestamp_))) { + LOG_WARN("failed to add start timestamp", K(ret), K(transfer_perf_info)); + } else if (OB_FAIL(dml.add_time_column("end_timestamp", transfer_perf_info.end_timestamp_))) { + LOG_WARN("failed to add end timestamp", K(ret), K(transfer_perf_info)); + } else if (OB_FAIL(dml.add_column("tablet_id", transfer_perf_info.tablet_id_.id()))) { + LOG_WARN("failed to add tablet id", K(ret), K(transfer_perf_info)); + } else if (OB_FAIL(dml.add_column("tablet_count", transfer_perf_info.tablet_count_))) { + LOG_WARN("failed to add tablet count", K(ret), K(transfer_perf_info)); + } + break; + } + default: { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("unknow module", K(ret), K(input_info.module_)); + break; + } + } + } + return ret; +} + +int ObStorageHADiagOperator::parse_result_( + common::sqlclient::ObMySQLResult &result, + int64_t &result_count) const +{ + int ret = OB_SUCCESS; + result_count = 0; + EXTRACT_INT_FIELD_MYSQL(result, "count", result_count, int64_t); + if (OB_FAIL(ret)) { + LOG_WARN("extract mysql failed", KR(ret)); + } + return ret; +} + +int ObStorageHADiagOperator::check_transfer_task_exist( + common::ObISQLClient &sql_proxy, + const uint64_t tenant_id, + const share::ObTransferTaskID &task_id, + int64_t &result_count) const +{ + int ret = OB_SUCCESS; + result_count = 0; + if (!is_valid_tenant_id(tenant_id) || !task_id.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(task_id)); + } else { + ObSqlString sql; + SMART_VAR(ObISQLClient::ReadResult, res) { + sqlclient::ObMySQLResult *result = nullptr; + if (OB_FAIL(sql.assign_fmt( + "SELECT COUNT(*) as count FROM %s WHERE task_id = %ld", + OB_ALL_TRANSFER_TASK_TNAME, task_id.id()))) { + LOG_WARN("fail to assign sql", KR(ret), K(task_id)); + } else if (OB_FAIL(sql_proxy.read(res, tenant_id, sql.ptr()))) { + LOG_WARN("execute sql failed", KR(ret), K(tenant_id), K(sql)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get mysql result failed", KR(ret), K(tenant_id), K(sql)); + } else { + if (OB_FAIL(result->next())) { + if (OB_ITER_END == ret) { + ret = OB_ENTRY_NOT_EXIST; + } else { + LOG_WARN("fail to get next result", KR(ret), K(sql)); + } + } else if (OB_FAIL(parse_result_(*result, result_count))) { + LOG_WARN("failed to parse result", KR(ret), K(result_count)); + } else if (OB_FAIL(result->next())) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("get next result failed", KR(ret), K(result_count)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("read more than one row", KR(ret), K(result_count)); + } + } + } + } + return ret; +} + +} // end namespace share +} // end namespace oceanbase diff --git a/src/share/ob_storage_ha_diagnose_operator.h b/src/share/ob_storage_ha_diagnose_operator.h new file mode 100644 index 000000000..4b1d9782e --- /dev/null +++ b/src/share/ob_storage_ha_diagnose_operator.h @@ -0,0 +1,91 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_SHARE_HA_DIAGNOSE_OPERATOR_ +#define OCEANBASE_SHARE_HA_DIAGNOSE_OPERATOR_ + +#include "share/ob_storage_ha_diagnose_struct.h" + +namespace oceanbase +{ +namespace common +{ +class ObISQLClient; + +namespace sqlclient +{ +class ObMySQLResult; +} +} + +namespace share +{ +class ObDMLSqlSplicer; + +class ObStorageHADiagOperator final +{ +public: + ObStorageHADiagOperator(); + ~ObStorageHADiagOperator() {} + + int init(); + + void reset(); + int get_batch_row_keys( + common::ObISQLClient &sql_proxy, + const uint64_t tenant_id, + const ObStorageHADiagModule &module, + const int64_t last_end_timestamp, + ObIArray ×tamp_array) const; + + int insert_row( + common::ObISQLClient &sql_proxy, + const uint64_t tenant_id, + const uint64_t report_tenant_id, + const ObStorageHADiagInfo &info_row, + const ObStorageHADiagModule &module); + int do_batch_delete( + common::ObISQLClient &sql_proxy, + const uint64_t tenant_id, + const ObStorageHADiagModule &module, + const ObIArray ×tamp_array, + const int64_t delete_timestamp, + int64_t &delete_index) const; + int check_transfer_task_exist( + common::ObISQLClient &sql_proxy, + const uint64_t tenant_id, + const share::ObTransferTaskID &task_id, + int64_t &result_count) const; + +private: + int fill_dml_( + const uint64_t tenant_id, + const uint64_t report_tenant_id, + const ObStorageHADiagInfo &input_info, + ObDMLSqlSplicer &dml); + + int get_table_name_(const ObStorageHADiagModule &module, const char *&table_name) const; + int gen_event_ts_(int64_t &event_ts); + int parse_result_( + common::sqlclient::ObMySQLResult &result, + int64_t &result_count) const; + +private: + bool is_inited_; + int64_t last_event_ts_; + + DISALLOW_COPY_AND_ASSIGN(ObStorageHADiagOperator); +}; + +} // end namespace share +} // end namespace oceanbase +#endif diff --git a/src/share/ob_storage_ha_diagnose_struct.cpp b/src/share/ob_storage_ha_diagnose_struct.cpp new file mode 100644 index 000000000..c34ec7457 --- /dev/null +++ b/src/share/ob_storage_ha_diagnose_struct.cpp @@ -0,0 +1,1195 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SHARE +#include "ob_storage_ha_diagnose_struct.h" + +using namespace oceanbase; +using namespace share; +/** + * ------------------------------ObStorageHAPerfDiagParams--------------------- + */ +ObStorageHAPerfDiagParams::ObStorageHAPerfDiagParams() + : dest_ls_id_(), + task_id_(), + task_type_(ObStorageHADiagTaskType::MAX_TYPE), + item_type_(ObStorageHACostItemType::MAX_TYPE), + name_(ObStorageHACostItemName::MAX_NAME), + tablet_id_(), + tablet_count_(0) {} + +ObStorageHAPerfDiagParams::~ObStorageHAPerfDiagParams() +{ + reset(); +} + +void ObStorageHAPerfDiagParams::reset() +{ + dest_ls_id_.reset(); + task_id_.reset(); + task_type_ = ObStorageHADiagTaskType::MAX_TYPE; + item_type_ = ObStorageHACostItemType::MAX_TYPE; + name_ = ObStorageHACostItemName::MAX_NAME; + tablet_id_.reset(); + tablet_count_ = 0; +} + +bool ObStorageHAPerfDiagParams::is_valid() const +{ + return dest_ls_id_.is_valid() + && task_type_ >= ObStorageHADiagTaskType::TRANSFER_START + && task_type_ < ObStorageHADiagTaskType::MAX_TYPE + && item_type_ >= ObStorageHACostItemType::ACCUM_COST_TYPE + && item_type_ < ObStorageHACostItemType::MAX_TYPE + && name_ >= ObStorageHACostItemName::TRANSFER_START_BEGIN + && name_ < ObStorageHACostItemName::MAX_NAME + && tablet_count_ > 0; +} + +int ObStorageHAPerfDiagParams::assign(const ObStorageHAPerfDiagParams ¶ms) +{ + int ret = OB_SUCCESS; + if (!params.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("params is invalid", K(ret), K(params)); + } else if (this != ¶ms) { + dest_ls_id_ = params.dest_ls_id_; + task_id_ = params.task_id_; + item_type_ = params.item_type_; + name_ = params.name_; + tablet_id_ = params.tablet_id_; + tablet_count_ = params.tablet_count_; + } + return ret; +} + +/** + * ------------------------------ObStorageHADiagTaskKey--------------------- + */ +ObStorageHADiagTaskKey::ObStorageHADiagTaskKey() + : tenant_id_(OB_INVALID_TENANT_ID), + task_id_(0), + module_(ObStorageHADiagModule::MAX_MODULE), + type_(ObStorageHADiagTaskType::MAX_TYPE), + retry_id_(0), + diag_type_(ObStorageHADiagType::MAX_TYPE), + tablet_id_() +{ +} + +void ObStorageHADiagTaskKey::reset() +{ + tenant_id_ = OB_INVALID_TENANT_ID; + task_id_ = 0; + module_ = ObStorageHADiagModule::MAX_MODULE; + type_ = ObStorageHADiagTaskType::MAX_TYPE; + retry_id_ = 0; + diag_type_ = ObStorageHADiagType::MAX_TYPE; + tablet_id_.reset(); +} + +bool ObStorageHADiagTaskKey::is_valid() const +{ + return tenant_id_ != OB_INVALID_TENANT_ID + && task_id_ >= 0 + && module_ < ObStorageHADiagModule::MAX_MODULE + && module_ >= ObStorageHADiagModule::TRANSFER_ERROR_DIAGNOSE + && type_ < ObStorageHADiagTaskType::MAX_TYPE + && type_ >= ObStorageHADiagTaskType::TRANSFER_START + && retry_id_ >= 0 + && diag_type_ < ObStorageHADiagType::MAX_TYPE + && diag_type_ >= ObStorageHADiagType::ERROR_DIAGNOSE; +} + +bool ObStorageHADiagTaskKey::operator==(const ObStorageHADiagTaskKey &key) const +{ + bool ret = (tenant_id_ == key.tenant_id_ + && task_id_ == key.task_id_ + && module_ == key.module_ + && type_ == key.type_ + && diag_type_ == key.diag_type_); + if (ObStorageHADiagTaskType::TRANSFER_BACKFILLED == type_ && type_ == key.type_) { + ret = ret && (tablet_id_ == key.tablet_id_); + } + + if (ObStorageHADiagType::ERROR_DIAGNOSE == key.diag_type_ && key.diag_type_ == diag_type_) { + ret = ret && (retry_id_ == key.retry_id_); + } + return ret; +} + +bool ObStorageHADiagTaskKey::operator<(const ObStorageHADiagTaskKey &key) const +{ + return tenant_id_ < key.tenant_id_; +} + +int ObStorageHADiagTaskKey::assign(const ObStorageHADiagTaskKey &key) +{ + int ret = OB_SUCCESS; + if (!key.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(key)); + } else if (this != &key) { + tenant_id_ = key.tenant_id_; + task_id_ = key.task_id_; + module_ = key.module_; + type_ = key.type_; + retry_id_ = key.retry_id_; + diag_type_ = key.diag_type_; + tablet_id_ = key.tablet_id_; + } + return ret; +} + +uint64_t ObStorageHADiagTaskKey::hash() const +{ + uint64_t hash_val = 0; + + hash_val = murmurhash(&tenant_id_, sizeof(tenant_id_), hash_val); + hash_val = murmurhash(&task_id_, sizeof(task_id_), hash_val); + hash_val = murmurhash(&module_, sizeof(module_), hash_val); + hash_val = murmurhash(&type_, sizeof(type_), hash_val); + hash_val = murmurhash(&retry_id_, sizeof(retry_id_), hash_val); + hash_val = murmurhash(&diag_type_, sizeof(diag_type_), hash_val); + hash_val = murmurhash(&tablet_id_, sizeof(tablet_id_), hash_val); + return hash_val; +} + +int ObStorageHADiagTaskKey::hash(uint64_t &hash_val) const +{ + hash_val = hash(); + return OB_SUCCESS; +} + +/** + * ------------------------------ObStorageHADiagInfo--------------------- + */ +ObStorageHADiagInfo::ObStorageHADiagInfo() + : task_id_(0), + ls_id_(), + module_(ObStorageHADiagModule::MAX_MODULE), + type_(ObStorageHADiagTaskType::MAX_TYPE), + retry_id_(0), + result_code_(0), + timestamp_(0), + result_msg_(ObStorageHACostItemName::MAX_NAME), + task_(nullptr), + tablet_id_() +{ +} + +int ObStorageHADiagInfo::assign(const ObStorageHADiagInfo &info) +{ + int ret = OB_SUCCESS; + if (!info.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("info is invalid", K(ret), K(info)); + } else { + task_id_ = info.task_id_; + ls_id_ = info.ls_id_; + module_ = info.module_; + type_ = info.type_ ; + retry_id_ = info.retry_id_; + result_code_ = info.result_code_; + timestamp_ = info.timestamp_; + result_msg_ = info.result_msg_; + tablet_id_ = info.tablet_id_; + } + return ret; +} + +void ObStorageHADiagInfo::reset() +{ + task_id_ = 0; + ls_id_.reset(); + module_ = ObStorageHADiagModule::MAX_MODULE; + type_ = ObStorageHADiagTaskType::MAX_TYPE; + retry_id_ = 0; + result_code_ = 0; + timestamp_ = 0; + result_msg_ = ObStorageHACostItemName::MAX_NAME; + task_ = nullptr; + tablet_id_.reset(); +} + +bool ObStorageHADiagInfo::is_valid() const +{ + return task_id_ > 0 + && ls_id_.is_valid() + && module_ < ObStorageHADiagModule::MAX_MODULE + && module_ >= ObStorageHADiagModule::TRANSFER_ERROR_DIAGNOSE + && type_ < ObStorageHADiagTaskType::MAX_TYPE + && type_ >= ObStorageHADiagTaskType::TRANSFER_START + && retry_id_ >= 0 + && result_code_ <= 0 + && timestamp_ >= 0; +} + +const char *ObStorageHADiagInfo::ObStorageDiagModuleStr[static_cast(ObStorageHADiagModule::MAX_MODULE)] = { + "TRANSFER_ERROR_DIAGNOSE", + "TRANSFER_PERF_DIAGNOSE" +}; + +const char *ObStorageHADiagInfo::ObStorageDiagTaskTypeStr[static_cast(ObStorageHADiagTaskType::MAX_TYPE)] = { + "TRANSFER_START", + "TRANSFER_DOING", + "TRANSFER_ABORT", + "TRANSFER_START_IN", + "TRANSFER_START_OUT", + "TRANSFER_FINISH_IN", + "TRANSFER_FINISH_OUT", + "TRANSFER_BACKFILLED" +}; + +const char * ObStorageHADiagInfo::ObTransferErrorDiagMsg[static_cast(share::ObStorageHACostItemName::MAX_NAME)] = { + "TRANSFER_START_BEGIN_ERROR", + "LOCK_MEMBER_LIST_ERROR", + "PRECHECK_LS_REPALY_SCN_ERROR", + "CHECK_START_STATUS_TRANSFER_TABLETS_ERROR", + "CHECK_SRC_LS_HAS_ACTIVE_TRANS_ERROR", + "UPDATE_ALL_TABLET_TO_LS_ERROR", + "LOCK_TABLET_ON_DEST_LS_FOR_TABLE_LOCK_ERROR", + "BLOCK_AND_KILL_TX_ERROR", + "REGISTER_TRANSFER_START_OUT_ERROR", + "DEST_LS_GET_START_SCN_ERROR", + "CHECK_SRC_LS_HAS_ACTIVE_TRANS_LATER_ERROR", + "UNBLOCK_TX_ERROR", + "WAIT_SRC_LS_REPLAY_TO_START_SCN_ERROR", + "SRC_LS_GET_TABLET_META_ERROR", + "REGISTER_TRANSFER_START_IN_ERROR", + "START_TRANS_COMMIT_ERROR", + "TRANSFER_START_END_ERROR", + "TRANSFER_FINISH_BEGIN_ERROR", + "UNLOCK_SRC_AND_DEST_LS_MEMBER_LIST_ERROR", + "CHECK_LS_LOGICAL_TABLE_REPLACED_ERROR", + "LOCK_LS_MEMBER_LIST_IN_DOING_ERROR", + "CHECK_LS_LOGICAL_TABLE_REPLACED_LATER_ERROR", + "REGISTER_TRANSFER_FINISH_IN_ERROR", + "WAIT_TRANSFER_TABLET_STATUS_NORMAL_ERROR", + "WAIT_ALL_LS_REPLICA_REPLAY_FINISH_SCN_ERROR", + "REGISTER_TRANSFER_OUT_ERROR", + "UNLOCK_TABLET_FOR_LOCK_ERROR", + "UNLOCK_LS_MEMBER_LIST_ERROR", + "FINISH_TRANS_COMMIT_ERROR", + "TRANSFER_FINISH_END_ERROR", + "TRANSFER_ABORT_BEGIN_ERROR", + "UNLOCK_MEMBER_LIST_IN_ABORT_ERROR", + "ABORT_TRANS_COMMIT_ERROR", + "TRANSFER_ABORT_END_ERROR", + "TRANSFER_BACKFILLED_TABLE_BEGIN_ERROR", + "TX_BACKFILL_ERROR", + "TABLET_REPLACE_ERROR", + "REPLACE_TIMESTAMP_ERROR", + "TRANSFER_BACKFILLED_END_ERROR", + "START_TRANSFER_OUT_FIRST_REPALY_LOG_TIMESTAMP_ERROR", + "START_TRANSFER_OUT_LOG_SCN_ERROR", + "START_TRANSFER_OUT_TX_ID_ERROR", + "START_TRANSFER_OUT_REPLAY_COST_ERROR", + "START_TRANSFER_IN_FIRST_REPALY_LOG_TIMESTAMP_ERROR", + "START_TRANSFER_IN_LOG_SCN_ERROR", + "START_TRANSFER_IN_TX_ID_ERROR", + "START_TRANSFER_IN_REPLAY_COST_ERROR", + "START_CHECK_LS_EXIST_TS_ERROR", + "START_CHECK_LS_REPLAY_POINT_TS_ERROR", + "FINISH_TRANSFER_OUT_FIRST_REPALY_LOG_TIMESTAMP_ERROR", + "FINISH_TRANSFER_OUT_LOG_SCN_ERROR", + "FINISH_TRANSFER_OUT_TX_ID_ERROR", + "FINISH_TRANSFER_OUT_REPLAY_COST_ERROR", + "FINISH_TRANSFER_IN_FIRST_REPALY_LOG_TIMESTAMP_ERROR", + "FINISH_TRANSFER_IN_LOG_SCN_ERROR", + "FINISH_TRANSFER_IN_TX_ID_ERROR", + "FINISH_TRANSFER_IN_REPLAY_COST_ERROR", + "UNLOCK_MEMBER_LIST_IN_START_ERROR", + "ON_REGISTER_SUCCESS_ERROR", + "ON_REPLAY_SUCCESS_ERROR", + "PREPARE_MERGE_CTX_ERROR", + "PREPARE_INDEX_TREE_ERROR", + "DO_BACKFILL_TX_ERROR", + "TRANSFER_REPLACE_BEGIN_ERROR", + "TRANSFER_REPLACE_END_ERROR", + "TRANSFER_BACKFILL_START_ERROR", + "STOP_LS_SCHEDULE_MEMDIUM_ERROR", +}; + +const char * ObStorageHADiagInfo::get_transfer_error_diagnose_msg() const +{ + const char *str = ""; + if (result_msg_ < ObStorageHACostItemName::TRANSFER_START_BEGIN || result_msg_ >= ObStorageHACostItemName::MAX_NAME) { + str = "Unstatistical errors"; + } else { + str = ObTransferErrorDiagMsg[static_cast(result_msg_)]; + } + return str; +} + +const char * ObStorageHADiagInfo::get_module_str() const +{ + const char *str = ""; + if (module_ >= ObStorageHADiagModule::MAX_MODULE || module_ < ObStorageHADiagModule::TRANSFER_ERROR_DIAGNOSE) { + str = "invalid_module"; + } else { + str = ObStorageDiagModuleStr[static_cast(module_)]; + } + return str; +} + +const char * ObStorageHADiagInfo::get_type_str() const +{ + const char *str = ""; + if (type_ >= ObStorageHADiagTaskType::MAX_TYPE || type_ < ObStorageHADiagTaskType::TRANSFER_START) { + str = "invalid_type"; + } else { + str = ObStorageDiagTaskTypeStr[static_cast(type_)]; + } + return str; +} +/** + * ------------------------------ObStorageHADiagTask--------------------- + */ +ObStorageHADiagTask::ObStorageHADiagTask() + : key_(), + val_(nullptr) +{ +} + +ObStorageHADiagTask::~ObStorageHADiagTask() +{ + reset(); +} + +void ObStorageHADiagTask::reset() +{ + key_.reset(); + val_ = nullptr; +} + +bool ObStorageHADiagTask::is_valid() const +{ + return key_.is_valid() && val_ != nullptr; +} +/** + * ------------------------------ObTransferErrorDiagInfo--------------------- + */ +ObTransferErrorDiagInfo::ObTransferErrorDiagInfo() + : ObStorageHADiagInfo(), + trace_id_(), + thread_id_(0) +{ +} + +int ObTransferErrorDiagInfo::get_info(char *info, const int64_t size) const +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(info) || size <= 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(size)); + } else { + const int64_t MAX_TRACE_ID_LENGTH = 64; + char trace_id[MAX_TRACE_ID_LENGTH] = { 0 }; + int64_t pos = 0; + memset(info, 0, size); + if (0 == trace_id_.to_string(trace_id, sizeof(trace_id))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to print trace id.", K(ret), K(trace_id_)); + } else if (OB_FAIL(databuff_printf(info, size, pos, "%s%s%s%ld", "trace_id:", trace_id, "|thread_id:", thread_id_))) { + LOG_WARN("failed to print info.", K(ret), K(size), K(trace_id_), K(pos), K(thread_id_)); + } + } + return ret; +} + +int ObTransferErrorDiagInfo::get_task_id(char *info, const int64_t size) const +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(info) || size <= 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(size)); + } else { + int64_t pos = 0; + memset(info, 0, size); + if (OB_FAIL(databuff_printf(info, size, pos, "%ld", task_id_))) { + LOG_WARN("failed to print task_id.", K(ret), K(size), K(pos), K(task_id_)); + } + } + return ret; +} + +bool ObTransferErrorDiagInfo::is_valid() const +{ + return ObStorageHADiagInfo::is_valid() + && trace_id_.is_valid() + && thread_id_ > 0; +} + +void ObTransferErrorDiagInfo::reset() +{ + ObStorageHADiagInfo::reset(); + trace_id_.reset(); + thread_id_ = 0; +} + +int ObTransferErrorDiagInfo::update(const ObStorageHADiagInfo &info) +{ + int ret = OB_NOT_SUPPORTED; + LOG_WARN("NOT SUPPORTED UPDATE", K(ret)); + return ret; +} + +int ObTransferErrorDiagInfo::assign(const ObStorageHADiagInfo &info) +{ + int ret = OB_SUCCESS; + if (!info.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("info is invalid", K(ret), K(info)); + } else if (ObStorageHADiagModule::TRANSFER_ERROR_DIAGNOSE != info.module_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("info module type is unexpected", K(ret), K(info.module_)); + } else if (OB_FAIL(ObStorageHADiagInfo::assign(info))) { + LOG_WARN("failed to assign info.", K(ret), K(info)); + } else { + trace_id_ = static_cast(info).trace_id_; + thread_id_ = static_cast(info).thread_id_; + } + return ret; +} + +int ObTransferErrorDiagInfo::deep_copy(ObIAllocator &allocator, ObStorageHADiagInfo *&out_info) const +{ + int ret = OB_SUCCESS; + void *buf = nullptr; + out_info = nullptr; + if (OB_ISNULL(buf = allocator.alloc(sizeof(ObTransferErrorDiagInfo)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc memory", K(ret)); + } else { + ObTransferErrorDiagInfo *info = nullptr; + if (OB_ISNULL(info = (new (buf) ObTransferErrorDiagInfo()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("info is nullptr", K(ret)); + } else { + buf = nullptr; + if (OB_FAIL(info->assign(*this))) { + LOG_WARN("failed to assign info", K(ret), KPC(this)); + } else { + out_info = info; + info = nullptr; + } + if (OB_NOT_NULL(info)) { + info->~ObTransferErrorDiagInfo(); + allocator.free(info); + info = nullptr; + } + } + if (nullptr != buf) { + allocator.free(buf); + buf = nullptr; + } + + } + return ret; +} +/** + * ------------------------------ObIStorageHACostItem--------------------- + */ +ObIStorageHACostItem::ObIStorageHACostItem() + : name_(ObStorageHACostItemName::MAX_NAME), + type_(ObStorageHACostItemType::MAX_TYPE), + retry_id_(0) +{ +} + +bool ObIStorageHACostItem::is_valid() const +{ + return name_ >= ObStorageHACostItemName::TRANSFER_START_BEGIN + && name_ < ObStorageHACostItemName::MAX_NAME + && type_ >= ObStorageHACostItemType::ACCUM_COST_TYPE + && type_ < ObStorageHACostItemType::MAX_TYPE + && retry_id_ >= 0; +} + +void ObIStorageHACostItem::reset() +{ + name_ = ObStorageHACostItemName::MAX_NAME; + type_ = ObStorageHACostItemType::MAX_TYPE; + retry_id_ = 0; +} + +uint32_t ObIStorageHACostItem::get_size() const +{ + return sizeof(ObStorageHACostItemName) + sizeof(ObStorageHACostItemType) + sizeof(int64_t); +} + +int ObIStorageHACostItem::assign(const ObIStorageHACostItem &item) +{ + int ret = OB_SUCCESS; + if (!item.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(item)); + } else { + name_ = item.name_; + type_ = item.type_; + retry_id_ = item.retry_id_; + } + return ret; +} + +const char *ObIStorageHACostItem::ObStorageHACostItemNameStr[static_cast(ObStorageHACostItemName::MAX_NAME)] = { + "TRANSFER_START_BEGIN", + "LOCK_MEMBER_LIST", + "PRECHECK_LS_REPALY_SCN", + "CHECK_START_STATUS_TRANSFER_TABLETS", + "CHECK_SRC_LS_HAS_ACTIVE_TRANS", + "UPDATE_ALL_TABLET_TO_LS", + "LOCK_TABLET_ON_DEST_LS_FOR_TABLE_LOCK", + "BLOCK_AND_KILL_TX", + "REGISTER_TRANSFER_START_OUT", + "DEST_LS_GET_START_SCN", + "CHECK_SRC_LS_HAS_ACTIVE_TRANS_LATER", + "UNBLOCK_TX", + "WAIT_SRC_LS_REPLAY_TO_START_SCN", + "SRC_LS_GET_TABLET_META", + "REGISTER_TRANSFER_START_IN", + "START_TRANS_COMMIT", + "TRANSFER_START_END", + "TRANSFER_FINISH_BEGIN", + "UNLOCK_SRC_AND_DEST_LS_MEMBER_LIST", + "CHECK_LS_LOGICAL_TABLE_REPLACED", + "LOCK_LS_MEMBER_LIST_IN_DOING", + "CHECK_LS_LOGICAL_TABLE_REPLACED_LATER", + "REGISTER_TRANSFER_FINISH_IN", + "WAIT_TRANSFER_TABLET_STATUS_NORMAL", + "WAIT_ALL_LS_REPLICA_REPLAY_FINISH_SCN", + "REGISTER_TRANSFER_OUT", + "UNLOCK_TABLET_FOR_LOCK", + "UNLOCK_LS_MEMBER_LIST", + "FINISH_TRANS_COMMIT", + "TRANSFER_FINISH_END", + "TRANSFER_ABORT_BEGIN", + "UNLOCK_MEMBER_LIST_IN_ABORT", + "ABORT_TRANS_COMMIT", + "TRANSFER_ABORT_END", + "TRANSFER_BACKFILLED_TABLE_BEGIN", + "TX_BACKFILL", + "TABLET_REPLACE", + "REPLACE_TIMESTAMP", + "TRANSFER_BACKFILLED_END", + "START_TRANSFER_OUT_FIRST_REPALY_LOG_TIMESTAMP", + "START_TRANSFER_OUT_LOG_SCN", + "START_TRANSFER_OUT_TX_ID", + "START_TRANSFER_OUT_REPLAY_COST", + "START_TRANSFER_IN_FIRST_REPALY_LOG_TIMESTAMP", + "START_TRANSFER_IN_LOG_SCN", + "START_TRANSFER_IN_TX_ID", + "START_TRANSFER_IN_REPLAY_COST", + "START_CHECK_LS_EXIST_TS", + "START_CHECK_LS_REPLAY_POINT_TS", + "FINISH_TRANSFER_OUT_FIRST_REPALY_LOG_TIMESTAMP", + "FINISH_TRANSFER_OUT_LOG_SCN", + "FINISH_TRANSFER_OUT_TX_ID", + "FINISH_TRANSFER_OUT_REPLAY_COST", + "FINISH_TRANSFER_IN_FIRST_REPALY_LOG_TIMESTAMP", + "FINISH_TRANSFER_IN_LOG_SCN", + "FINISH_TRANSFER_IN_TX_ID", + "FINISH_TRANSFER_IN_REPLAY_COST", + "UNLOCK_MEMBER_LIST_IN_START", + "ON_REGISTER_SUCCESS", + "ON_REPLAY_SUCCESS", + "PREPARE_MERGE_CTX", + "PREPARE_INDEX_TREE", + "DO_BACKFILL_TX", + "TRANSFER_REPLACE_BEGIN", + "TRANSFER_REPLACE_END", + "TRANSFER_BACKFILL_START", + "STOP_LS_SCHEDULE_MEMDIUM", +}; + +const char *ObIStorageHACostItem::ObStorageHACostItemTypeStr[static_cast(ObStorageHACostItemType::MAX_TYPE)] = { + "ACCUM_COST_TYPE", + "FLUENT_TIMESTAMP_TYPE", + "CRUCIAL_TIMESTAMP", +}; + +const char *ObIStorageHACostItem::get_item_name() const +{ + const char *str = ""; + if (name_ < ObStorageHACostItemName::TRANSFER_START_BEGIN || name_ >= ObStorageHACostItemName::MAX_NAME) { + str = "invalid_name"; + } else { + str = ObStorageHACostItemNameStr[static_cast(name_)]; + } + return str; +} + +const char *ObIStorageHACostItem::get_item_type() const +{ + const char *str = ""; + if (type_ < ObStorageHACostItemType::ACCUM_COST_TYPE || type_ >= ObStorageHACostItemType::MAX_TYPE) { + str = "invalid_type"; + } else { + str = ObStorageHACostItemTypeStr[static_cast(type_)]; + } + return str; +} +/** + * ------------------------------ObStorageHACostAccumItem--------------------- + */ +ObStorageHACostAccumItem::ObStorageHACostAccumItem() + : ObIStorageHACostItem(), + cost_time_(0) {} + +int ObStorageHACostAccumItem::update(const ObIStorageHACostItem &item) +{ + int ret = OB_SUCCESS; + if (ObStorageHACostItemType::ACCUM_COST_TYPE != item.type_) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("crucial cost accum type can not update", K(ret), K(item.type_)); + } else { + cost_time_ += static_cast(item).cost_time_; + retry_id_ = static_cast(item).retry_id_; + } + return ret; +} + +int ObStorageHACostAccumItem::get_str(char *str, const int64_t size, int64_t &pos) const +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(str) || size <= 0 || pos < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(size), K(pos)); + } else if (OB_FAIL(databuff_printf(str, size, pos, "%s:%ldus,retry_id:%ld|", get_item_name(), cost_time_, retry_id_))) { + LOG_WARN("failed to print info.", K(ret), K(size), K(name_), K(pos), K(cost_time_)); + } + return ret; +} + +int ObStorageHACostAccumItem::deep_copy(ObIAllocator &allocator, ObIStorageHACostItem *&out_item) const +{ + int ret = OB_SUCCESS; + void *buf = nullptr; + out_item = nullptr; + if (OB_ISNULL(buf = allocator.alloc(sizeof(ObStorageHACostAccumItem)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc memory", K(ret)); + } else { + ObStorageHACostAccumItem *item = nullptr; + if (OB_ISNULL(item = (new (buf) ObStorageHACostAccumItem()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("item is nullptr", K(ret)); + } else { + buf = nullptr; + if (OB_FAIL(item->assign(*this))) { + LOG_WARN("failed to assign info", K(ret), KPC(this)); + } else { + out_item = item; + item = nullptr; + } + if (OB_NOT_NULL(item)) { + item->~ObStorageHACostAccumItem(); + allocator.free(item); + item = nullptr; + } + } + if (OB_NOT_NULL(buf)) { + allocator.free(buf); + buf = nullptr; + } + } + return ret; +} + +bool ObStorageHACostAccumItem::is_valid() const +{ + return ObIStorageHACostItem::is_valid() + && cost_time_ >= 0; +} + +void ObStorageHACostAccumItem::reset() +{ + ObIStorageHACostItem::reset(); + cost_time_ = 0; +} + +int ObStorageHACostAccumItem::assign(const ObIStorageHACostItem &item) +{ + int ret = OB_SUCCESS; + if (!item.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(item)); + } else if (ObStorageHACostItemType::ACCUM_COST_TYPE != item.type_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected item type", K(ret), K(item.type_)); + } else if (OB_FAIL(ObIStorageHACostItem::assign(item))) { + LOG_WARN("fail to assign item", K(ret), K(item)); + } else { + cost_time_ = static_cast(item).cost_time_; + } + return ret; +} + +uint32_t ObStorageHACostAccumItem::get_size() const +{ + return ObIStorageHACostItem::get_size() + sizeof(uint64_t); +} +/** + * ------------------------------ObStorageHATimestampItem--------------------- + */ +ObStorageHATimestampItem::ObStorageHATimestampItem() + : ObIStorageHACostItem(), + timestamp_(0) {} + +int ObStorageHATimestampItem::get_str(char *str, const int64_t size, int64_t &pos) const +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(str) || size <= 0 || pos < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(size), K(pos)); + } else if (OB_FAIL(databuff_printf(str, size, pos, "%s:%ld,retry_id:%ld|", get_item_name(), timestamp_, retry_id_))) { + LOG_WARN("failed to print info.", K(ret), K(size), K(name_), K(pos), K(timestamp_)); + } + return ret; +} + +bool ObStorageHATimestampItem::is_valid() const +{ + return ObIStorageHACostItem::is_valid() + && timestamp_ >= 0; +} + +void ObStorageHATimestampItem::reset() +{ + ObIStorageHACostItem::reset(); + timestamp_ = 0; +} + +int ObStorageHATimestampItem::assign(const ObIStorageHACostItem &item) +{ + int ret = OB_SUCCESS; + if (!item.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(item)); + } else if (ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE != item.type_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected item type", K(ret), K(item.type_)); + } else if (OB_FAIL(ObIStorageHACostItem::assign(item))) { + LOG_WARN("fail to assign item", K(ret), K(item)); + } else { + timestamp_ = static_cast(item).timestamp_; + } + return ret; +} + +int ObStorageHATimestampItem::deep_copy(ObIAllocator &allocator, ObIStorageHACostItem *&out_item) const +{ + int ret = OB_SUCCESS; + void *buf = nullptr; + out_item = nullptr; + if (OB_ISNULL(buf = allocator.alloc(sizeof(ObStorageHATimestampItem)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc memory", K(ret)); + } else { + ObStorageHATimestampItem *item = nullptr; + if (OB_ISNULL(item = (new (buf) ObStorageHATimestampItem()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("item is nullptr", K(ret)); + } else { + buf = nullptr; + if (OB_FAIL(item->assign(*this))) { + LOG_WARN("failed to assign info", K(ret), KPC(this)); + } else { + out_item = item; + item = nullptr; + } + if (OB_NOT_NULL(item)) { + item->~ObStorageHATimestampItem(); + allocator.free(item); + item = nullptr; + } + } + if (OB_NOT_NULL(buf)) { + allocator.free(buf); + buf = nullptr; + } + } + return ret; +} + +int ObStorageHATimestampItem::update(const ObIStorageHACostItem &item) +{ + int ret = OB_SUCCESS; + if (ObStorageHACostItemType::ACCUM_COST_TYPE == item.type_) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("crucial cost accum type can not update", K(ret), K(item.type_)); + } else { + // do nothing, FLUENT_TIMESTAMP_TYPE and CRUCIAL_TIMESTAMP only + // record the time of the first generation and do not update with retries. + } + return ret; +} + +uint32_t ObStorageHATimestampItem::get_size() const +{ + return ObIStorageHACostItem::get_size() + sizeof(uint64_t); +} +/** + * ------------------------------ObTransferPerfDiagInfo--------------------- + */ +ObTransferPerfDiagInfo::ObTransferPerfDiagInfo() + : ObStorageHADiagInfo(), + end_timestamp_(0), + tablet_count_(0), + items_(), + item_map_(), + item_size_(0), + is_inited_(false), + allocator_(nullptr) +{ +} + +ObTransferPerfDiagInfo::~ObTransferPerfDiagInfo() +{ + destroy(); +} + +int ObTransferPerfDiagInfo::init(ObIAllocator *allocator, const int64_t tenant_id) +{ + int ret = OB_SUCCESS; + ObMemAttr attr(tenant_id, "PerfDiagItem"); + if (is_inited_) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", K(ret)); + } else if (OB_ISNULL(allocator)) { + ret = OB_INVALID_ARGUMENT; + } else if (OB_FAIL(item_map_.create(PERF_DIAG_BUCKET_NUM, attr))) { + LOG_WARN("fail to create item map", K(ret)); + } else { + allocator_ = allocator; + is_inited_ = true; + } + return ret; +} + +bool ObTransferPerfDiagInfo::is_valid() const +{ + return is_inited_ && end_timestamp_ > 0 && tablet_count_ > 0 + && !items_.is_empty() && !item_map_.empty(); +} + +void ObTransferPerfDiagInfo::clear() +{ + int ret = OB_SUCCESS; + ObIStorageHACostItem *item = nullptr; + if (!is_inited_) { + if (items_.get_size() > 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("items_ should is null", K(ret), K(items_.get_size())); + } + } else { + while (OB_SUCC(ret) && items_.get_size() > 0) { + if (OB_ISNULL(item = items_.remove_first())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to remove first item", K(ret)); + } else { + item->~ObIStorageHACostItem(); + allocator_->free(item); + item = nullptr; + } + } + } + ObStorageHADiagInfo::reset(); + end_timestamp_ = 0; + tablet_count_ = 0; + item_size_ = 0; + items_.clear(); +} + +void ObTransferPerfDiagInfo::destroy() +{ + clear(); + allocator_ = nullptr; + is_inited_ = false; + item_map_.destroy(); +} + +void ObTransferPerfDiagInfo::reuse() +{ + clear(); + item_map_.reuse(); +} + +int ObTransferPerfDiagInfo::add_item(const ObIStorageHACostItem &item) +{ + int ret = OB_SUCCESS; + ObIStorageHACostItem *copy_item = nullptr; + bool add_list = false; + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (!item.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("item is invalid", K(ret), K(item)); + } else if (item_size_ + item.get_size() > ITEM_MAX_SIZE) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("items size overflow", K(ret), K(item_size_), K(item.get_size())); + } else if (OB_FAIL(item.deep_copy(*allocator_, copy_item))) { + LOG_WARN("fail to deep copy item", K(ret), K(item)); + } else if (!items_.add_last(copy_item)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to add item", K(ret)); + } else { + add_list = true; + int is_overwrite = 0; // do not overwrite + if (OB_FAIL(item_map_.set_refactored(static_cast(item.name_), copy_item, is_overwrite))) { + if (OB_HASH_EXIST == ret) { + // overwrite ret + ret = OB_ERR_UNEXPECTED; + LOG_WARN("item exist", K(ret)); + } else { + LOG_WARN("fail to add item to set", K(ret)); + } + } else { + item_size_ += item.get_size(); + } + } + if (OB_FAIL(ret) && NULL != copy_item) { + if (add_list) { + int tmp_ret = OB_SUCCESS; + if (OB_ISNULL(items_.remove(copy_item))) { + tmp_ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to remove item", K(tmp_ret), K(*copy_item)); + } + } + copy_item->~ObIStorageHACostItem(); + allocator_->free(copy_item); + copy_item = NULL; + } + return ret; +} + +int ObTransferPerfDiagInfo::find_item_(const ObStorageHACostItemName &name, bool &found, ObIStorageHACostItem *&item) +{ + int ret = OB_SUCCESS; + item = nullptr; + found = false; + if (name >= ObStorageHACostItemName::MAX_NAME || name < ObStorageHACostItemName::TRANSFER_START_BEGIN) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(name)); + } else if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_FAIL(item_map_.get_refactored(static_cast(name), item))) { + if (OB_HASH_NOT_EXIST == ret) { + //overwrite ret + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to get retry id from map", K(ret), K(name)); + } + } else { + found = true; + } + return ret; +} + +int ObTransferPerfDiagInfo::update_item_list_(const ObTransferPerfDiagInfo &info) +{ + int ret = OB_SUCCESS; + if (info.items_.is_empty()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(info.items_)); + } else { + ObIStorageHACostItem *old_item = nullptr; + bool found = false; + DLIST_FOREACH_X(iter, info.items_, OB_SUCC(ret)) { + if (OB_FAIL(find_item_(iter->name_, found, old_item))) { + LOG_WARN("fail to find item", K(ret), KPC(iter)); + } else if (!found) { + if (OB_FAIL(this->add_item(*iter))) { + LOG_WARN("fail to add item", K(ret), KPC(iter)); + } + } else if (OB_ISNULL(old_item)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("old_item is nullptr", K(ret)); + } else if (OB_FAIL(old_item->update(*iter))) { + LOG_WARN("fail to update item", K(ret), KPC(iter)); + } + } + } + return ret; +} + +int ObTransferPerfDiagInfo::update(const ObStorageHADiagInfo &info) +{ + int ret = OB_SUCCESS; + if (!info.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("info is invalid", K(ret), K(info)); + } else if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE != info.module_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected info module", K(ret), K(info.module_)); + } else { + const ObTransferPerfDiagInfo &diag_info = static_cast(info); + if (OB_FAIL(update_item_list_(diag_info))) { + LOG_WARN("fail to update item list", K(ret), K(diag_info)); + } else if (OB_ISNULL(task_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("task_ is nullptr", K(ret)); + } else { + timestamp_ = 0 == timestamp_ ? diag_info.timestamp_ : timestamp_; + end_timestamp_ = diag_info.end_timestamp_; + task_->key_.retry_id_ = diag_info.items_.get_first()->retry_id_; + retry_id_ = diag_info.items_.get_first()->retry_id_; + } + } + return ret; +} + +bool ObTransferPerfDiagInfo::is_inited() const +{ + return is_inited_; +} + +int ObTransferPerfDiagInfo::copy_item_list(const ObTransferPerfDiagInfo &info) +{ + int ret = OB_SUCCESS; + ObIStorageHACostItem *copy_item = nullptr; + if (!info.is_inited()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("info is not inited", K(ret)); + } else if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else { + DLIST_FOREACH_X(iter, info.items_, OB_SUCC(ret)) { + if (OB_FAIL(iter->deep_copy(*allocator_, copy_item))) { + LOG_WARN("fail to deep copy item", K(ret), KPC(iter)); + } else if (!items_.add_last(copy_item)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to add item", K(ret), KPC(copy_item)); + } else if (OB_FAIL(item_map_.set_refactored(static_cast(iter->name_), copy_item, 0/*is_overwrite*/))) { + LOG_WARN("fail to add item to map", K(ret), KPC(iter)); + } else { + item_size_ += copy_item->get_size(); + } + } + } + return ret; +} + +int ObTransferPerfDiagInfo::deep_copy(ObIAllocator &allocator, ObStorageHADiagInfo *&out_info) const +{ + int ret = OB_SUCCESS; + void *buf = nullptr; + out_info = nullptr; + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_ISNULL(buf = allocator.alloc(sizeof(ObTransferPerfDiagInfo)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc memory", K(ret)); + } else { + ObTransferPerfDiagInfo *info = nullptr; + if (OB_ISNULL(info = (new (buf) ObTransferPerfDiagInfo()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("item is nullptr", K(ret)); + } else { + buf = nullptr; + if (OB_FAIL(info->init(&allocator, MTL_ID()))) { + LOG_WARN("fail to init transfer perf info", K(ret)); + } else if (OB_FAIL(info->copy_item_list(*this))) { + LOG_WARN("fail to assign item list", K(ret), K(info)); + } else if (OB_FAIL(info->assign(*this))) { + LOG_WARN("failed to assign info", K(ret), KPC(this)); + } else { + out_info = info; + info = nullptr; + } + if (OB_NOT_NULL(info)) { + info->~ObTransferPerfDiagInfo(); + allocator.free(info); + info = nullptr; + } + } + if (nullptr != buf) { + allocator.free(buf); + buf = nullptr; + } + } + return ret; +} + +int ObTransferPerfDiagInfo::get_info(char *str, const int64_t size) const +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(str) || size <= 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(size)); + } else if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else { + int64_t pos = 0; + memset(str, 0, size); + DLIST_FOREACH_X(item, items_, OB_SUCC(ret)) { + //unset error code for print Truncated info + if (pos + strlen(item->get_item_name()) >= size) { + LOG_WARN("size overwrite", K(ret), K(pos), K(item), "item_len", strlen(item->get_item_name()), K(size)); + break; + } else if (OB_FAIL(item->get_str(str, size, pos))) { + LOG_WARN("fail to get info", K(ret), K(size), K(pos)); + } + } + } + return ret; +} + +int ObTransferPerfDiagInfo::get_task_id(char *str, const int64_t size) const +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(str) || size <= 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(size)); + } else if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else { + int64_t pos = 0; + memset(str, 0, size); + if (OB_FAIL(databuff_printf(str, size, pos, "%ld", task_id_))) { + LOG_WARN("failed to print task_id.", K(ret), K(size), K(pos), K(task_id_)); + } + } + return ret; +} + +int ObTransferPerfDiagInfo::assign(const ObStorageHADiagInfo &info) +{ + int ret = OB_SUCCESS; + if (!info.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(info)); + } else if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE != info.module_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("info module type is unexpected", K(ret), K(info.module_)); + } else if (OB_FAIL(ObStorageHADiagInfo::assign(info))) { + LOG_WARN("fail to assign info", K(ret), K(info)); + } else { + const ObTransferPerfDiagInfo &diag_info = static_cast(info); + end_timestamp_ = diag_info.end_timestamp_; + tablet_count_ = diag_info.tablet_count_; + } + return ret; +} \ No newline at end of file diff --git a/src/share/ob_storage_ha_diagnose_struct.h b/src/share/ob_storage_ha_diagnose_struct.h new file mode 100644 index 000000000..d4c90f8fd --- /dev/null +++ b/src/share/ob_storage_ha_diagnose_struct.h @@ -0,0 +1,395 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEABASE_SHARE_HA_MONITOR_STRUCT_ +#define OCEABASE_SHARE_HA_MONITOR_STRUCT_ + +#include "share/transfer/ob_transfer_info.h" +#include "share/ob_ls_id.h" + +namespace oceanbase +{ +namespace share +{ +enum class ObStorageHADiagType +{ + ERROR_DIAGNOSE = 0, + PERF_DIAGNOSE = 1, + MAX_TYPE +}; + +enum class ObStorageHADiagModule +{ + TRANSFER_ERROR_DIAGNOSE = 0, + TRANSFER_PERF_DIAGNOSE = 1, + MAX_MODULE +}; + +enum class ObStorageHADiagTaskType +{ + TRANSFER_START = 0, + TRANSFER_DOING = 1, + TRANSFER_ABORT = 2, + TRANSFER_START_IN = 3, + TRANSFER_START_OUT = 4, + TRANSFER_FINISH_IN = 5, + TRANSFER_FINISH_OUT = 6, + TRANSFER_BACKFILLED = 7, + MAX_TYPE +}; + +enum class ObStorageHACostItemName : int +{ + // start stage + TRANSFER_START_BEGIN = 0, + LOCK_MEMBER_LIST = 1, + PRECHECK_LS_REPALY_SCN = 2, + CHECK_START_STATUS_TRANSFER_TABLETS = 3, + CHECK_SRC_LS_HAS_ACTIVE_TRANS = 4, + UPDATE_ALL_TABLET_TO_LS = 5, + LOCK_TABLET_ON_DEST_LS_FOR_TABLE_LOCK = 6, + BLOCK_AND_KILL_TX = 7, + REGISTER_TRANSFER_START_OUT = 8, + DEST_LS_GET_START_SCN = 9, + CHECK_SRC_LS_HAS_ACTIVE_TRANS_LATER = 10, + UNBLOCK_TX = 11, + WAIT_SRC_LS_REPLAY_TO_START_SCN = 12, + SRC_LS_GET_TABLET_META = 13, + REGISTER_TRANSFER_START_IN = 14, + START_TRANS_COMMIT = 15, + TRANSFER_START_END = 16, + // doing stage + TRANSFER_FINISH_BEGIN = 17, + UNLOCK_SRC_AND_DEST_LS_MEMBER_LIST = 18, + CHECK_LS_LOGICAL_TABLE_REPLACED = 19, + LOCK_LS_MEMBER_LIST_IN_DOING = 20, + CHECK_LS_LOGICAL_TABLE_REPLACED_LATER = 21, + REGISTER_TRANSFER_FINISH_IN = 22, + WAIT_TRANSFER_TABLET_STATUS_NORMAL = 23, + WAIT_ALL_LS_REPLICA_REPLAY_FINISH_SCN = 24, + REGISTER_TRANSFER_OUT = 25, + UNLOCK_TABLET_FOR_LOCK = 26, + UNLOCK_LS_MEMBER_LIST = 27, + FINISH_TRANS_COMMIT = 28, + TRANSFER_FINISH_END = 29, + // abort stage + TRANSFER_ABORT_BEGIN = 30, + UNLOCK_MEMBER_LIST_IN_ABORT = 31, + ABORT_TRANS_COMMIT = 32, + TRANSFER_ABORT_END = 33, + // backfill stage + TRANSFER_BACKFILLED_TABLE_BEGIN = 34, + TX_BACKFILL = 35, + TABLET_REPLACE = 36, + REPLACE_TIMESTAMP = 37, + TRANSFER_BACKFILLED_END = 38, + // start stage log replay + START_TRANSFER_OUT_FIRST_REPALY_LOG_TIMESTAMP = 39, + START_TRANSFER_OUT_LOG_SCN = 40, + START_TRANSFER_OUT_TX_ID = 41, + START_TRANSFER_OUT_REPLAY_COST = 42, + START_TRANSFER_IN_FIRST_REPALY_LOG_TIMESTAMP = 43, + START_TRANSFER_IN_LOG_SCN = 44, + START_TRANSFER_IN_TX_ID = 45, + START_TRANSFER_IN_REPLAY_COST = 46, + START_CHECK_LS_EXIST_TS = 47, + START_CHECK_LS_REPLAY_POINT_TS = 48, + // doing stage log replay + FINISH_TRANSFER_OUT_FIRST_REPALY_LOG_TIMESTAMP = 49, + FINISH_TRANSFER_OUT_LOG_SCN = 50, + FINISH_TRANSFER_OUT_TX_ID = 51, + FINISH_TRANSFER_OUT_REPLAY_COST = 52, + FINISH_TRANSFER_IN_FIRST_REPALY_LOG_TIMESTAMP = 53, + FINISH_TRANSFER_IN_LOG_SCN = 54, + FINISH_TRANSFER_IN_TX_ID = 55, + FINISH_TRANSFER_IN_REPLAY_COST = 56, + // start stage supplement + UNLOCK_MEMBER_LIST_IN_START = 57, + // reply stage supplement + ON_REGISTER_SUCCESS = 58, + ON_REPLAY_SUCCESS = 59, + // backfilled stage supplement + PREPARE_MERGE_CTX = 60, + PREPARE_INDEX_TREE = 61, + DO_BACKFILL_TX = 62, + TRANSFER_REPLACE_BEGIN = 63, + TRANSFER_REPLACE_END = 64, + TRANSFER_BACKFILL_START = 65, + // start + STOP_LS_SCHEDULE_MEMDIUM = 66, + MAX_NAME +}; + +enum class ObStorageHACostItemType +{ + ACCUM_COST_TYPE = 0, + FLUENT_TIMESTAMP_TYPE = 1, + CRUCIAL_TIMESTAMP = 2, + MAX_TYPE +}; + +struct ObStorageHAPerfDiagParams final +{ +public: + ObStorageHAPerfDiagParams(); + ~ObStorageHAPerfDiagParams(); + void reset(); + bool is_valid() const; + int assign(const ObStorageHAPerfDiagParams ¶ms); + + TO_STRING_KV(K_(dest_ls_id), K_(task_id), K_(task_type), + K_(item_type), K_(name), K_(tablet_id), K_(tablet_count)); + + ObLSID dest_ls_id_; + share::ObTransferTaskID task_id_; + ObStorageHADiagTaskType task_type_; + ObStorageHACostItemType item_type_; + ObStorageHACostItemName name_; + common::ObTabletID tablet_id_; + int64_t tablet_count_; + DISALLOW_COPY_AND_ASSIGN(ObStorageHAPerfDiagParams); +}; + +struct ObStorageHADiagTaskKey final +{ +public: + ObStorageHADiagTaskKey(); + ~ObStorageHADiagTaskKey() { reset(); } + void reset(); + bool is_valid() const; + uint64_t hash() const; + int hash(uint64_t &hash_val) const; + bool operator==(const ObStorageHADiagTaskKey &key) const; + bool operator<(const ObStorageHADiagTaskKey &key) const; + int assign(const ObStorageHADiagTaskKey &key); + + TO_STRING_KV( + K_(tenant_id), + K_(task_id), + K_(module), + K_(type), + K_(retry_id), + K_(diag_type), + K_(tablet_id)); + + uint64_t tenant_id_; + int64_t task_id_; + ObStorageHADiagModule module_; + ObStorageHADiagTaskType type_; + int64_t retry_id_; + ObStorageHADiagType diag_type_; + common::ObTabletID tablet_id_; +}; + +struct ObStorageHADiagTask; +struct ObStorageHADiagInfo +{ +public: + ObStorageHADiagInfo(); + virtual ~ObStorageHADiagInfo() { reset(); } + virtual int get_info(char *info, const int64_t size) const = 0; + virtual int update(const ObStorageHADiagInfo &info) = 0; + virtual int assign(const ObStorageHADiagInfo &other); + virtual int deep_copy(ObIAllocator &allocator, ObStorageHADiagInfo *&out_info) const = 0; + virtual int get_task_id(char *info, const int64_t size) const = 0; + const char * get_module_str() const; + const char * get_type_str() const; + const char * get_transfer_error_diagnose_msg() const; + void reset(); + bool is_valid() const; + + const static char *ObStorageDiagModuleStr[static_cast(ObStorageHADiagModule::MAX_MODULE)]; + const static char *ObStorageDiagTaskTypeStr[static_cast(ObStorageHADiagTaskType::MAX_TYPE)]; + const static char *ObTransferErrorDiagMsg[static_cast(share::ObStorageHACostItemName::MAX_NAME)]; + + TO_STRING_KV( + K_(task_id), + K_(ls_id), + K_(module), + K_(type), + K_(retry_id), + K_(result_code), + K_(timestamp), + K_(result_msg), + K_(tablet_id)); + + int64_t task_id_; + ObLSID ls_id_; + ObStorageHADiagModule module_; + ObStorageHADiagTaskType type_; + int64_t retry_id_; + int64_t result_code_; + int64_t timestamp_; + ObStorageHACostItemName result_msg_; + ObStorageHADiagTask *task_; // TODO(zhixing.yh) exist risk，it will be deleted after key placed in info + ObTabletID tablet_id_; + + DISALLOW_COPY_AND_ASSIGN(ObStorageHADiagInfo); +}; + +struct ObStorageHADiagTask : public common::ObDLinkBase +{ +public: + ObStorageHADiagTask(); + virtual ~ObStorageHADiagTask(); + void reset(); + bool is_valid() const; + TO_STRING_KV(K_(key), KPC_(val)); + + ObStorageHADiagTaskKey key_; + ObStorageHADiagInfo *val_; +}; + +struct ObTransferErrorDiagInfo : public ObStorageHADiagInfo +{ +public: + ObTransferErrorDiagInfo(); + virtual ~ObTransferErrorDiagInfo() { reset(); } + virtual int get_info(char *info, const int64_t size) const override; + virtual int get_task_id(char *info, const int64_t size) const override; + virtual int update(const ObStorageHADiagInfo &info) override; + virtual int deep_copy(ObIAllocator &allocator, ObStorageHADiagInfo *&out_info) const override; + virtual int assign(const ObStorageHADiagInfo &another) override; + bool is_valid() const; + void reset(); + + INHERIT_TO_STRING_KV( + "ObStorageHADiagInfo", + ObStorageHADiagInfo, + K_(trace_id), + K_(thread_id)); + + common::ObCurTraceId::TraceId trace_id_; + int64_t thread_id_; +}; + +struct ObIStorageHACostItem : public common::ObDLinkBase +{ +public: + ObIStorageHACostItem(); + virtual ~ObIStorageHACostItem() { reset(); } + virtual int update(const ObIStorageHACostItem &item) = 0; + virtual int get_str(char *str, const int64_t size, int64_t &pos) const = 0; + virtual int assign(const ObIStorageHACostItem &item); + virtual int deep_copy(ObIAllocator &allocator, ObIStorageHACostItem *&item) const = 0; + virtual uint32_t get_size() const; + bool is_valid() const; + void reset(); + const char * get_item_name() const; + const char * get_item_type() const; + + const static char *ObStorageHACostItemNameStr[static_cast(ObStorageHACostItemName::MAX_NAME)]; + const static char *ObStorageHACostItemTypeStr[static_cast(ObStorageHACostItemType::MAX_TYPE)]; + + TO_STRING_KV( + K_(name), + K_(type), + K_(retry_id)); + +public: + ObStorageHACostItemName name_; + ObStorageHACostItemType type_; + int64_t retry_id_; +}; + +struct ObStorageHACostAccumItem : public ObIStorageHACostItem +{ +public: + ObStorageHACostAccumItem(); + virtual ~ObStorageHACostAccumItem() { reset(); } + virtual int update(const ObIStorageHACostItem &item) override; + virtual int get_str(char *str, const int64_t size, int64_t &pos) const override; + virtual int deep_copy(ObIAllocator &allocator, ObIStorageHACostItem *&item) const override; + virtual uint32_t get_size() const override; + bool is_valid() const; + void reset(); + virtual int assign(const ObIStorageHACostItem &item) override; + + INHERIT_TO_STRING_KV( + "ObIStorageHACostItem", + ObIStorageHACostItem, + K_(cost_time)); +public: + uint64_t cost_time_; +}; + +struct ObStorageHATimestampItem : public ObIStorageHACostItem +{ +public: + ObStorageHATimestampItem(); + virtual ~ObStorageHATimestampItem() { reset(); } + virtual int update(const ObIStorageHACostItem &item) override; + virtual int get_str(char *str, const int64_t size, int64_t &pos) const override; + virtual int assign(const ObIStorageHACostItem &item) override; + virtual int deep_copy(ObIAllocator &allocator, ObIStorageHACostItem *&item) const override; + virtual uint32_t get_size() const override; + bool is_valid() const; + void reset(); + + INHERIT_TO_STRING_KV( + "ObIStorageHACostItem", + ObIStorageHACostItem, + K_(timestamp)); +public: + uint64_t timestamp_; +}; + +struct ObTransferPerfDiagInfo : public ObStorageHADiagInfo +{ +public: + ObTransferPerfDiagInfo(); + virtual ~ObTransferPerfDiagInfo(); + typedef ObDList ItemList; + typedef hash::ObHashMap ItemMap; + int init(ObIAllocator *allocator, const int64_t tenant_id); + virtual int update(const ObStorageHADiagInfo &info) override; + virtual int deep_copy(ObIAllocator &allocator, ObStorageHADiagInfo *&out_info) const override; + virtual int get_task_id(char *info, const int64_t size) const override; + virtual int get_info(char *info, const int64_t size) const override; + virtual int assign(const ObStorageHADiagInfo &info) override; + int copy_item_list(const ObTransferPerfDiagInfo &info); + int add_item(const ObIStorageHACostItem &item); + bool is_inited() const; + bool is_valid() const; + void reuse(); + void clear(); + void destroy(); + + INHERIT_TO_STRING_KV( + "ObStorageHADiagInfo", + ObStorageHADiagInfo, + K_(is_inited), + K_(end_timestamp), + K_(tablet_count), + K_(items)); + +private: + int find_item_(const ObStorageHACostItemName &name, bool &found, ObIStorageHACostItem *&item); + int update_item_list_(const ObTransferPerfDiagInfo &info); + static constexpr int64_t PERF_DIAG_BUCKET_NUM = 100; + static constexpr int64_t ITEM_MAX_SIZE = (1 << 12); // 4K + +public: + uint64_t end_timestamp_; + int64_t tablet_count_; + ItemList items_; + ItemMap item_map_; + uint64_t item_size_; + +private: + bool is_inited_; + ObIAllocator *allocator_; +}; + +} +} +#endif \ No newline at end of file diff --git a/src/share/parameter/ob_parameter_seed.ipp b/src/share/parameter/ob_parameter_seed.ipp index 3dbc2e81c..db5871d76 100644 --- a/src/share/parameter/ob_parameter_seed.ipp +++ b/src/share/parameter/ob_parameter_seed.ipp @@ -1836,6 +1836,13 @@ DEF_BOOL(_enable_range_extraction_for_not_in, OB_TENANT_PARAMETER, "True", // DEF_BOOL(_enable_new_query_range_extraction, OB_TENANT_PARAMETER, "True", // "decide whether use new algorithm to extract query range.", // ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +DEF_TIME(_ha_diagnose_history_recycle_interval, OB_CLUSTER_PARAMETER, "7d", "[2m, 180d]", + "The recycle interval time of diagnostic history data. Range: [2m, 180d]", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +ERRSIM_DEF_TIME(errsim_transfer_diagnose_server_wait_time, OB_CLUSTER_PARAMETER, "5m", "[1s,1h]", + "the wakeup wait time of the transfer diagnose server. " + "Range: [0s, 1h]", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); ERRSIM_DEF_STR(palf_inject_receive_log_error_zone, OB_CLUSTER_PARAMETER, "", "specifies the zone name that palf module want to inject error when receive log", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); ERRSIM_DEF_STR(migrate_check_member_list_error_zone, OB_CLUSTER_PARAMETER, "", "specifies the zone name that migrate want to inject error when change member list", diff --git a/src/share/rc/ob_tenant_base.h b/src/share/rc/ob_tenant_base.h index 0b851bc25..35eaf0045 100755 --- a/src/share/rc/ob_tenant_base.h +++ b/src/share/rc/ob_tenant_base.h @@ -200,6 +200,7 @@ namespace observer } namespace storage { class MockTenantModuleEnv; + class ObStorageHADiagMgr; } namespace share @@ -357,6 +358,7 @@ using ObTableScanIteratorObjPool = common::ObServerObjectPoolwakeup(); } else { + process_perf_diagnose_info_(ObStorageHACostItemName::CHECK_LS_LOGICAL_TABLE_REPLACED_LATER, + 0/*start_ts*/, tablet_list.count(), round_, false/*is_report*/); // 4. The leader node of dest_ls registers the multi-source transaction // ObInnerSQLConnection->register_multi_source_data, and the type is TX_FINISH_TRANSFER_IN (two-way barrier) // The content of the log is src_ls_id, dest_ls_id, tablet_list. This step requires forcibly flushing the redo @@ -280,13 +290,22 @@ int ObTxFinishTransfer::do_tx_transfer_doing_(const ObTransferTaskID &task_id, c // 8. unlock table lock on src ls for tablet (must be successful) else if (OB_FAIL(ObTransferLockUtil::unlock_tablet_on_src_ls_for_table_lock( trans, tenant_id, src_ls_id, lock_owner_id, table_lock_tablet_list))) { + diagnose_result_msg_ = share::ObStorageHACostItemName::UNLOCK_TABLET_FOR_LOCK; LOG_WARN("failed to unlock tablet on src ls for table lock", KR(ret), K(tenant_id), K(src_ls_id), K(lock_owner_id), K(table_lock_tablet_list)); } // 9. unlock member list - else if (OB_FAIL( - unlock_ls_member_list_(tenant_id, dest_ls_id, member_list, lock_status, CONFIG_CHANGE_TIMEOUT))) { - LOG_WARN("failed to unlock ls member list", K(ret), K(tenant_id), K(dest_ls_id), K(member_list)); + else { + process_perf_diagnose_info_(ObStorageHACostItemName::UNLOCK_TABLET_FOR_LOCK, + 0/*start_ts*/, tablet_list.count(), round_, false/*is_report*/); + if (OB_FAIL( + unlock_ls_member_list_(tenant_id, dest_ls_id, member_list, lock_status, CONFIG_CHANGE_TIMEOUT))) { + diagnose_result_msg_ = share::ObStorageHACostItemName::UNLOCK_LS_MEMBER_LIST; + LOG_WARN("failed to unlock ls member list", K(ret), K(tenant_id), K(dest_ls_id), K(member_list)); + } else { + process_perf_diagnose_info_(ObStorageHACostItemName::UNLOCK_LS_MEMBER_LIST, + 0/*start_ts*/, tablet_list.count(), round_, false/*is_report*/); + } } #ifdef ERRSIM @@ -303,9 +322,15 @@ int ObTxFinishTransfer::do_tx_transfer_doing_(const ObTransferTaskID &task_id, c if (OB_TMP_FAIL(commit_trans_(is_commit, trans))) { if (OB_SUCCESS == ret) { ret = tmp_ret; + diagnose_result_msg_ = share::ObStorageHACostItemName::FINISH_TRANS_COMMIT; } } else if (is_commit) { round = 0; + ObTransferUtils::reset_related_info(dest_ls_id); + } + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::FINISH_TRANS_COMMIT, + 0/*start_ts*/, tablet_list.count(), round_, false/*is_report*/); } // 11. After the dest_ls leader succeeds, // it will report the corresponding results to RS. @@ -316,9 +341,28 @@ int ObTxFinishTransfer::do_tx_transfer_doing_(const ObTransferTaskID &task_id, c } } } - if (OB_TMP_FAIL(record_server_event_(ret, is_ready, tmp_round, start_scn))) { + if (OB_TMP_FAIL(record_server_event_(ret, is_ready, round_, start_scn))) { LOG_WARN("failed to record server event", K(tmp_ret), K(ret), K(is_ready)); } +#ifdef ERRSIM + if (OB_SUCC(ret)) { + ret = EN_TRANSFER_DIAGNOSE_DOING_FAILED ? : OB_SUCCESS; + if (OB_FAIL(ret)) { + STORAGE_LOG(WARN, "fake EN_TRANSFER_DIAGNOSE_DOING_FAILED", K(ret)); + } + } +#endif + if (OB_FAIL(ret)) { + if (OB_TMP_FAIL(ObStorageHADiagMgr::add_transfer_error_diagnose_info(task_id, dest_ls_id, + ObStorageHADiagTaskType::TRANSFER_DOING, round_, ret, diagnose_result_msg_))) { + LOG_WARN("failed to add error diagnose info", K(tmp_ret), K(ret), K(task_id), K(dest_ls_id), K(round_)); + } + } + + if (OB_SUCC(ret) && is_ready) { + process_perf_diagnose_info_(ObStorageHACostItemName::TRANSFER_FINISH_END, + 0/*start_ts*/, tablet_list.count(), round_, true/*is_report*/); + } return ret; } @@ -353,6 +397,11 @@ int ObTxFinishTransfer::unlock_src_and_dest_ls_member_list_(const uint64_t tenan } UNUSEDx(member_list); + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::UNLOCK_SRC_AND_DEST_LS_MEMBER_LIST, round_, false/*is_report*/); + } else { + diagnose_result_msg_ = share::ObStorageHACostItemName::UNLOCK_SRC_AND_DEST_LS_MEMBER_LIST; + } return ret; } @@ -417,6 +466,11 @@ int ObTxFinishTransfer::wait_transfer_tablet_status_normal_( } } } + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::WAIT_TRANSFER_TABLET_STATUS_NORMAL, 0/*start_ts*/, tablet_list.count(), round_, false/*is_report*/); + } else { + diagnose_result_msg_ = share::ObStorageHACostItemName::WAIT_TRANSFER_TABLET_STATUS_NORMAL; + } return ret; } @@ -503,6 +557,11 @@ int ObTxFinishTransfer::check_ls_logical_table_replaced(const uint64_t tenant_id } else { LOG_INFO("check ls logical table replace", K(tenant_id), K(dest_ls_id), K(addr_array), K(tablet_list), K(quorum), K(all_backfilled)); } + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::CHECK_LS_LOGICAL_TABLE_REPLACED, 0/*start_ts*/, tablet_list.count(), round_, false/*is_report*/); + } else { + diagnose_result_msg_ = share::ObStorageHACostItemName::CHECK_LS_LOGICAL_TABLE_REPLACED; + } return ret; } @@ -592,7 +651,7 @@ int ObTxFinishTransfer::do_tx_finish_transfer_in_(const ObTransferTaskID &task_i ret = OB_ERR_UNEXPECTED; LOG_WARN("conn should not be null", K(ret)); } else if (OB_FAIL(build_tx_finish_transfer_in_info_( - src_ls_id, dest_ls_id, start_scn, tablet_list, finish_transfer_in_info))) { + task_id, src_ls_id, dest_ls_id, start_scn, tablet_list, finish_transfer_in_info))) { LOG_WARN("failed to build tx finish transfer in info", K(ret), K(src_ls_id), @@ -618,6 +677,11 @@ int ObTxFinishTransfer::do_tx_finish_transfer_in_(const ObTransferTaskID &task_i } #endif } + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::REGISTER_TRANSFER_FINISH_IN, 0/*start_ts*/, tablet_list.count(), round_, false/*is_report*/); + } else { + diagnose_result_msg_ = share::ObStorageHACostItemName::REGISTER_TRANSFER_FINISH_IN; + } return ret; } @@ -641,7 +705,7 @@ int ObTxFinishTransfer::do_tx_finish_transfer_out_(const ObTransferTaskID &task_ ret = OB_ERR_UNEXPECTED; LOG_WARN("conn should not be null", K(ret)); } else if (OB_FAIL(build_tx_finish_transfer_out_info_( - src_ls_id, dest_ls_id, finish_scn, tablet_list, finish_transfer_out_info))) { + task_id, src_ls_id, dest_ls_id, finish_scn, tablet_list, finish_transfer_out_info))) { LOG_WARN("failed to build tx finish transfer out info", K(ret), K(src_ls_id), @@ -666,9 +730,13 @@ int ObTxFinishTransfer::do_tx_finish_transfer_out_(const ObTransferTaskID &task_ } } #endif - DEBUG_SYNC(AFTER_FINISH_TRANSFER_OUT); } + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::REGISTER_TRANSFER_OUT, 0/*start_ts*/, tablet_list.count(), round_, false/*is_report*/); + } else { + diagnose_result_msg_ = share::ObStorageHACostItemName::REGISTER_TRANSFER_OUT; + } return ret; } @@ -711,7 +779,11 @@ int ObTxFinishTransfer::wait_all_ls_replica_replay_scn_(const ObTransferTaskID & #endif DEBUG_SYNC(AFTER_DOING_TRANSFER_WAIT_REPLAY_SCN); - + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::WAIT_ALL_LS_REPLICA_REPLAY_FINISH_SCN, round_, false/*is_report*/); + } else { + diagnose_result_msg_ = share::ObStorageHACostItemName::WAIT_ALL_LS_REPLICA_REPLAY_FINISH_SCN; + } return ret; } @@ -822,10 +894,16 @@ int ObTxFinishTransfer::lock_ls_member_list_(const uint64_t tenant_id, const sha DEBUG_SYNC(AFTER_DOING_TRANSFER_LOCK_MEMBER_LIST); } + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::LOCK_LS_MEMBER_LIST_IN_DOING, round_, false/*is_report*/); + } else { + diagnose_result_msg_ = share::ObStorageHACostItemName::LOCK_LS_MEMBER_LIST_IN_DOING; + } return ret; } -int ObTxFinishTransfer::build_tx_finish_transfer_in_info_(const share::ObLSID &src_ls_id, +int ObTxFinishTransfer::build_tx_finish_transfer_in_info_( + const share::ObTransferTaskID &task_id, const share::ObLSID &src_ls_id, const share::ObLSID &dest_ls_id, const share::SCN &start_scn, const common::ObArray &tablet_list, ObTXFinishTransferInInfo &transfer_in_info) { @@ -838,6 +916,7 @@ int ObTxFinishTransfer::build_tx_finish_transfer_in_info_(const share::ObLSID &s transfer_in_info.src_ls_id_ = src_ls_id; transfer_in_info.dest_ls_id_ = dest_ls_id; transfer_in_info.start_scn_ = start_scn; + transfer_in_info.task_id_ = task_id; transfer_in_info.data_version_ = DEFAULT_MIN_DATA_VERSION; if (OB_FAIL(transfer_in_info.tablet_list_.assign(tablet_list))) { LOG_WARN("failed to assign tablet list", K(ret), K(tablet_list)); @@ -846,7 +925,8 @@ int ObTxFinishTransfer::build_tx_finish_transfer_in_info_(const share::ObLSID &s return ret; } -int ObTxFinishTransfer::build_tx_finish_transfer_out_info_(const share::ObLSID &src_ls_id, +int ObTxFinishTransfer::build_tx_finish_transfer_out_info_( + const share::ObTransferTaskID &task_id, const share::ObLSID &src_ls_id, const share::ObLSID &dest_ls_id, const share::SCN &finish_scn, const common::ObArray &tablet_list, ObTXFinishTransferOutInfo &transfer_out_info) { @@ -859,6 +939,7 @@ int ObTxFinishTransfer::build_tx_finish_transfer_out_info_(const share::ObLSID & transfer_out_info.src_ls_id_ = src_ls_id; transfer_out_info.dest_ls_id_ = dest_ls_id; transfer_out_info.finish_scn_ = finish_scn; + transfer_out_info.task_id_ = task_id; transfer_out_info.data_version_ = DEFAULT_MIN_DATA_VERSION; if (OB_FAIL(transfer_out_info.tablet_list_.assign(tablet_list))) { LOG_WARN("failed to assign tablet list", K(ret), K(tablet_list)); @@ -1111,5 +1192,47 @@ int ObTxFinishTransfer::write_server_event_(const int32_t result, const ObSqlStr return ret; } +void ObTxFinishTransfer::process_perf_diagnose_info_( + const ObStorageHACostItemName name, + const int64_t start_ts, + const int64_t tablet_count, + const int64_t round, const bool is_report) const +{ + int ret = OB_SUCCESS; + ObArenaAllocator alloc; + ObTransferPerfDiagInfo info; + ObStorageHATimestampItem item; + item.name_ = name; + item.type_ = ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE; + item.retry_id_ = round; + item.timestamp_ = ObTimeUtil::current_time(); + common::ObTabletID tablet_id; + share::ObStorageHADiagTaskKey key; + if (OB_FAIL(info.init(&alloc, MTL_ID()))) { + LOG_WARN("fail to init info", K(ret)); + } else if (OB_FAIL(info.add_item(item))) { + LOG_WARN("fail to add item", K(ret), K(item)); + } else if (OB_FAIL(ObStorageHADiagMgr::construct_diagnose_info_key(task_id_, ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE, + ObStorageHADiagTaskType::TRANSFER_DOING, ObStorageHADiagType::PERF_DIAGNOSE, round, tablet_id, key))) { + LOG_WARN("failed to construct error diagnose info key", K(ret), K(task_id_), K(round), K(tablet_id)); + } else if (OB_FAIL(ObStorageHADiagMgr::construct_diagnose_info(task_id_, dest_ls_id_, + ObStorageHADiagTaskType::TRANSFER_DOING, round, OB_SUCCESS, ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE, info))) { + LOG_WARN("failed to construct diagnose info", K(ret), K(task_id_), K(round), K(dest_ls_id_)); + } else if (OB_FAIL(ObStorageHADiagMgr::add_transfer_perf_diagnose_info(key, start_ts, tablet_count, is_report, info))) { + LOG_WARN("failed to add perf diagnose info", K(ret), K(key), K(tablet_count), K(round), K(info), K(start_ts), K(is_report)); + } +} + +void ObTxFinishTransfer::process_perf_diagnose_info_( + const ObStorageHACostItemName name, + const int64_t round, + const bool is_report) const +{ + // start_ts and tablet_count only need to be effective on the first call + // needn't to update in the later + process_perf_diagnose_info_(name, + 0/*start_ts*/, 0/*tablet_count*/, round_, false/*is_report*/); +} + } // namespace storage } // namespace oceanbase diff --git a/src/storage/high_availability/ob_finish_transfer.h b/src/storage/high_availability/ob_finish_transfer.h index 2026d7786..5908fd87e 100644 --- a/src/storage/high_availability/ob_finish_transfer.h +++ b/src/storage/high_availability/ob_finish_transfer.h @@ -162,7 +162,8 @@ private: // @param[in]: start_scn // @param[in]: tablet_list // @param[out]: transfer_out_info - int build_tx_finish_transfer_in_info_(const share::ObLSID &src_ls_id, const share::ObLSID &dest_ls_id, + int build_tx_finish_transfer_in_info_(const share::ObTransferTaskID &task_id, const share::ObLSID &src_ls_id, + const share::ObLSID &dest_ls_id, const share::SCN &start_scn, const common::ObArray &tablet_list, ObTXFinishTransferInInfo &transfer_out_info); @@ -172,7 +173,8 @@ private: // @param[in]: finish_scn // @param[in]: tablet_list // @param[out]: transfer_out_info - int build_tx_finish_transfer_out_info_(const share::ObLSID &src_ls_id, const share::ObLSID &dest_ls_id, + int build_tx_finish_transfer_out_info_(const share::ObTransferTaskID &task_id, const share::ObLSID &src_ls_id, + const share::ObLSID &dest_ls_id, const share::SCN &finish_scn, const common::ObArray &tablet_list, ObTXFinishTransferOutInfo &transfer_out_info); @@ -268,6 +270,17 @@ private: const ObSqlString &extra_info, const share::ObTransferStatus &status) const; + void process_perf_diagnose_info_( + const ObStorageHACostItemName name, + const int64_t start_ts, + const int64_t tablet_count, + const int64_t round, const bool is_report) const; + + void process_perf_diagnose_info_( + const ObStorageHACostItemName name, + const int64_t round, + const bool is_report) const; + private: static const int64_t DEFAULT_WAIT_INTERVAL_US = 10_ms; static const int64_t TASK_EXECUTE_LONG_WARNING_THRESHOLD = 60_min; @@ -281,6 +294,8 @@ private: mutable lib::ObMutex mutex_; common::ObThreadCond cond_; common::ObMySQLProxy *sql_proxy_; + int64_t round_; + share::ObStorageHACostItemName diagnose_result_msg_; DISALLOW_COPY_AND_ASSIGN(ObTxFinishTransfer); }; diff --git a/src/storage/high_availability/ob_storage_ha_diagnose_mgr.cpp b/src/storage/high_availability/ob_storage_ha_diagnose_mgr.cpp new file mode 100644 index 000000000..a29c79655 --- /dev/null +++ b/src/storage/high_availability/ob_storage_ha_diagnose_mgr.cpp @@ -0,0 +1,747 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE +#include "ob_storage_ha_diagnose_mgr.h" +#include "storage/meta_mem/ob_tenant_meta_mem_mgr.h" +#include "observer/omt/ob_tenant_config_mgr.h" + +namespace oceanbase +{ +using namespace share; + +namespace storage +{ +/** + * ------------------------------ObStorageHADiagIterator--------------------- + */ +ObStorageHADiagIterator::ObStorageHADiagIterator() + : keys_(), + cnt_(0), + cur_idx_(0), + is_opened_(false) +{ +} + +int ObStorageHADiagIterator::open(const ObStorageHADiagType &type) +{ + int ret = OB_SUCCESS; + if (is_opened_) { + ret = OB_INIT_TWICE; + LOG_WARN("The ObStorageHADiagIterator has been opened", K(ret)); + } else if (type >= ObStorageHADiagType::MAX_TYPE + || type < ObStorageHADiagType::ERROR_DIAGNOSE) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument!", K(ret), K(type)); + } else { + ObStorageHADiagMgr *mgr = nullptr; + if (OB_ISNULL(mgr = MTL(ObStorageHADiagMgr *))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get ObStorageHADiagMgr from MTL", K(ret)); + } else if (OB_FAIL(mgr->get_iter_array(type, keys_))) { + LOG_WARN("failed to get iter info", K(ret), K(type)); + } else { + cur_idx_ = 0; + cnt_ = keys_.count(); + is_opened_ = true; + } + } + return ret; +} + +int ObStorageHADiagIterator::get_next_info(ObStorageHADiagInfo &info) +{ + int ret = OB_SUCCESS; + if (!is_opened_) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (cur_idx_ >= cnt_) { + ret = OB_ITER_END; + } else if (cnt_ < 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected array count", K(ret), K(cnt_)); + } else { + ObStorageHADiagMgr *mgr = nullptr; + if (OB_ISNULL(mgr = MTL(ObStorageHADiagMgr *))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get ObStorageHADiagMgr from MTL", K(ret)); + } else { + while (OB_SUCC(ret) && cur_idx_ <= cnt_) { + if (cur_idx_ == cnt_) { + ret = OB_ITER_END; + break; + } else if (OB_FAIL(mgr->get(keys_.at(cur_idx_++), info))) { + //overwrite ret + if (OB_ENTRY_NOT_EXIST == ret) { + ret = OB_SUCCESS; + LOG_INFO("info has already been cleaned", K(ret), "key", keys_.at(cur_idx_ - 1)); + } else { + LOG_WARN("fail to get info", K(ret), "key", keys_.at(cur_idx_ - 1)); + } + } else { + break; + } + } + } + } + return ret; +} + +bool ObStorageHADiagIterator::is_valid() const +{ + return cur_idx_ >= 0 && cnt_ >= 0 && !keys_.empty(); +} + +void ObStorageHADiagIterator::reset() +{ + is_opened_= false; + keys_.reset(); + cnt_ = 0; + cur_idx_ = 0; +} + +int ObStorageHADiagIterator::get_cur_key(ObStorageHADiagTaskKey &key) const +{ + int ret = OB_SUCCESS; + key.reset(); + if (!is_opened_) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else { + if (0 == keys_.count() || cur_idx_ >= cnt_) { + ret = OB_ITER_END; + } else { + key = keys_.at(cur_idx_); + if (!key.is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("key is invalid", K(ret), K(key)); + } + } + } + return ret; +} + +bool ObStorageHADiagIterator::is_opened() const +{ + return is_opened_; +} +/** + * ------------------------------ObStorageHADiagMgr--------------------- + */ +ObStorageHADiagMgr::ObStorageHADiagMgr() + : is_inited_(false), + allocator_(), + task_list_(), + service_(nullptr), + lock_() +{ +} + +int ObStorageHADiagMgr::mtl_init(ObStorageHADiagMgr *&storage_ha_diag_mgr) +{ + return storage_ha_diag_mgr->init(MTL_ID(), INFO_PAGE_SIZE, INFO_MAX_SIZE); +} + +int ObStorageHADiagMgr::init( + const uint64_t tenant_id, + const int64_t page_size, + const int64_t max_size) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObStorageHADiagMgr has already been inited", K(ret)); + } else if (OB_INVALID_TENANT_ID == tenant_id + || page_size <= 0 + || max_size <= 0 + || max_size < page_size) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(tenant_id), K(page_size), K(max_size)); + } else { + int64_t upper_align_size = upper_align(max_size, page_size); + if (OB_FAIL(allocator_.init(ObMallocAllocator::get_instance(), + page_size, + lib::ObMemAttr(tenant_id, "SHADiagMgr"), + 0/*init_size*/, + upper_align_size, + upper_align_size))) { + LOG_WARN("failed to init allocator", K(ret), K(page_size), K(tenant_id), K(upper_align_size)); + } + } + if (OB_SUCC(ret)) { + service_ = &(ObStorageHADiagService::instance()); + is_inited_ = true; + } else { + clear(); + } + return ret; +} + +void ObStorageHADiagMgr::clear() +{ + is_inited_ = false; + allocator_.reset(); + task_list_.reset(); + service_ = nullptr; +} + +void ObStorageHADiagMgr::destroy() +{ + is_inited_ = false; + common::SpinWLockGuard guard(lock_); + clear_with_no_lock_(); + allocator_.reset(); + service_ = nullptr; +} + +void ObStorageHADiagMgr::clear_with_no_lock_() +{ + DLIST_FOREACH_REMOVESAFE_NORET(iter, task_list_) { + ObStorageHADiagTask *tmp_task = task_list_.remove(iter); + free_(tmp_task); + } + task_list_.clear(); +} + +int ObStorageHADiagMgr::add_or_update( + const ObStorageHADiagTaskKey &task_key, + const ObStorageHADiagInfo &input_info, + const bool is_report) +{ + int ret = OB_SUCCESS; + ObStorageHADiagInfo *info = nullptr; + ObStorageHADiagTask *task = nullptr; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (!task_key.is_valid() || !input_info.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(task_key), K(input_info)); + } else { + common::SpinWLockGuard guard(lock_); + if (OB_FAIL(get_task_(task_key, info))) { + //overwrite ret + if (OB_ENTRY_NOT_EXIST == ret) { + if (OB_FAIL(add_task_(task_key, input_info, task))) { + LOG_WARN("failed to add task", K(ret), K(task_key), K(input_info)); + } + } else { + LOG_WARN("failed to check task exist", K(ret), K(task_key)); + } + } else { + if (OB_FAIL(info->update(input_info))) { + LOG_WARN("failed to update info", K(ret), K(input_info)); + } + } + } + if (OB_FAIL(ret)) { + //do nothing + } else if (is_report) { + if (OB_FAIL(service_->add_task(task_key))) { + LOG_WARN("failed to add task to service", K(ret), K(task_key)); + } else if (REACH_TENANT_TIME_INTERVAL(1 * 100 * 1000L)) { + service_->wakeup(); + } + } + return ret; +} + +int ObStorageHADiagMgr::add_task_( + const ObStorageHADiagTaskKey &task_key, + const ObStorageHADiagInfo &input_info, + ObStorageHADiagTask *&out_task) +{ + int ret = OB_SUCCESS; + out_task = nullptr; + if (!task_key.is_valid() || !input_info.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(task_key), K(input_info)); + } else { + while (OB_SUCC(ret)) { + if (OB_FAIL(create_task_(task_key, input_info, out_task))) { + if (OB_ALLOCATE_MEMORY_FAILED == ret) { + // overwrite ret + if (!task_list_.is_empty() && OB_FAIL(purge_task_(task_list_.get_first()))) { + LOG_WARN("failed to purge task", K(ret), K(task_list_)); + } + } else { + LOG_WARN("failed to create task", K(ret), K(task_key), K(input_info)); + } + } else { + break; + } + } + if (OB_FAIL(ret)) { + //do nothing + } else if (!task_list_.add_last(out_task)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to add list", K(ret), KPC(out_task)); + } + } + return ret; +} +//TODO(zhixing.yh) modify task key relationship with info, info include task key +int ObStorageHADiagMgr::create_task_( + const ObStorageHADiagTaskKey &task_key, + const ObStorageHADiagInfo &input_info, + ObStorageHADiagTask *&task) +{ + int ret = OB_SUCCESS; + task = nullptr; + void *buf = nullptr; + ObStorageHADiagInfo *info = nullptr; + ObStorageHADiagTask *tmp_task = nullptr; + if (!task_key.is_valid() || !input_info.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(task_key), K(input_info)); + } else if (OB_FAIL(input_info.deep_copy(allocator_, info))) { + LOG_WARN("failed to alloc memory for diagnose info", K(ret), K(input_info)); + } else if (OB_ISNULL(buf = allocator_.alloc(sizeof(ObStorageHADiagTask)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc memory for ObStorageHADiagTask failed", K(ret)); + } else if (OB_ISNULL(tmp_task = new (buf) ObStorageHADiagTask())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("new storage ha monitor info is nullptr", K(ret)); + } else { + buf = nullptr; + if (OB_FAIL(tmp_task->key_.assign(task_key))) { + LOG_WARN("failed to assign task key", K(ret), K(task_key)); + } else { + tmp_task->val_ = info; + info->task_ = tmp_task; + info = nullptr; + } + if (OB_NOT_NULL(info)) { + tmp_task->~ObStorageHADiagTask(); + allocator_.free(tmp_task); + tmp_task = nullptr; + } + } + + if (OB_FAIL(ret)) { + if (OB_NOT_NULL(info)) { + free_info_(info); + } + if (OB_NOT_NULL(buf)) { + allocator_.free(buf); + buf = nullptr; + } + } else { + task = tmp_task; + } + + return ret; +} + +int ObStorageHADiagMgr::purge_task_(ObStorageHADiagTask *task) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(task)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("task is nullptr", K(ret)); + } else { + ObStorageHADiagTask *tmp_task = task_list_.remove(task); + free_(tmp_task); + } + return ret; +} + +int ObStorageHADiagMgr::del(const ObStorageHADiagTaskKey &task_key) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (!task_key.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("task key is invalid", K(ret), K(task_key)); + } else { + common::SpinWLockGuard guard(lock_); + DLIST_FOREACH_X(iter, task_list_, OB_SUCC(ret)) { + if (iter->key_ == task_key) { + if (OB_FAIL(purge_task_(iter))) { + LOG_WARN("failed to purge task", K(ret), K(task_key)); + } + break; + } + } + } + + return ret; +} + +int ObStorageHADiagMgr::get_iter_array( + const ObStorageHADiagType &type, + ObIArray &array) const +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (type < ObStorageHADiagType::ERROR_DIAGNOSE + || type >= ObStorageHADiagType::MAX_TYPE) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(type)); + } else { + common::SpinRLockGuard guard(lock_); + DLIST_FOREACH_X(it, task_list_, OB_SUCC(ret)) { + if (type == it->key_.diag_type_) { + if (OB_FAIL(array.push_back(it->key_))) { + LOG_WARN("failed to push back array.", K(ret), K(it->key_)); + } + } + } + } + return ret; +} + +int ObStorageHADiagMgr::get(const ObStorageHADiagTaskKey &key, ObStorageHADiagInfo &info) const +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (!key.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(key)); + } else { + common::SpinRLockGuard guard(lock_); + if (OB_FAIL(copy_with_nolock_(key, info))) { + LOG_WARN("failed to copy info", K(ret), K(key)); + } + } + return ret; +} + +int ObStorageHADiagMgr::get_task_( + const ObStorageHADiagTaskKey &key, + ObStorageHADiagInfo *&info) const +{ + int ret = OB_SUCCESS; + info = nullptr; + bool found_info = false; + DLIST_FOREACH_X(iter, task_list_, OB_SUCC(ret)) { + if (iter->key_ == key) { + info = iter->val_; + found_info = true; + break; + } + } + if (OB_SUCC(ret)) { + if (!found_info) { + ret = OB_ENTRY_NOT_EXIST; + } + } + return ret; +} + +int ObStorageHADiagMgr::copy_with_nolock_(const ObStorageHADiagTaskKey &key, ObStorageHADiagInfo &info) const +{ + int ret = OB_SUCCESS; + ObStorageHADiagInfo *task_info = nullptr; + if (OB_FAIL(get_task_(key, task_info))) { + LOG_WARN("fail to check task exist", K(ret), K(key)); + } else if (OB_FAIL(info.assign(*task_info))) { + LOG_WARN("fail to copy info", K(ret), KPC(task_info)); + } else if (ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE == key.module_ + && OB_FAIL(static_cast(info).copy_item_list( + static_cast(*task_info)))) { + LOG_WARN("fail to copy item list to info", K(ret), KPC(task_info)); + } + return ret; +} + +void ObStorageHADiagMgr::free_info_(ObStorageHADiagInfo *&info) +{ + if (OB_NOT_NULL(info)) { + info->~ObStorageHADiagInfo(); + allocator_.free(info); + info = nullptr; + } +} + +void ObStorageHADiagMgr::free_task_(ObStorageHADiagTask *&task) +{ + if (OB_NOT_NULL(task)) { + task->~ObStorageHADiagTask(); + allocator_.free(task); + task = nullptr; + } +} + +void ObStorageHADiagMgr::free_(ObStorageHADiagTask *&task) +{ + if (OB_NOT_NULL(task)) { + free_info_(task->val_); + free_task_(task); + } +} +//TODO(zhixing.yh) modify task key relationship with info, info include task key +//can not reset info, perf diagnose info perhaps already init +int ObStorageHADiagMgr::construct_diagnose_info( + const share::ObTransferTaskID task_id, const share::ObLSID &ls_id, + const share::ObStorageHADiagTaskType type, const int64_t retry_id, + const int result_code, const share::ObStorageHADiagModule module, + share::ObStorageHADiagInfo &info) +{ + int ret = OB_SUCCESS; + if (!task_id.is_valid() + || !ls_id.is_valid() + || type >= ObStorageHADiagTaskType::MAX_TYPE + || type < ObStorageHADiagTaskType::TRANSFER_START + || module >= ObStorageHADiagModule::MAX_MODULE + || module < ObStorageHADiagModule::TRANSFER_ERROR_DIAGNOSE + || retry_id < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(task_id), K(ls_id), K(type), K(module), K(retry_id), K(result_code)); + } else { + info.task_id_ = task_id.id(); + info.ls_id_ = ls_id; + info.module_ = module; + info.type_ = type; + info.retry_id_ = retry_id; + info.result_code_ = result_code; + } + return ret; +} + +int ObStorageHADiagMgr::construct_error_diagnose_info(share::ObTransferErrorDiagInfo &info) +{ + int ret = OB_SUCCESS; + info.timestamp_ = ObTimeUtility::current_time(); + info.trace_id_ = *ObCurTraceId::get_trace_id(); + info.thread_id_ = GETTID(); + return ret; +} + +int ObStorageHADiagMgr::construct_diagnose_info_key( + const share::ObTransferTaskID task_id, + const share::ObStorageHADiagModule module, + const share::ObStorageHADiagTaskType type, + const share::ObStorageHADiagType diag_type, + const int64_t retry_id, + const common::ObTabletID &tablet_id, + share::ObStorageHADiagTaskKey &key) +{ + int ret = OB_SUCCESS; + key.reset(); + // tablet_id do not check, it is only effective in backfill + if (!task_id.is_valid() + || module >= ObStorageHADiagModule::MAX_MODULE + || module < ObStorageHADiagModule::TRANSFER_ERROR_DIAGNOSE + || type >= ObStorageHADiagTaskType::MAX_TYPE + || type < ObStorageHADiagTaskType::TRANSFER_START + || diag_type >= ObStorageHADiagType::MAX_TYPE + || diag_type < ObStorageHADiagType::ERROR_DIAGNOSE + || retry_id < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(task_id), K(module), K(type), + K(diag_type), K(retry_id)); + } else { + key.tenant_id_ = MTL_ID(); + key.task_id_ = task_id.id(); + key.module_ = module; + key.type_ = type; + key.retry_id_ = retry_id; + key.diag_type_ = diag_type; + key.tablet_id_ = tablet_id; + } + return ret; +} + +//TODO(zhixing.yh) add result_msg +int ObStorageHADiagMgr::add_info_to_mgr( + const ObStorageHADiagInfo &info, + const ObStorageHADiagTaskKey &key, + const bool is_report) +{ + int ret = OB_SUCCESS; + ObStorageHADiagMgr *mgr = nullptr; + if (!info.is_valid() || !key.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(info), K(key)); + } else if (OB_ISNULL(mgr = MTL(ObStorageHADiagMgr *))) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "failed to get ObStorageHADiagMgr from mtl", K(ret)); + } else if (OB_FAIL(mgr->add_or_update(key, info, is_report))) { + STORAGE_LOG(WARN, "failed to add transfer diag info", K(ret), K(key), K(info)); + } else { + STORAGE_LOG(DEBUG, "success to add transfer diag info", K(ret), K(key), K(info)); + } + return ret; +} + +int ObStorageHADiagMgr::append_perf_diagnose_info( + const common::ObTabletID &tablet_id, + share::ObStorageHADiagInfo &info) +{ + int ret = OB_SUCCESS; + //do not check tablet_id invalid + info.tablet_id_ = tablet_id; + return ret; +} + +int ObStorageHADiagMgr::append_error_diagnose_info( + const common::ObTabletID &tablet_id, + const share::ObStorageHACostItemName result_msg, + share::ObStorageHADiagInfo &info) +{ + int ret = OB_SUCCESS; + // MAX_NAME also is valid + if (result_msg < share::ObStorageHACostItemName::TRANSFER_START_BEGIN + || result_msg > share::ObStorageHACostItemName::MAX_NAME) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(result_msg)); + } else { + //do not check tablet_id invalid + info.result_msg_ = result_msg; + info.tablet_id_ = tablet_id; + } + return ret; +} + +int ObStorageHADiagMgr::add_transfer_error_diagnose_info( + const share::ObTransferTaskID task_id, const share::ObLSID &ls_id, + const share::ObStorageHADiagTaskType type, const int64_t retry_id, + const int result_code, const common::ObTabletID &tablet_id, const share::ObStorageHACostItemName result_msg) +{ + int ret = OB_SUCCESS; + share::ObTransferErrorDiagInfo info; + share::ObStorageHADiagTaskKey key; + if (!task_id.is_valid() + || !ls_id.is_valid() + || type >= ObStorageHADiagTaskType::MAX_TYPE + || type < ObStorageHADiagTaskType::TRANSFER_START + || retry_id < 0 + || OB_SUCCESS == result_code) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(task_id), K(ls_id), K(type), K(retry_id), K(result_code)); + } else if (OB_FAIL(construct_diagnose_info(task_id, ls_id, type, retry_id, result_code, + ObStorageHADiagModule::TRANSFER_ERROR_DIAGNOSE, info))) { + LOG_WARN("failed to construct error diagnose info", K(ret), K(task_id), K(ls_id), K(type), K(retry_id), K(result_code)); + } else if (OB_FAIL(append_error_diagnose_info(tablet_id, result_msg, info))) { + LOG_WARN("fail to append diagnose info", K(ret), K(tablet_id), K(result_msg)); + } else if (OB_FAIL(construct_error_diagnose_info(info))) { + LOG_WARN("failed to construct error diagnose info", K(ret)); + } else if (OB_FAIL(construct_diagnose_info_key(task_id, ObStorageHADiagModule::TRANSFER_ERROR_DIAGNOSE, + type, ObStorageHADiagType::ERROR_DIAGNOSE, retry_id, tablet_id, key))) { + LOG_WARN("failed to construct error diagnose info key", K(ret), K(task_id), K(type), K(retry_id), K(tablet_id)); + } else if (OB_FAIL(add_info_to_mgr(info, key, true/*is_report*/))) { + LOG_WARN("failed to add error diagnose info", K(ret), K(info), K(key)); + } + return ret; +} + +int ObStorageHADiagMgr::add_transfer_error_diagnose_info( + const share::ObTransferTaskID task_id, const share::ObLSID &ls_id, + const share::ObStorageHADiagTaskType type, const int64_t retry_id, + const int result_code, const share::ObStorageHACostItemName result_msg) +{ + int ret = OB_SUCCESS; + common::ObTabletID unused_tablet_id; + if (OB_FAIL(add_transfer_error_diagnose_info(task_id, ls_id, type, retry_id, + result_code, unused_tablet_id, result_msg))) { + LOG_WARN("fail to add transfer error diagnose info", K(ret), K(task_id), K(ls_id), K(type), + K(retry_id), K(result_code), K(unused_tablet_id), K(result_msg)); + } + return ret; +} + +int ObStorageHADiagMgr::construct_perf_diagnose_info( + const int64_t tablet_count, + const int64_t start_timestamp, + share::ObTransferPerfDiagInfo &info) +{ + int ret = OB_SUCCESS; + if (start_timestamp < 0 || tablet_count < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(start_timestamp)); + } else { + info.timestamp_ = start_timestamp; + info.end_timestamp_ = ObTimeUtility::current_time(); + info.tablet_count_ = tablet_count; + } + return ret; +} + +int ObStorageHADiagMgr::add_transfer_perf_diagnose_info( + const share::ObStorageHADiagTaskKey &key, + const int64_t start_timestamp, const int64_t tablet_count, + const bool is_report, share::ObTransferPerfDiagInfo &info) +{ + int ret = OB_SUCCESS; + if (!key.is_valid() || start_timestamp < 0 || tablet_count < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(key), K(start_timestamp), K(tablet_count)); + } else if (OB_FAIL(construct_perf_diagnose_info(tablet_count, start_timestamp, info))) { + LOG_WARN("failed to construct perf diagnose info", K(ret), K(tablet_count), K(start_timestamp)); + } else if (OB_FAIL(add_info_to_mgr(info, key, is_report))) { + LOG_WARN("failed to add perf diagnose info", K(ret), K(info), K(key), K(is_report)); + } + return ret; +} + +int ObStorageHADiagMgr::add_key_to_service_(const ObIArray &task_keys) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < task_keys.count(); i++) { + const ObStorageHADiagTaskKey &key = task_keys.at(i); + if (OB_FAIL(service_->add_task(key))) { + LOG_WARN("failed to add task to service", K(ret), K(key)); + } + } + return ret; +} + +int ObStorageHADiagMgr::add_task_key_to_report_array_( + const share::ObTransferTaskID &task_id, + ObIArray &task_keys) const +{ + int ret = OB_SUCCESS; + task_keys.reset(); + if (!task_id.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(task_id)); + } else { + common::SpinRLockGuard guard(lock_); + DLIST_FOREACH_X(iter, task_list_, OB_SUCC(ret)) { + if ((ObStorageHADiagType::PERF_DIAGNOSE == iter->key_.diag_type_ + && iter->key_.task_id_ == task_id.id() + && (ObStorageHADiagTaskType::TRANSFER_START == iter->key_.type_ + || ObStorageHADiagTaskType::TRANSFER_DOING == iter->key_.type_ + || ObStorageHADiagTaskType::TRANSFER_ABORT == iter->key_.type_))) { + if (OB_FAIL(task_keys.push_back(iter->key_))) { + LOG_WARN("failed to add task", K(ret), K(iter->key_)); + } + } + } + } + return ret; +} + +int ObStorageHADiagMgr::report_task(const share::ObTransferTaskID &task_id) +{ + int ret = OB_SUCCESS; + ObArray task_keys; + if (!task_id.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(task_id)); + } else if (OB_FAIL(add_task_key_to_report_array_(task_id, task_keys))) { + LOG_WARN("failed to add task to report array", K(ret), K(task_id)); + } else if (OB_FAIL(add_key_to_service_(task_keys))) { + LOG_WARN("failed to add task to service", K(ret), K(task_keys)); + } + return ret; +} + +} +} diff --git a/src/storage/high_availability/ob_storage_ha_diagnose_mgr.h b/src/storage/high_availability/ob_storage_ha_diagnose_mgr.h new file mode 100644 index 000000000..5be2321ee --- /dev/null +++ b/src/storage/high_availability/ob_storage_ha_diagnose_mgr.h @@ -0,0 +1,158 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEABASE_STORAGE_HA_DIAGNOSE_ +#define OCEABASE_STORAGE_HA_DIAGNOSE_ + +#include "share/ob_storage_ha_diagnose_struct.h" +#include "ob_storage_ha_diagnose_service.h" +#include "storage/ob_i_store.h" +#include "lib/list/ob_dlist.h" + +namespace oceanbase +{ +using namespace share; +namespace storage +{ +class ObStorageHADiagIterator final +{ +public: + ObStorageHADiagIterator(); + ~ObStorageHADiagIterator() { reset(); } + int open(const ObStorageHADiagType &type); + int get_next_info(ObStorageHADiagInfo &info); + bool is_valid() const; + int get_cur_key(ObStorageHADiagTaskKey &key) const; + void reset(); + bool is_opened() const; + + TO_STRING_KV( + K_(cnt), + K_(cur_idx), + K_(is_opened)); + +private: + ObArray keys_; + int64_t cnt_; + int64_t cur_idx_; + bool is_opened_; + + DISALLOW_COPY_AND_ASSIGN(ObStorageHADiagIterator); +}; + +class ObStorageHADiagMgr final +{ +public: + ObStorageHADiagMgr(); + ~ObStorageHADiagMgr() { destroy(); } + static int mtl_init(ObStorageHADiagMgr *&storage_ha_diag_mgr); + int init( + const uint64_t tenant_id, + const int64_t page_size = INFO_PAGE_SIZE, + const int64_t max_size = INFO_MAX_SIZE); + + int add_or_update( + const ObStorageHADiagTaskKey &task_key, + const ObStorageHADiagInfo &input_info, + const bool is_report); + int del(const ObStorageHADiagTaskKey &task_keys); + int get_iter_array(const ObStorageHADiagType &type, ObIArray &array) const; + int get(const ObStorageHADiagTaskKey &key, ObStorageHADiagInfo &info) const; + void destroy(); + void clear(); + + static int construct_diagnose_info( + const share::ObTransferTaskID task_id, const share::ObLSID &ls_id, + const share::ObStorageHADiagTaskType type, const int64_t retry_id, + const int result_code, const share::ObStorageHADiagModule module, + share::ObStorageHADiagInfo &info); + static int construct_diagnose_info_key( + const share::ObTransferTaskID task_id, + const share::ObStorageHADiagModule module, + const share::ObStorageHADiagTaskType type, + const share::ObStorageHADiagType diag_type, + const int64_t retry_id, + const common::ObTabletID &tablet_id, + share::ObStorageHADiagTaskKey &key); + static int construct_error_diagnose_info(share::ObTransferErrorDiagInfo &info); + + static int add_info_to_mgr( + const ObStorageHADiagInfo &info, + const ObStorageHADiagTaskKey &key, + const bool is_report); + + static int append_perf_diagnose_info( + const common::ObTabletID &tablet_id, + share::ObStorageHADiagInfo &info); + + static int append_error_diagnose_info( + const common::ObTabletID &tablet_id, + const share::ObStorageHACostItemName result_msg, + share::ObStorageHADiagInfo &info); + + static int add_transfer_error_diagnose_info( + const share::ObTransferTaskID task_id, const share::ObLSID &ls_id, + const share::ObStorageHADiagTaskType type, const int64_t retry_id, + const int result_code, const share::ObStorageHACostItemName result_msg); + static int add_transfer_error_diagnose_info( + const share::ObTransferTaskID task_id, const share::ObLSID &ls_id, + const share::ObStorageHADiagTaskType type, const int64_t retry_id, + const int result_code, const common::ObTabletID &tablet_id, const share::ObStorageHACostItemName result_msg); + static int construct_perf_diagnose_info( + const int64_t tablet_count, + const int64_t start_timestamp, + share::ObTransferPerfDiagInfo &info); + static int add_transfer_perf_diagnose_info( + const share::ObStorageHADiagTaskKey &key, const int64_t start_timestamp, + const int64_t tablet_count, const bool is_report, share::ObTransferPerfDiagInfo &info); + int report_task(const share::ObTransferTaskID &task_id); + int add_task_key_to_report_array_( + const share::ObTransferTaskID &task_id, + ObIArray &task_keys) const; + int add_key_to_service_(const ObIArray &task_keys); + +public: + static const int64_t INFO_PAGE_SIZE = (1 << 16); // 64KB + static const int64_t INFO_PAGE_SIZE_LIMIT = (1 << 11); // 2KB + static const int64_t INFO_MAX_SIZE = 16LL * 1024LL * 1024LL; // 16MB // lowest + +private: + int create_task_( + const ObStorageHADiagTaskKey &task_key, + const ObStorageHADiagInfo &input_info, + ObStorageHADiagTask *&task); + int purge_task_(ObStorageHADiagTask *task); + + int add_task_( + const ObStorageHADiagTaskKey &task_key, + const ObStorageHADiagInfo &input_info, + ObStorageHADiagTask *&out_task); + void clear_with_no_lock_(); + int get_task_(const ObStorageHADiagTaskKey &key, ObStorageHADiagInfo *&info) const; + int copy_with_nolock_(const ObStorageHADiagTaskKey &key, ObStorageHADiagInfo &info) const; + void free_task_(ObStorageHADiagTask *&task); + void free_info_(ObStorageHADiagInfo *&val); + void free_(ObStorageHADiagTask *&task); + +private: + bool is_inited_; + common::ObFIFOAllocator allocator_; + ObDList task_list_; + ObStorageHADiagService *service_; + SpinRWLock lock_; + + DISALLOW_COPY_AND_ASSIGN(ObStorageHADiagMgr); +}; + +} +} +#endif diff --git a/src/storage/high_availability/ob_storage_ha_diagnose_service.cpp b/src/storage/high_availability/ob_storage_ha_diagnose_service.cpp new file mode 100644 index 000000000..d29a7565e --- /dev/null +++ b/src/storage/high_availability/ob_storage_ha_diagnose_service.cpp @@ -0,0 +1,690 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "ob_storage_ha_diagnose_service.h" +#include "ob_storage_ha_diagnose_mgr.h" +#include "observer/ob_server_struct.h" +#include "storage/tx_storage/ob_ls_handle.h" +#include "storage/tx_storage/ob_ls_service.h" +#include "observer/omt/ob_multi_tenant.h" + +namespace oceanbase +{ +using namespace share; +namespace storage +{ + +ObStorageHADiagService::ObStorageHADiagService() + : is_inited_(false), + thread_cond_(), + wakeup_cnt_(0), + op_(), + sql_proxy_(nullptr), + task_keys_(), + lock_(), + err_diag_end_timestamp_(0), + perf_diag_end_timestamp_(0) +{ +} + +int ObStorageHADiagService::init(common::ObMySQLProxy *sql_proxy) +{ + int ret = OB_SUCCESS; + if (is_inited_) { + ret = OB_INIT_TWICE; + LOG_WARN("storage ha diagnose service is aleady init", K(ret)); + } else if (OB_FAIL(op_.init())) { + LOG_WARN("failed to init op", K(ret)); + } else if (OB_FAIL(thread_cond_.init(ObWaitEventIds::HA_SERVICE_COND_WAIT))) { + LOG_WARN("failed to init ha service thread cond", K(ret)); + } else { + lib::ObMemAttr attr(OB_SERVER_TENANT_ID, "StorageHADiag"); + common::SpinWLockGuard guard(lock_); + if (OB_FAIL(task_keys_.create(REPORT_KEY_MAX_NUM, attr))) { + LOG_WARN("failed to init hash map", K(ret)); + } else { + sql_proxy_ = sql_proxy; + is_inited_ = true; + } + } + + return ret; +} + +void ObStorageHADiagService::wakeup() +{ + ObThreadCondGuard cond_guard(thread_cond_); + wakeup_cnt_++; + thread_cond_.signal(); +} + +void ObStorageHADiagService::destroy() +{ + COMMON_LOG(INFO, "ObStorageHADiagService starts to destroy"); + is_inited_ = false; + thread_cond_.destroy(); + wakeup_cnt_ = 0; + sql_proxy_ = nullptr; + common::SpinWLockGuard guard(lock_); + task_keys_.destroy(); + err_diag_end_timestamp_ = 0; + perf_diag_end_timestamp_ = 0; + COMMON_LOG(INFO, "ObStorageHADiagService destroyed"); +} + +void ObStorageHADiagService::stop() +{ + if (is_inited_) { + COMMON_LOG(INFO, "ObStorageHADiagService starts to stop"); + lib::ThreadPool::stop(); + wakeup(); + COMMON_LOG(INFO, "ObStorageHADiagService stopped"); + } +} + +void ObStorageHADiagService::wait() +{ + if (is_inited_) { + COMMON_LOG(INFO, "ObStorageHADiagService starts to wait"); + lib::ThreadPool::wait(); + COMMON_LOG(INFO, "ObStorageHADiagService finish to wait"); + } +} + +int ObStorageHADiagService::start() +{ + int ret = OB_SUCCESS; + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("storage ha monitor service do not init", K(ret)); + } else { + if (OB_FAIL(lib::ThreadPool::start())) { + COMMON_LOG(WARN, "ObStorageHADiagService start thread failed", K(ret)); + } else { + COMMON_LOG(INFO, "ObStorageHADiagService start"); + } + } + return ret; +} + +void ObStorageHADiagService::run1() +{ + int ret = OB_SUCCESS; + lib::set_thread_name("ObStorageHADiagService"); + uint64_t data_version = 0; + while (!has_set_stop()) { + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (observer::ObServiceStatus::SS_SERVING != GCTX.status_) { + ret = OB_SERVER_IS_INIT; + LOG_WARN("server is not serving", K(ret), K(GCTX.status_)); + } else if (OB_FAIL(GET_MIN_DATA_VERSION(OB_SYS_TENANT_ID, data_version))) { + LOG_WARN("failed to get data version", K(ret)); + } else if (!((data_version >= DATA_VERSION_4_2_2_0 && data_version < DATA_VERSION_4_3_0_0) + || data_version >= DATA_VERSION_4_3_1_0)) { /* [4.2.2, 4.3.0) && [4.3.1, ~) */ + //do nothing + } else if (OB_FAIL(do_clean_history_(ObStorageHADiagModule::TRANSFER_ERROR_DIAGNOSE, + err_diag_end_timestamp_))) { + LOG_WARN("failed to do clean error diagnose info", K(ret), K(err_diag_end_timestamp_)); + } else if (OB_FAIL(do_clean_history_(ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE, + perf_diag_end_timestamp_))) { + LOG_WARN("failed to do clean perf diagnose info", K(ret), K(perf_diag_end_timestamp_)); + } else if (OB_FAIL(do_report_())) { + LOG_WARN("failed to do report", K(ret)); + } else if (OB_FAIL(do_clean_transfer_related_info_())) { + LOG_WARN("failed to do clean transfer related info", K(ret)); + } + + ObThreadCondGuard guard(thread_cond_); + if (has_set_stop() || wakeup_cnt_ > 0) { + wakeup_cnt_ = 0; + } else { + int64_t wait_time_ms = 5 * 60 * 1000L; +#ifdef ERRSIM + wait_time_ms = GCONF.errsim_transfer_diagnose_server_wait_time / 1000; +#endif + thread_cond_.wait(wait_time_ms); + } + } +} + +ObStorageHADiagService &ObStorageHADiagService::instance() +{ + static ObStorageHADiagService storage_ha_diag_service; + return storage_ha_diag_service; +} + +int ObStorageHADiagService::get_info_from_type_(const ObStorageHADiagTaskKey &key, ObStorageHADiagInfo *&info, + ObTransferErrorDiagInfo &transfer_err_diag, + ObTransferPerfDiagInfo &transfer_perf_diag, + ObIAllocator &alloc) +{ + int ret = OB_SUCCESS; + info = nullptr; + if (key.is_valid()) { + switch(key.module_) { + case ObStorageHADiagModule::TRANSFER_ERROR_DIAGNOSE: { + info = &transfer_err_diag; + break; + } + case ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE: { + if (!transfer_perf_diag.is_inited() && OB_FAIL(transfer_perf_diag.init(&alloc, OB_SERVER_TENANT_ID))) { + LOG_WARN("fail to init info", K(ret)); + } else { + info = &transfer_perf_diag; + } + break; + } + default: { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid module", K(ret), K(key.module_)); + break; + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error", K(ret), K(key)); + } + + return ret; +} + +int ObStorageHADiagService::add_task(const ObStorageHADiagTaskKey &key) +{ + int ret =OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (!key.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(key)); + } else { + common::SpinWLockGuard guard(lock_); + int64_t now = ObTimeUtility::current_time(); + if (REPORT_KEY_MAX_NUM == task_keys_.size()) { + if (OB_FAIL(remove_oldest_without_lock_())) { + LOG_WARN("failed to remove first", K(ret), K(task_keys_.size())); + } + } + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(task_keys_.set_refactored(key, now))) { + LOG_WARN("failed to add key to hash map", K(ret), K(key), K(now)); + } + } + return ret; +} + +int ObStorageHADiagService::get_oldest_key_without_lock_(ObStorageHADiagTaskKey &key) const +{ + int ret = OB_SUCCESS; + key.reset(); + int64_t old_timestamp = INT64_MAX; + TaskKeyMap::const_iterator it = task_keys_.begin(); + for (; OB_SUCC(ret) && it != task_keys_.end(); ++it) { + int64_t timestamp = it->second; + if (timestamp < old_timestamp) { + old_timestamp = timestamp; + key = it->first; + } + } + if (!key.is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("key is invalid", K(ret), K(key)); + } + return ret; +} + +int ObStorageHADiagService::del_task_in_mgr_(const ObStorageHADiagTaskKey &task_key) const +{ + int ret = OB_SUCCESS; + ObStorageHADiagMgr *mgr = nullptr; + if (!task_key.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(task_key)); + } else { + MAKE_TENANT_SWITCH_SCOPE_GUARD(guard); + if (OB_FAIL(guard.switch_to(task_key.tenant_id_, false))) { + if (OB_TENANT_NOT_IN_SERVER == ret) { + // overwrite ret + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to switch tenant", K(ret), K(task_key.tenant_id_)); + } + } else { + if (OB_ISNULL(mgr = MTL(ObStorageHADiagMgr *))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get ObStorageHADiagMgr", K(ret), K(task_key.tenant_id_)); + } else if (OB_FAIL(mgr->del(task_key))) { + LOG_WARN("fail to del task key", K(ret), K(task_key)); + } + } + } + return ret; +} + +int ObStorageHADiagService::remove_oldest_without_lock_() +{ + int ret = OB_SUCCESS; + ObStorageHADiagTaskKey key; + if (OB_FAIL(get_oldest_key_without_lock_(key))) { + LOG_WARN("failed to get oldest key", K(ret), K(key)); + } else if (OB_FAIL(clean_task_key_without_lock_(key))) { + LOG_WARN("failed to clean task", K(ret), K(key)); + } else if (OB_FAIL(del_task_in_mgr_(key))) { + LOG_WARN("failed to del task in mgr", K(ret), K(key)); + } else { + LOG_INFO("succeed to remove task key", K(ret), K(key)); + } + return ret; +} + +int ObStorageHADiagService::do_clean_history_(const ObStorageHADiagModule module, int64_t &end_timestamp) +{ + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + ObArray timestamp_array; + int64_t delete_index = 0; + while (OB_SUCC(ret)) { + const int64_t now = ObTimeUtility::current_time(); + const int64_t delete_timestamp = now - GCONF._ha_diagnose_history_recycle_interval; + if (OB_FAIL(op_.get_batch_row_keys(*sql_proxy_, OB_SYS_TENANT_ID, + module, end_timestamp, timestamp_array))) { + LOG_WARN("failed to get row keys", K(ret), K(end_timestamp), K(timestamp_array)); + } else if (0 == timestamp_array.count()) { + break; + LOG_INFO("have not item need to clean ", K(ret), K(timestamp_array.count())); + } else if (timestamp_array.at(0) > delete_timestamp) { + break; + LOG_INFO("have not item need to clean ", K(ret), K(delete_timestamp), K(timestamp_array.at(0))); + } else if (OB_FAIL(op_.do_batch_delete(*sql_proxy_, OB_SYS_TENANT_ID, + module, timestamp_array, delete_timestamp, delete_index))) { + LOG_WARN("failed to delete batch row", K(ret), K(timestamp_array), K(delete_timestamp)); + } else { + end_timestamp = timestamp_array.at(delete_index); + } + } + end_timestamp = 0; + return ret; +} + +int ObStorageHADiagService::insert_inner_table_( + ObMySQLTransaction &trans, + ObStorageHADiagMgr *mgr, + const ObStorageHADiagTaskKey &task_key, + ObStorageHADiagInfo &info) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(mgr) || !task_key.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(task_key), KP(mgr)); + } else if (OB_FAIL(mgr->get(task_key, info))) { + //overwrite ret + if (OB_ENTRY_NOT_EXIST == ret) { + ret = OB_SUCCESS; + } + LOG_INFO("info has already been cleaned", K(ret), K(task_key)); + } else if (OB_FAIL(op_.insert_row(trans, OB_SYS_TENANT_ID, task_key.tenant_id_, info, info.module_))) { + LOG_WARN("failed to insert row", K(ret), K(info)); + } + return ret; +} + +int ObStorageHADiagService::deep_copy_keys_(TaskKeyArray &do_report_keys) const +{ + int ret =OB_SUCCESS; + do_report_keys.reset(); + common::SpinRLockGuard guard(lock_); + TaskKeyMap::const_iterator it = task_keys_.begin(); + for (int64_t i = 0; OB_SUCC(ret) && it != task_keys_.end() && i < ONCE_REPORT_KEY_MAX_NUM; ++it, ++i) { + const ObStorageHADiagTaskKey &key = it->first; + if (OB_FAIL(do_report_keys.push_back(key))) { + LOG_WARN("failed to add key to report keys", K(ret), K(key), K(do_report_keys)); + } + } + return ret; +} + +int ObStorageHADiagService::clean_task_key_without_lock_(const ObStorageHADiagTaskKey &task_key) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(task_keys_.erase_refactored(task_key))) { + if (OB_HASH_NOT_EXIST == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to erase_refactored", KR(ret), K(task_key)); + } + } + return ret; +} + +int ObStorageHADiagService::report_to_inner_table_( + ObMySQLTransaction &trans, + ObStorageHADiagMgr *mgr, + const ObStorageHADiagTaskKey &task_key) +{ + int ret = OB_SUCCESS; + ObStorageHADiagInfo *info = nullptr; + ObArenaAllocator alloc; + ObTransferErrorDiagInfo transfer_err_diag; + ObTransferPerfDiagInfo transfer_perf_diag; + if (OB_ISNULL(mgr) || !task_key.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(task_key)); + } else { + if (OB_FAIL(get_info_from_type_(task_key, info, transfer_err_diag, transfer_perf_diag, alloc))) { + LOG_WARN("failed to get info", K(ret), K(task_key), K(transfer_err_diag), K(transfer_perf_diag)); + } else if (OB_FAIL(insert_inner_table_(trans, mgr, task_key, *info))) { + LOG_WARN("failed to insert inner table", K(ret), K(task_key)); + } + } + return ret; +} + +int ObStorageHADiagService::start_trans_( + ObTimeoutCtx &timeout_ctx, + ObMySQLTransaction &trans) +{ + int ret = OB_SUCCESS; + int64_t stmt_timeout = 10_s; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObStorageHADiagService do not init", K(ret)); + } else if (OB_FAIL(timeout_ctx.set_trx_timeout_us(stmt_timeout))) { + LOG_WARN("fail to set trx timeout", K(ret), K(stmt_timeout)); + } else if (OB_FAIL(timeout_ctx.set_timeout(stmt_timeout))) { + LOG_WARN("set timeout context failed", K(ret)); + } else if (OB_FAIL(trans.start(sql_proxy_, OB_SYS_TENANT_ID))) { + LOG_WARN("failed to start trans", K(ret)); + } + return ret; +} + +int ObStorageHADiagService::commit_trans_( + const int32_t result, + ObMySQLTransaction &trans) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObStorageHADiagService do not init", K(ret)); + } else if (OB_FAIL(trans.end(OB_SUCCESS == result))) { + LOG_WARN("end transaction failed", K(ret)); + } + + return ret; +} + +int ObStorageHADiagService::report_batch_task_key_(ObMySQLTransaction &trans, const TaskKeyArray &task_keys) +{ + int ret = OB_SUCCESS; + uint64_t tenant_id = OB_INVALID_TENANT_ID; + ObStorageHADiagMgr *mgr = nullptr; + for (int64_t cur_index = 0; OB_SUCC(ret) && cur_index < task_keys.count(); cur_index++) { + const ObStorageHADiagTaskKey &task_key = task_keys.at(cur_index); + if (!task_key.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument!", K(ret), K(task_key)); + } else { + MAKE_TENANT_SWITCH_SCOPE_GUARD(guard); + if (task_key.tenant_id_ != tenant_id) { + if (OB_FAIL(guard.switch_to(task_key.tenant_id_, false))) { + if (OB_TENANT_NOT_IN_SERVER == ret) { + // overwrite ret + ret = OB_SUCCESS; + continue; + } else { + LOG_WARN("fail to switch tenant", K(ret), K(task_key.tenant_id_)); + } + } else { + tenant_id = task_key.tenant_id_; + if (OB_ISNULL(mgr = MTL(ObStorageHADiagMgr *))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get ObStorageHADiagMgr", K(ret), K(tenant_id)); + } + } + } + if (OB_FAIL(ret)) { + //do nothing + } else if (OB_FAIL(report_to_inner_table_(trans, mgr, task_key))) { + LOG_WARN("fail to report to inner table", K(ret), K(task_key)); + } + } + } + return ret; +} + +int ObStorageHADiagService::batch_del_task_in_mgr_(const TaskKeyArray &task_keys) +{ + int ret = OB_SUCCESS; + uint64_t tenant_id = OB_INVALID_TENANT_ID; + ObStorageHADiagMgr *mgr = nullptr; + for (int64_t cur_index = 0; OB_SUCC(ret) && cur_index < task_keys.count(); cur_index++) { + const ObStorageHADiagTaskKey &task_key = task_keys.at(cur_index); + if (!task_key.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument!", K(ret), K(task_key)); + } else { + MAKE_TENANT_SWITCH_SCOPE_GUARD(guard); + if (task_key.tenant_id_ != tenant_id) { + if (OB_FAIL(guard.switch_to(task_key.tenant_id_, false))) { + if (OB_TENANT_NOT_IN_SERVER == ret) { + // overwrite ret + ret = OB_SUCCESS; + continue; + } else { + LOG_WARN("fail to switch tenant", K(ret), K(task_key.tenant_id_)); + } + } else { + tenant_id = task_key.tenant_id_; + if (OB_ISNULL(mgr = MTL(ObStorageHADiagMgr *))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get ObStorageHADiagMgr", K(ret), K(tenant_id)); + } + } + } + if (OB_FAIL(ret)) { + //do nothing + } else if (OB_FAIL(mgr->del(task_key))) { + LOG_WARN("fail to del task key", K(ret), K(task_key)); + } else { + common::SpinWLockGuard guard(lock_); + if (OB_FAIL(clean_task_key_without_lock_(task_key))) { + LOG_WARN("failed to clean task keys", K(ret), K(task_key)); + } + } + } + } + return ret; +} + +int ObStorageHADiagService::report_process_(const TaskKeyArray &task_keys) +{ + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + uint64_t tenant_id = OB_INVALID_TENANT_ID; + ObMySQLTransaction trans; + ObTimeoutCtx timeout_ctx; + ObStorageHADiagMgr *mgr = nullptr; + if (OB_FAIL(start_trans_(timeout_ctx, trans))) { + LOG_WARN("failed to start trans", K(ret)); + } else { + if (OB_FAIL(report_batch_task_key_(trans, task_keys))) { + LOG_WARN("failed to batch report task key", K(ret), K(task_keys)); + } + if (OB_TMP_FAIL(commit_trans_(ret, trans))) { + LOG_WARN("failed to commit trans", K(tmp_ret), K(ret)); + if (OB_SUCCESS == ret) { + ret = tmp_ret; + } + } + if (OB_SUCCESS == ret && OB_SUCCESS == tmp_ret) { + if (OB_FAIL(batch_del_task_in_mgr_(task_keys))) { + LOG_WARN("failed to delete tasks", K(ret), K(task_keys)); + } + } + } + return ret; +} + +int ObStorageHADiagService::do_report_() +{ + int ret = OB_SUCCESS; + TaskKeyArray do_report_keys; + if (OB_FAIL(deep_copy_keys_(do_report_keys))) { + LOG_WARN("failed to copy keys", K(ret), K(do_report_keys)); + } else { + std::sort(do_report_keys.begin(), do_report_keys.end()); + if (OB_FAIL(report_process_(do_report_keys))) { + LOG_WARN("failed to do report", K(ret), K(do_report_keys)); + } + } + + return ret; +} + +int ObStorageHADiagService::do_clean_related_info_(ObLSService *ls_service, const share::ObLSID &ls_id, const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + ObLSHandle ls_handle; + ObLS *ls = nullptr; + share::ObTransferTaskID task_id; + int64_t result_count = 0; + if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::HA_MOD))) { + if (OB_ENTRY_NOT_EXIST == ret) { + // overwrite ret + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to get ls", K(ret), K(ls_id)); + } + } else if (OB_ISNULL(ls = ls_handle.get_ls())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls should not be NULL", K(ret), KP(ls), K(ls_id)); + } else if (OB_FAIL(ls->get_transfer_handler()->get_related_info_task_id(task_id))) { + LOG_WARN("failed to get transfer task id", K(ret), K(task_id)); + } else if (!task_id.is_valid()) { + // do nothing + } else if (OB_FAIL(op_.check_transfer_task_exist(*sql_proxy_, tenant_id, task_id, result_count))) { + if (OB_TABLE_NOT_EXIST == ret) { + // overwrite ret + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to check transfer task exist", K(ret), K(tenant_id), K(task_id), K(result_count)); + } + } else if (0 != result_count) { + // do not clean related info + } else if (OB_FAIL(ls->get_transfer_handler()->reset_related_info(task_id))) { + LOG_WARN("failed to reset transfer related info", K(ret), K(task_id)); + } + return ret; +} + +int ObStorageHADiagService::scheduler_clean_related_info_(ObLSService *ls_service, const ObIArray &ls_id_array, const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + if (ls_id_array.empty()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls id array empty", K(ret), K(ls_id_array)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < ls_id_array.count(); ++i) { + const share::ObLSID &ls_id = ls_id_array.at(i); + if (OB_FAIL(do_clean_related_info_(ls_service, ls_id, tenant_id))) { + LOG_WARN("failed to do clean related info", K(ret), K(ls_id), K(tenant_id)); + } + } + } + return ret; +} + +int ObStorageHADiagService::get_ls_id_array_(ObLSService *ls_service, ObIArray &ls_id_array) +{ + int ret = OB_SUCCESS; + ls_id_array.reset(); + common::ObSharedGuard ls_iter_guard; + ObLSIterator *ls_iter = nullptr; + if (OB_FAIL(ls_service->get_ls_iter(ls_iter_guard, ObLSGetMod::HA_MOD))) { + LOG_WARN("failed to get ls iter", K(ret)); + } else if (OB_ISNULL(ls_iter = ls_iter_guard.get_ptr())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls iter should not be NULL", K(ret)); + } else { + while (OB_SUCC(ret)) { + ObLS *ls = nullptr; + if (OB_FAIL(ls_iter->get_next(ls))) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + break; + } + } else if (OB_ISNULL(ls)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls should not be NULL", K(ret), KP(ls)); + } else if (OB_FAIL(ls_id_array.push_back(ls->get_ls_id()))) { + LOG_WARN("failed to push ls id into array", K(ret), KP(ls)); + } + } + } + return ret; +} + +int ObStorageHADiagService::do_clean_related_info_in_ls_(ObLSService *ls_service, const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + ObArray ls_id_array; + if (OB_FAIL(get_ls_id_array_(ls_service, ls_id_array))) { + LOG_WARN("failed to get ls id array", K(ret)); + } else if (ls_id_array.empty()) { + // do nothing + } else if (OB_FAIL(scheduler_clean_related_info_(ls_service, ls_id_array, tenant_id))) { + LOG_WARN("failed to scheduler clean related info", K(ret), K(ls_id_array), K(tenant_id)); + } + return ret; +} + +int ObStorageHADiagService::do_clean_transfer_related_info_() +{ + int ret = OB_SUCCESS; + common::ObArray tenant_ids; + ObLSService * ls_service = nullptr; + if (OB_FAIL(GCTX.omt_->get_mtl_tenant_ids(tenant_ids))) { + LOG_WARN("fail to get_mtl_tenant_ids", K(ret)); + } else { + for (int64_t i = 0; i < tenant_ids.size(); i++) { + uint64_t tenant_id = tenant_ids.at(i); + if (GCTX.omt_->is_available_tenant(tenant_id)) { + MAKE_TENANT_SWITCH_SCOPE_GUARD(guard); + if (OB_FAIL(guard.switch_to(tenant_id, false))) { + if (OB_TENANT_NOT_IN_SERVER == ret) { + // overwrite ret + ret = OB_SUCCESS; + continue; + } else { + LOG_WARN("fail to switch tenant", K(ret), K(tenant_id)); + } + } else { + if (OB_ISNULL(ls_service = (MTL(ObLSService *)))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls service should not be NULL", K(ret), KP(ls_service)); + } else if (OB_FAIL(do_clean_related_info_in_ls_(ls_service, tenant_id))) { + LOG_WARN("fail to clean related info", K(ret), K(tenant_id)); + } + } + } + } + } + return ret; +} + +} +} diff --git a/src/storage/high_availability/ob_storage_ha_diagnose_service.h b/src/storage/high_availability/ob_storage_ha_diagnose_service.h new file mode 100644 index 000000000..28089fcca --- /dev/null +++ b/src/storage/high_availability/ob_storage_ha_diagnose_service.h @@ -0,0 +1,104 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEABASE_STORAGE_HA_DIAG_SERVICE_ +#define OCEABASE_STORAGE_HA_DIAG_SERVICE_ + +#include "share/ob_storage_ha_diagnose_struct.h" +#include "share/ob_storage_ha_diagnose_operator.h" + +namespace oceanbase +{ +using namespace share; +namespace storage +{ + +class ObStorageHADiagService : public lib::ThreadPool +{ +public: + ObStorageHADiagService(); + virtual ~ObStorageHADiagService() {} + + int init(common::ObMySQLProxy *sql_proxy); + void run1() final; + void destroy(); + void wakeup(); + void stop(); + void wait(); + int start(); + +public: + int add_task(const ObStorageHADiagTaskKey &key); + static ObStorageHADiagService &instance(); + +private: + typedef ObArray TaskKeyArray; + typedef hash::ObHashMap TaskKeyMap; + static const int64_t ONCE_REPORT_KEY_MAX_NUM = 100; + static const int64_t REPORT_KEY_MAX_NUM = 1000; + +private: + int do_clean_history_(const ObStorageHADiagModule module, int64_t &end_timestamp); + int do_report_(); + int get_info_from_type_(const ObStorageHADiagTaskKey &key, ObStorageHADiagInfo *&info, + ObTransferErrorDiagInfo &transfer_err_diag, + ObTransferPerfDiagInfo &transfer_perf_diag, + ObIAllocator &alloc); + int insert_inner_table_( + ObMySQLTransaction &trans, + ObStorageHADiagMgr *mgr, + const ObStorageHADiagTaskKey &task_key, + ObStorageHADiagInfo &info); + int deep_copy_keys_(TaskKeyArray &do_report_keys) const; + int report_process_(const TaskKeyArray &task_keys); + int add_keys_( + const int64_t index, + const TaskKeyArray &task_keys, + TaskKeyArray &new_task_keys) const; + int clean_task_key_without_lock_(const ObStorageHADiagTaskKey &task_key); + int report_to_inner_table_( + ObMySQLTransaction &trans, + ObStorageHADiagMgr *mgr, + const ObStorageHADiagTaskKey &task_key); + int remove_oldest_without_lock_(); + int get_oldest_key_without_lock_(ObStorageHADiagTaskKey &key) const; + int start_trans_( + ObTimeoutCtx &timeout_ctx, + ObMySQLTransaction &trans); + int commit_trans_( + const int32_t result, + ObMySQLTransaction &trans); + int report_batch_task_key_(ObMySQLTransaction &trans, const TaskKeyArray &task_keys); + int batch_del_task_in_mgr_(const TaskKeyArray &task_keys); + int get_ls_id_array_(ObLSService *ls_service, ObIArray &ls_id_array); + int do_clean_related_info_(ObLSService *ls_service, const share::ObLSID &ls_id, const uint64_t tenant_id); + int scheduler_clean_related_info_(ObLSService *ls_service, const ObIArray &ls_id_array, const uint64_t tenant_id); + int do_clean_related_info_in_ls_(ObLSService *ls_service, const uint64_t tenant_id); + int do_clean_transfer_related_info_(); + int del_task_in_mgr_(const ObStorageHADiagTaskKey &task_key) const; + +private: + bool is_inited_; + common::ObThreadCond thread_cond_; + int64_t wakeup_cnt_; + ObStorageHADiagOperator op_; + common::ObMySQLProxy *sql_proxy_; + TaskKeyMap task_keys_; + SpinRWLock lock_; + int64_t err_diag_end_timestamp_; + int64_t perf_diag_end_timestamp_; + DISALLOW_COPY_AND_ASSIGN(ObStorageHADiagService); +}; + +} +} +#endif diff --git a/src/storage/high_availability/ob_storage_ha_utils.cpp b/src/storage/high_availability/ob_storage_ha_utils.cpp index 7862a8e99..e85967c39 100644 --- a/src/storage/high_availability/ob_storage_ha_utils.cpp +++ b/src/storage/high_availability/ob_storage_ha_utils.cpp @@ -34,7 +34,11 @@ #include "rootserver/ob_tenant_info_loader.h" #include "src/observer/omt/ob_tenant_config.h" #include "common/errsim_module/ob_errsim_module_type.h" +#include "ob_storage_ha_diagnose_mgr.h" +#include "share/ob_storage_ha_diagnose_struct.h" #include "common/ob_role.h" +#include "share/rc/ob_tenant_base.h" +#include "observer/omt/ob_tenant.h" using namespace oceanbase::share; @@ -702,5 +706,447 @@ int ObTransferUtils::check_ls_replay_scn( return ret; } +void ObTransferUtils::add_transfer_error_diagnose_in_backfill( + const share::ObLSID &dest_ls_id, + const share::SCN &log_sync_scn, + const int result_code, + const common::ObTabletID &tablet_id, + const share::ObStorageHACostItemName result_msg) +{ + int ret = OB_SUCCESS; + ObTransferHandler *transfer_handler = nullptr; + ObLSHandle ls_handle; + if (!dest_ls_id.is_valid() + || !log_sync_scn.is_valid() + || OB_SUCCESS == result_code + || !tablet_id.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(dest_ls_id), K(result_code), K(tablet_id), K(log_sync_scn)); + } else { + ObMigrationStatus migration_status = ObMigrationStatus::OB_MIGRATION_STATUS_MAX; + if (OB_FAIL(get_ls_migrate_status_(ls_handle, dest_ls_id, migration_status))) { //in migrate status + LOG_WARN("fail to get migration status", K(ret), K(dest_ls_id)); + } else if (OB_FAIL(get_transfer_handler_(ls_handle, dest_ls_id, transfer_handler))) { + LOG_WARN("failed to get ls", K(ret), K(ls_handle), K(dest_ls_id)); + } else if (OB_FAIL(transfer_handler->record_error_diagnose_info_in_backfill( + log_sync_scn, dest_ls_id, result_code, tablet_id, migration_status, result_msg))) { + LOG_WARN("failed to record diagnose info", K(ret), K(log_sync_scn), + K(dest_ls_id), K(result_code), K(tablet_id), K(migration_status), K(result_msg)); + } + } +} + +void ObTransferUtils::add_transfer_error_diagnose_in_replay( + const share::ObTransferTaskID &task_id, + const share::ObLSID &dest_ls_id, + const int result_code, + const bool clean_related_info, + const share::ObStorageHADiagTaskType type, + const share::ObStorageHACostItemName result_msg) +{ + int ret = OB_SUCCESS; + ObTransferHandler *transfer_handler = nullptr; + ObLSHandle ls_handle; + if (!task_id.is_valid()) { + // do nothing + } else if (!dest_ls_id.is_valid() + || type < ObStorageHADiagTaskType::TRANSFER_START + || type >= ObStorageHADiagTaskType::MAX_TYPE) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(dest_ls_id), K(type)); + } else if (OB_FAIL(get_transfer_handler_(ls_handle, dest_ls_id, transfer_handler))) { + LOG_WARN("failed to get ls", K(ret), K(ls_handle), K(dest_ls_id)); + } else if (OB_FAIL(transfer_handler->record_error_diagnose_info_in_replay( + task_id, dest_ls_id, result_code, clean_related_info, type, result_msg))) { + LOG_WARN("failed to record diagnose info", K(ret), K(dest_ls_id), + K(task_id), K(result_code), K(clean_related_info), K(type), K(result_msg)); + } +} + +void ObTransferUtils::set_transfer_related_info( + const share::ObLSID &dest_ls_id, + const share::ObTransferTaskID &task_id, + const share::SCN &start_scn) +{ + int ret = OB_SUCCESS; + ObTransferHandler *transfer_handler = nullptr; + ObLSHandle ls_handle; + if (!dest_ls_id.is_valid() || !task_id.is_valid() || !start_scn.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument!", K(ret), K(dest_ls_id), K(task_id), K(start_scn)); + } else if (OB_FAIL(get_transfer_handler_(ls_handle, dest_ls_id, transfer_handler))) { + LOG_WARN("failed to get ls", K(ret), K(ls_handle), K(dest_ls_id)); + } else if (OB_FAIL(transfer_handler->set_related_info(task_id, start_scn))) { + LOG_WARN("failed to set transfer related info", K(ret), K(dest_ls_id), K(task_id), K(start_scn)); + } +} + +int ObTransferUtils::get_ls_( + ObLSHandle &ls_handle, + const share::ObLSID &dest_ls_id, + ObLS *&ls) +{ + int ret = OB_SUCCESS; + ObLSService *ls_service = MTL(ObLSService*); + ls = nullptr; + if (!dest_ls_id.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument!", K(ret), K(dest_ls_id)); + } else if (OB_FAIL(ls_service->get_ls(dest_ls_id, ls_handle, ObLSGetMod::MDS_TABLE_MOD))) { + LOG_WARN("failed to get ls", K(ret), K(ls_handle), K(dest_ls_id)); + } else if (OB_ISNULL(ls = ls_handle.get_ls())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls is null", K(ret), K(dest_ls_id), K(ls_handle)); + } + return ret; +} + +int ObTransferUtils::get_ls_migrate_status_( + ObLSHandle &ls_handle, + const share::ObLSID &dest_ls_id, + ObMigrationStatus &migration_status) +{ + int ret = OB_SUCCESS; + migration_status = ObMigrationStatus::OB_MIGRATION_STATUS_MAX; + ObLS *ls = nullptr; + if (!dest_ls_id.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument!", K(ret), K(dest_ls_id)); + } else if (OB_FAIL(get_ls_(ls_handle, dest_ls_id, ls))) { + LOG_WARN("failed to get ls", K(ret), K(ls_handle), K(dest_ls_id)); + } else if (OB_FAIL(ls->get_migration_status(migration_status))) { + LOG_WARN("fail to get migration status", K(ret), K(dest_ls_id)); + } + + return ret; +} + +int ObTransferUtils::get_transfer_handler_( + ObLSHandle &ls_handle, + const share::ObLSID &dest_ls_id, + ObTransferHandler *&transfer_handler) +{ + int ret = OB_SUCCESS; + ObLS *ls = nullptr; + transfer_handler = nullptr; + if (!dest_ls_id.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument!", K(ret), K(dest_ls_id)); + } else if (OB_FAIL(get_ls_(ls_handle, dest_ls_id, ls))) { + LOG_WARN("failed to get ls", K(ret), K(ls_handle), K(dest_ls_id)); + } else { + transfer_handler = ls->get_transfer_handler(); + } + return ret; +} + +void ObTransferUtils::reset_related_info(const share::ObLSID &dest_ls_id) +{ + int ret = OB_SUCCESS; + ObTransferHandler *transfer_handler = nullptr; + ObLSHandle ls_handle; + if (!dest_ls_id.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument!", K(ret), K(dest_ls_id)); + } else if (OB_FAIL(get_transfer_handler_(ls_handle, dest_ls_id, transfer_handler))) { + LOG_WARN("failed to get ls", K(ls_handle), K(ret), K(dest_ls_id)); + } else { + transfer_handler->reset_related_info(); + } +} + +void ObTransferUtils::add_transfer_perf_diagnose_in_replay_( + const share::ObStorageHAPerfDiagParams ¶ms, + const int result, + const uint64_t timestamp, + const int64_t start_ts, + const bool is_report) +{ + int ret = OB_SUCCESS; + ObTransferHandler *transfer_handler = nullptr; + ObLSHandle ls_handle; + if (!params.is_valid() + || start_ts < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(params), K(start_ts)); + } else if (OB_FAIL(get_transfer_handler_(ls_handle, params.dest_ls_id_, transfer_handler))) { + LOG_WARN("failed to get ls", K(ret), K(ls_handle), K(params.dest_ls_id_)); + } else if (OB_FAIL(transfer_handler->record_perf_diagnose_info_in_replay( + params, result, timestamp, start_ts, is_report))) { + LOG_WARN("failed to record perf diagnose info", + K(ret), K(params), K(result), K(timestamp), K(start_ts), K(is_report)); + } +} + +void ObTransferUtils::add_transfer_perf_diagnose_in_backfill( + const share::ObStorageHAPerfDiagParams ¶ms, + const share::SCN &log_sync_scn, + const int result_code, + const uint64_t timestamp, + const int64_t start_ts, + const bool is_report) +{ + int ret = OB_SUCCESS; + ObTransferHandler *transfer_handler = nullptr; + ObLSHandle ls_handle; + if (!params.is_valid() + || !log_sync_scn.is_valid() + || start_ts < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(params), K(log_sync_scn), K(start_ts)); + } else { + ObMigrationStatus migration_status = ObMigrationStatus::OB_MIGRATION_STATUS_MAX; + if (OB_FAIL(get_ls_migrate_status_(ls_handle, params.dest_ls_id_, migration_status))) { //in migrate status + LOG_WARN("fail to get migration status", K(ret), K(params.dest_ls_id_)); + } else if (OB_FAIL(get_transfer_handler_(ls_handle, params.dest_ls_id_, transfer_handler))) { + LOG_WARN("failed to get ls", K(ret), K(ls_handle), K(params.dest_ls_id_)); + } else if (OB_FAIL(transfer_handler->record_perf_diagnose_info_in_backfill( + params, log_sync_scn, result_code, migration_status, timestamp, start_ts, is_report))) { + LOG_WARN("failed to record perf diagnose info", + K(ret), K(params), K(result_code), K(timestamp), K(start_ts), K(is_report)); + } + } +} + +void ObTransferUtils::construct_perf_diag_backfill_params_( + const share::ObLSID &dest_ls_id, + const common::ObTabletID &tablet_id, + const share::ObStorageHACostItemType item_type, + const share::ObStorageHACostItemName item_name, + share::ObStorageHAPerfDiagParams ¶ms) +{ + int ret = OB_SUCCESS; + if (!dest_ls_id.is_valid() + || !tablet_id.is_valid() + || item_type >= ObStorageHACostItemType::MAX_TYPE + || item_type < ObStorageHACostItemType::ACCUM_COST_TYPE + || item_name >= ObStorageHACostItemName::MAX_NAME + || item_name < ObStorageHACostItemName::TRANSFER_START_BEGIN) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(dest_ls_id), K(tablet_id), K(item_type), K(item_name)); + } else { + params.dest_ls_id_ = dest_ls_id; + params.task_type_ = ObStorageHADiagTaskType::TRANSFER_BACKFILLED; + params.item_type_ = item_type; + params.name_ = item_name; + params.tablet_id_ = tablet_id; + params.tablet_count_ = 1; + } +} + +void ObTransferUtils::construct_perf_diag_replay_params_( + const share::ObLSID &dest_ls_id, + const share::ObTransferTaskID &task_id, + const share::ObStorageHADiagTaskType task_type, + const share::ObStorageHACostItemType item_type, + const share::ObStorageHACostItemName item_name, + const int64_t tablet_count, + share::ObStorageHAPerfDiagParams ¶ms) +{ + int ret = OB_SUCCESS; + if (!dest_ls_id.is_valid() + || !task_id.is_valid() + || task_type >= ObStorageHADiagTaskType::MAX_TYPE + || task_type < ObStorageHADiagTaskType::TRANSFER_START + || item_type >= ObStorageHACostItemType::MAX_TYPE + || item_type < ObStorageHACostItemType::ACCUM_COST_TYPE + || item_name >= ObStorageHACostItemName::MAX_NAME + || item_name < ObStorageHACostItemName::TRANSFER_START_BEGIN + || tablet_count < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(task_id), K(dest_ls_id), K(task_type), K(item_type), K(item_name), K(tablet_count)); + } else { + params.dest_ls_id_ = dest_ls_id; + params.task_id_ = task_id; + params.task_type_ = task_type; + params.item_type_ = item_type; + params.name_ = item_name; + params.tablet_count_ = tablet_count; + } +} + +void ObTransferUtils::process_backfill_perf_diag_info( + const share::ObLSID &dest_ls_id, + const common::ObTabletID &tablet_id, + const share::ObStorageHACostItemType item_type, + const share::ObStorageHACostItemName name, + share::ObStorageHAPerfDiagParams ¶ms) +{ + int ret = OB_SUCCESS; + params.reset(); + if (!tablet_id.is_valid() + || !dest_ls_id.is_valid() + || item_type >= ObStorageHACostItemType::MAX_TYPE + || item_type < ObStorageHACostItemType::ACCUM_COST_TYPE + || name >= ObStorageHACostItemName::MAX_NAME + || name < ObStorageHACostItemName::TRANSFER_START_BEGIN) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(dest_ls_id), K(name), K(item_type), K(tablet_id)); + } else { + construct_perf_diag_backfill_params_( + dest_ls_id, + tablet_id, + item_type, + name, + params); + } +} + +void ObTransferUtils::process_start_out_perf_diag_info( + const ObTXStartTransferOutInfo &tx_start_transfer_out_info, + const share::ObStorageHACostItemType item_type, + const share::ObStorageHACostItemName name, + const int result, + const uint64_t timestamp, + const int64_t start_ts, + const bool is_report) +{ + int ret = OB_SUCCESS; + if (!tx_start_transfer_out_info.task_id_.is_valid()) { + // do nothing + } else if (!tx_start_transfer_out_info.is_valid() + || item_type >= ObStorageHACostItemType::MAX_TYPE + || item_type < ObStorageHACostItemType::ACCUM_COST_TYPE + || name >= ObStorageHACostItemName::MAX_NAME + || name < ObStorageHACostItemName::TRANSFER_START_BEGIN + || start_ts < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(tx_start_transfer_out_info), K(name), K(item_type), K(start_ts)); + } else { + share::ObStorageHAPerfDiagParams params; + construct_perf_diag_replay_params_( + tx_start_transfer_out_info.dest_ls_id_, + tx_start_transfer_out_info.task_id_, + ObStorageHADiagTaskType::TRANSFER_START_OUT, + item_type, + name, + tx_start_transfer_out_info.tablet_list_.count(), + params); + add_transfer_perf_diagnose_in_replay_( + params, + result, + timestamp, + start_ts, + is_report); + } +} + +void ObTransferUtils::process_start_in_perf_diag_info( + const ObTXStartTransferInInfo &tx_start_transfer_in_info, + const share::ObStorageHACostItemType item_type, + const share::ObStorageHACostItemName name, + const int result, + const uint64_t timestamp, + const int64_t start_ts, + const bool is_report) +{ + int ret = OB_SUCCESS; + if (!tx_start_transfer_in_info.task_id_.is_valid()) { + // do nothing + } else if (!tx_start_transfer_in_info.is_valid() + || item_type >= ObStorageHACostItemType::MAX_TYPE + || item_type < ObStorageHACostItemType::ACCUM_COST_TYPE + || name >= ObStorageHACostItemName::MAX_NAME + || name < ObStorageHACostItemName::TRANSFER_START_BEGIN + || start_ts < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(tx_start_transfer_in_info), K(name), K(item_type), K(start_ts)); + } else { + share::ObStorageHAPerfDiagParams params; + construct_perf_diag_replay_params_( + tx_start_transfer_in_info.dest_ls_id_, + tx_start_transfer_in_info.task_id_, + ObStorageHADiagTaskType::TRANSFER_START_IN, + item_type, + name, + tx_start_transfer_in_info.tablet_meta_list_.count(), + params); + add_transfer_perf_diagnose_in_replay_( + params, + result, + timestamp, + start_ts, + is_report); + } +} + +void ObTransferUtils::process_finish_in_perf_diag_info( + const ObTXFinishTransferInInfo &tx_finish_transfer_in_info, + const share::ObStorageHACostItemType item_type, + const share::ObStorageHACostItemName name, + const int result, + const uint64_t timestamp, + const int64_t start_ts, + const bool is_report) +{ + int ret = OB_SUCCESS; + if (!tx_finish_transfer_in_info.task_id_.is_valid()) { + // do nothing + } else if (!tx_finish_transfer_in_info.is_valid() + || item_type >= ObStorageHACostItemType::MAX_TYPE + || item_type < ObStorageHACostItemType::ACCUM_COST_TYPE + || name >= ObStorageHACostItemName::MAX_NAME + || name < ObStorageHACostItemName::TRANSFER_START_BEGIN + || start_ts < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(tx_finish_transfer_in_info), K(name), K(item_type), K(start_ts)); + } else { + share::ObStorageHAPerfDiagParams params; + construct_perf_diag_replay_params_( + tx_finish_transfer_in_info.dest_ls_id_, + tx_finish_transfer_in_info.task_id_, + ObStorageHADiagTaskType::TRANSFER_FINISH_IN, + item_type, + name, + tx_finish_transfer_in_info.tablet_list_.count(), + params); + add_transfer_perf_diagnose_in_replay_( + params, + result, + timestamp, + start_ts, + is_report); + } +} + +void ObTransferUtils::process_finish_out_perf_diag_info( + const ObTXFinishTransferOutInfo &tx_finish_transfer_out_info, + const share::ObStorageHACostItemType item_type, + const share::ObStorageHACostItemName name, + const int result, + const uint64_t timestamp, + const int64_t start_ts, + const bool is_report) +{ + int ret = OB_SUCCESS; + if (!tx_finish_transfer_out_info.task_id_.is_valid()) { + // do nothing + } else if (!tx_finish_transfer_out_info.is_valid() + || item_type >= ObStorageHACostItemType::MAX_TYPE + || item_type < ObStorageHACostItemType::ACCUM_COST_TYPE + || name >= ObStorageHACostItemName::MAX_NAME + || name < ObStorageHACostItemName::TRANSFER_START_BEGIN + || start_ts < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(tx_finish_transfer_out_info), K(name), K(item_type), K(start_ts)); + } else { + share::ObStorageHAPerfDiagParams params; + construct_perf_diag_replay_params_( + tx_finish_transfer_out_info.dest_ls_id_, + tx_finish_transfer_out_info.task_id_, + ObStorageHADiagTaskType::TRANSFER_FINISH_OUT, + item_type, + name, + tx_finish_transfer_out_info.tablet_list_.count(), + params); + add_transfer_perf_diagnose_in_replay_( + params, + result, + timestamp, + start_ts, + is_report); + } +} + } // end namespace storage } // end namespace oceanbase diff --git a/src/storage/high_availability/ob_storage_ha_utils.h b/src/storage/high_availability/ob_storage_ha_utils.h index 8261820d5..ecf21ae86 100644 --- a/src/storage/high_availability/ob_storage_ha_utils.h +++ b/src/storage/high_availability/ob_storage_ha_utils.h @@ -14,6 +14,8 @@ #include "common/ob_tablet_id.h" #include "lib/mysqlclient/ob_isql_client.h" #include "ob_storage_ha_struct.h" +#include "share/ob_storage_ha_diagnose_struct.h" +#include "ob_transfer_struct.h" namespace oceanbase { @@ -23,7 +25,8 @@ class SCN; } namespace storage { - +class ObBackfillTXCtx; +class ObTransferHandler; class ObStorageHAUtils { public: @@ -65,6 +68,7 @@ private: struct ObTransferUtils { +public: static bool is_need_retry_error(const int err); static int block_tx(const uint64_t tenant_id, const share::ObLSID &ls_id, const share::SCN >s_scn); static int kill_tx(const uint64_t tenant_id, const share::ObLSID &ls_id, const share::SCN >s_scn); @@ -84,6 +88,107 @@ struct ObTransferUtils const common::ObIArray &member_addr_list, ObTimeoutCtx &timeout_ctx, common::ObIArray &finished_addr_list); + static void add_transfer_error_diagnose_in_replay( + const share::ObTransferTaskID &task_id, + const share::ObLSID &dest_ls_id, + const int result_code, + const bool clean_related_info, + const share::ObStorageHADiagTaskType type, + const share::ObStorageHACostItemName result_msg); + static void add_transfer_error_diagnose_in_backfill( + const share::ObLSID &dest_ls_id, + const share::SCN &log_sync_scn, + const int result_code, + const common::ObTabletID &tablet_id, + const share::ObStorageHACostItemName result_msg); + static void set_transfer_related_info( + const share::ObLSID &dest_ls_id, + const share::ObTransferTaskID &task_id, + const share::SCN &start_scn); + static void reset_related_info(const share::ObLSID &dest_ls_id); + + static void add_transfer_perf_diagnose_in_backfill( + const share::ObStorageHAPerfDiagParams ¶ms, + const share::SCN &log_sync_scn, + const int result_code, + const uint64_t timestamp, + const int64_t start_ts, + const bool is_report); + + static void process_backfill_perf_diag_info( + const share::ObLSID &dest_ls_id, + const common::ObTabletID &tablet_id, + const share::ObStorageHACostItemType item_type, + const share::ObStorageHACostItemName name, + share::ObStorageHAPerfDiagParams ¶ms); + + static void process_start_out_perf_diag_info( + const ObTXStartTransferOutInfo &tx_start_transfer_out_info, + const share::ObStorageHACostItemType item_type, + const share::ObStorageHACostItemName name, + const int result, + const uint64_t timestamp, + const int64_t start_ts, + const bool is_report); + static void process_start_in_perf_diag_info( + const ObTXStartTransferInInfo &tx_start_transfer_in_info, + const share::ObStorageHACostItemType item_type, + const share::ObStorageHACostItemName name, + const int result, + const uint64_t timestamp, + const int64_t start_ts, + const bool is_report); + static void process_finish_out_perf_diag_info( + const ObTXFinishTransferOutInfo &tx_finish_transfer_out_info, + const share::ObStorageHACostItemType item_type, + const share::ObStorageHACostItemName name, + const int result, + const uint64_t timestamp, + const int64_t start_ts, + const bool is_report); + static void process_finish_in_perf_diag_info( + const ObTXFinishTransferInInfo &tx_finish_transfer_in_info, + const share::ObStorageHACostItemType item_type, + const share::ObStorageHACostItemName name, + const int result, + const uint64_t timestamp, + const int64_t start_ts, + const bool is_report); +private: + static int get_ls_( + ObLSHandle &ls_handle, + const share::ObLSID &dest_ls_id, + ObLS *&ls); + static int get_transfer_handler_( + ObLSHandle &ls_handle, + const share::ObLSID &dest_ls_id, + ObTransferHandler *&transfer_handler); + static int get_ls_migrate_status_( + ObLSHandle &ls_handle, + const share::ObLSID &dest_ls_id, + ObMigrationStatus &migration_status); + + static void construct_perf_diag_replay_params_( + const share::ObLSID &dest_ls_id_, + const share::ObTransferTaskID &task_id_, + const share::ObStorageHADiagTaskType task_type_, + const share::ObStorageHACostItemType item_type_, + const share::ObStorageHACostItemName item_name, + const int64_t tablet_count, + share::ObStorageHAPerfDiagParams ¶ms); + + static void add_transfer_perf_diagnose_in_replay_( + const share::ObStorageHAPerfDiagParams ¶ms, + const int result, + const uint64_t timestamp, + const int64_t start_ts, + const bool is_report); + static void construct_perf_diag_backfill_params_( + const share::ObLSID &dest_ls_id, + const common::ObTabletID &tablet_id, + const share::ObStorageHACostItemType item_type, + const share::ObStorageHACostItemName item_name, + share::ObStorageHAPerfDiagParams ¶ms); }; } // end namespace storage diff --git a/src/storage/high_availability/ob_tablet_backfill_tx.cpp b/src/storage/high_availability/ob_tablet_backfill_tx.cpp index 18322a0fa..c3fb6192c 100644 --- a/src/storage/high_availability/ob_tablet_backfill_tx.cpp +++ b/src/storage/high_availability/ob_tablet_backfill_tx.cpp @@ -18,6 +18,8 @@ #include "storage/ob_storage_struct.h" #include "storage/tablet/ob_tablet_iterator.h" #include "storage/tablet/ob_tablet.h" +#include "storage/high_availability/ob_storage_ha_diagnose_mgr.h" +#include "storage/high_availability/ob_storage_ha_utils.h" #include "storage/compaction/ob_partition_merger.h" namespace oceanbase @@ -442,7 +444,10 @@ int ObTabletBackfillTXTask::process() { int ret = OB_SUCCESS; LOG_INFO("start to do tablet backfill tx task", KPC(ha_dag_net_ctx_), K(tablet_info_), K(ls_id_)); - + const int64_t start_ts = ObTimeUtility::current_time(); + share::ObStorageHACostItemName diagnose_result_msg = share::ObStorageHACostItemName::TRANSFER_BACKFILL_START; + process_transfer_perf_diagnose_(start_ts, start_ts, false/*is_report*/, + share::ObStorageHACostItemName::TRANSFER_BACKFILL_START, ret); if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("tablet backfill tx task task do not init", K(ret)); @@ -455,12 +460,19 @@ int ObTabletBackfillTXTask::process() if (OB_FAIL(ret)) { int tmp_ret = OB_SUCCESS; ObTabletBackfillTXDag *tablet_backfill_tx_dag = static_cast(this->get_dag()); + share::ObLSID dest_ls_id; + share::SCN log_sync_scn; if (OB_ISNULL(tablet_backfill_tx_dag)) { tmp_ret = OB_ERR_UNEXPECTED; LOG_ERROR("tablet backfill tx dag should not be NULL", K(tmp_ret), KP(tablet_backfill_tx_dag)); } else if (OB_SUCCESS != (tmp_ret = tablet_backfill_tx_dag->set_result(ret))) { LOG_WARN("failed to set result", K(ret), KPC(ha_dag_net_ctx_), K(ls_id_), K(tablet_info_)); } + if (OB_TMP_FAIL(get_diagnose_support_info_(dest_ls_id, log_sync_scn))) { + LOG_WARN("failed to get diagnose support info", K(tmp_ret)); + } else { + ObTransferUtils::add_transfer_error_diagnose_in_backfill(dest_ls_id, log_sync_scn, ret, tablet_info_.tablet_id_, diagnose_result_msg); + } } return ret; } @@ -692,6 +704,7 @@ int ObTabletBackfillTXTask::generate_table_backfill_tx_task_( ObSSTableMetaHandle sst_meta_hdl; ObTabletTableBackfillTXTask *table_backfill_tx_task = nullptr; bool is_add_task = false; + if (OB_ISNULL(table)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("table should not be NULL or table type is unexpected", K(ret), KPC(table)); @@ -740,6 +753,41 @@ int ObTabletBackfillTXTask::generate_table_backfill_tx_task_( return ret; } +int ObTabletBackfillTXTask::get_diagnose_support_info_(share::ObLSID &dest_ls_id, share::SCN &log_sync_scn) const +{ + int ret = OB_SUCCESS; + dest_ls_id.reset(); + log_sync_scn.reset(); + if (ObIHADagNetCtx::TRANSFER_BACKFILL_TX != ha_dag_net_ctx_->get_dag_net_ctx_type()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected type", K(ret), "ctx type", ha_dag_net_ctx_->get_dag_net_ctx_type()); + } else { + dest_ls_id = static_cast(ha_dag_net_ctx_)->get_ls_id(); + log_sync_scn = backfill_tx_ctx_->log_sync_scn_; + } + return ret; +} + +void ObTabletBackfillTXTask::process_transfer_perf_diagnose_( + const int64_t timestamp, + const int64_t start_ts, + const bool is_report, + const ObStorageHACostItemName name, + const int result) const +{ + int ret = OB_SUCCESS; + share::ObLSID dest_ls_id; + share::SCN log_sync_scn; + if (OB_FAIL(get_diagnose_support_info_(dest_ls_id, log_sync_scn))) { + LOG_WARN("failed to get diagnose support info", K(ret)); + } else { + ObStorageHAPerfDiagParams params; + ObTransferUtils::process_backfill_perf_diag_info(dest_ls_id, tablet_info_.tablet_id_, + ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, name, params); + ObTransferUtils::add_transfer_perf_diagnose_in_backfill(params, log_sync_scn, result, timestamp, start_ts, is_report); + } +} + /******************ObTabletTableBackfillTXTask*********************/ ObTabletTableBackfillTXTask::ObTabletTableBackfillTXTask() : ObITask(TASK_TYPE_MIGRATE_PREPARE), @@ -802,7 +850,10 @@ int ObTabletTableBackfillTXTask::process() int ret = OB_SUCCESS; LOG_INFO("start do tablet table backfill tx task", K(ls_id_), K(tablet_id_), K(table_handle_)); bool need_merge = true; - + const int64_t start_ts = ObTimeUtility::current_time(); + share::ObStorageHACostItemName diagnose_result_msg = share::ObStorageHACostItemName::MAX_NAME; + process_transfer_perf_diagnose_(start_ts, start_ts, false/*is_report*/, + ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::TRANSFER_BACKFILLED_TABLE_BEGIN, ret); if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("tablet table backfill tx task do not init", K(ret)); @@ -813,11 +864,13 @@ int ObTabletTableBackfillTXTask::process() } else if (!need_merge) { LOG_INFO("tablet table no need merge", K(ret), K_(tablet_handle)); } else if (OB_FAIL(prepare_merge_ctx_())) { + diagnose_result_msg = share::ObStorageHACostItemName::PREPARE_MERGE_CTX; LOG_WARN("failed to prepare merge ctx", K(ret), KPC(this)); } else if (OB_FAIL(do_backfill_tx_())) { if (OB_NO_NEED_MERGE == ret) { ret = OB_SUCCESS; } else { + diagnose_result_msg = share::ObStorageHACostItemName::DO_BACKFILL_TX; LOG_WARN("failed to do backfill tx", K(ret), KPC(this)); } } @@ -832,6 +885,30 @@ int ObTabletTableBackfillTXTask::process() LOG_WARN("failed to set result", K(ret), KPC(ha_dag_net_ctx_), K(ls_id_), K(tablet_id_)); } } +#ifdef ERRSIM + if (OB_SUCC(ret)) { + ret = EN_TRANSFER_DIAGNOSE_BACKFILL_FAILED ? : OB_SUCCESS; + if (OB_FAIL(ret)) { + STORAGE_LOG(WARN, "fake EN_TRANSFER_DIAGNOSE_BACKFILL_FAILED", K(ret)); + } + } +#endif + if (OB_FAIL(ret)) { + int tmp_ret = OB_SUCCESS; + share::ObLSID dest_ls_id; + share::SCN log_sync_scn; + if (OB_TMP_FAIL(get_diagnose_support_info_(dest_ls_id, log_sync_scn))) { + LOG_WARN("failed to get diagnose support info", K(tmp_ret)); + } else { + ObTransferUtils::add_transfer_error_diagnose_in_backfill(dest_ls_id, log_sync_scn, ret, tablet_id_, diagnose_result_msg); + } + } + + if (OB_SUCC(ret)) { + const int64_t end_ts = ObTimeUtility::current_time(); + process_transfer_perf_diagnose_(end_ts, start_ts, false/*is_report*/, + ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::TX_BACKFILL, ret); + } return ret; } @@ -851,7 +928,7 @@ int ObTabletTableBackfillTXTask::prepare_merge_ctx_() } else { // init tablet merge dag param static_param.dag_param_.ls_id_ = ls_id_; - static_param.dag_param_.merge_type_ = table_handle_.get_table()->is_memtable() ? ObMergeType::MINI_MERGE : ObMergeType::BACKFILL_TX_MERGE; + static_param.dag_param_.merge_type_ = table_handle_.get_table()->is_memtable() ? compaction::ObMergeType::MINI_MERGE : compaction::ObMergeType::BACKFILL_TX_MERGE; static_param.report_ = nullptr; static_param.dag_param_.tablet_id_ = tablet_id_; // init version range and sstable @@ -984,6 +1061,42 @@ int ObTabletTableBackfillTXTask::update_merge_sstable_() return ret; } +int ObTabletTableBackfillTXTask::get_diagnose_support_info_(share::ObLSID &dest_ls_id, share::SCN &log_sync_scn) const +{ + int ret = OB_SUCCESS; + dest_ls_id.reset(); + log_sync_scn.reset(); + if (ObIHADagNetCtx::TRANSFER_BACKFILL_TX != ha_dag_net_ctx_->get_dag_net_ctx_type()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected type", K(ret), "ctx type", ha_dag_net_ctx_->get_dag_net_ctx_type()); + } else { + dest_ls_id = static_cast(ha_dag_net_ctx_)->get_ls_id(); + log_sync_scn = backfill_tx_ctx_->log_sync_scn_; + } + return ret; +} + +void ObTabletTableBackfillTXTask::process_transfer_perf_diagnose_( + const int64_t timestamp, + const int64_t start_ts, + const bool is_report, + const ObStorageHACostItemType type, + const ObStorageHACostItemName name, + const int result) const +{ + int ret = OB_SUCCESS; + share::ObLSID dest_ls_id; + share::SCN log_sync_scn; + if (OB_FAIL(get_diagnose_support_info_(dest_ls_id, log_sync_scn))) { + LOG_WARN("failed to get diagnose support info", K(ret)); + } else { + ObStorageHAPerfDiagParams params; + ObTransferUtils::process_backfill_perf_diag_info(dest_ls_id, tablet_id_, + type, name, params); + ObTransferUtils::add_transfer_perf_diagnose_in_backfill(params, log_sync_scn, result, timestamp, start_ts, is_report); + } +} + /******************ObFinishTabletBackfillTXTask*********************/ ObFinishTabletBackfillTXTask::ObFinishTabletBackfillTXTask() : ObITask(TASK_TYPE_MIGRATE_PREPARE), diff --git a/src/storage/high_availability/ob_tablet_backfill_tx.h b/src/storage/high_availability/ob_tablet_backfill_tx.h index cf9b0b742..e6ca33915 100644 --- a/src/storage/high_availability/ob_tablet_backfill_tx.h +++ b/src/storage/high_availability/ob_tablet_backfill_tx.h @@ -121,6 +121,13 @@ private: int get_all_backfill_tx_tables_( ObTablet *tablet, common::ObIArray &table_array); + int get_diagnose_support_info_(share::ObLSID &dest_ls_id, share::SCN &log_sync_scn) const; + void process_transfer_perf_diagnose_( + const int64_t timestamp, + const int64_t start_ts, + const bool is_report, + const ObStorageHACostItemName name, + const int result) const; private: bool is_inited_; ObBackfillTXCtx *backfill_tx_ctx_; @@ -148,6 +155,14 @@ private: int do_backfill_tx_(); int prepare_partition_merge_(); int update_merge_sstable_(); + int get_diagnose_support_info_(share::ObLSID &dest_ls_id, share::SCN &log_sync_scn) const; + void process_transfer_perf_diagnose_( + const int64_t timestamp, + const int64_t start_ts, + const bool is_report, + const ObStorageHACostItemType type, + const ObStorageHACostItemName name, + const int result) const; private: bool is_inited_; diff --git a/src/storage/high_availability/ob_transfer_backfill_tx.cpp b/src/storage/high_availability/ob_transfer_backfill_tx.cpp index 6c2f9e713..9e61c4b9a 100644 --- a/src/storage/high_availability/ob_transfer_backfill_tx.cpp +++ b/src/storage/high_availability/ob_transfer_backfill_tx.cpp @@ -24,6 +24,7 @@ #include "share/ob_debug_sync_point.h" #include "lib/utility/ob_tracepoint.h" #include "storage/tablet/ob_tablet.h" +#include "storage/high_availability/ob_storage_ha_utils.h" #include "storage/high_availability/ob_transfer_handler.h" namespace oceanbase @@ -1692,6 +1693,7 @@ int ObTransferReplaceTableTask::do_replace_logical_tables_(ObLS *ls) ObTablet *tablet = nullptr; ObTabletCreateDeleteMdsUserData user_data; DEBUG_SYNC(TRANSFER_REPLACE_TABLE_BEFORE); + share::ObStorageHACostItemName diagnose_result_msg = share::ObStorageHACostItemName::TRANSFER_REPLACE_BEGIN; if (OB_ISNULL(ls)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("ls is nullptr", K(ret)); @@ -1701,6 +1703,9 @@ int ObTransferReplaceTableTask::do_replace_logical_tables_(ObLS *ls) const ObTabletBackfillInfo tablet_info = ctx_->tablet_infos_.at(i); bool in_migration = false; ObMigrationStatus migration_status = ObMigrationStatus::OB_MIGRATION_STATUS_MAX; + const int64_t start_ts = ObTimeUtility::current_time(); + process_transfer_perf_diagnose_(start_ts, start_ts, false/*is_report*/, ls->get_ls_id(), + ctx_->backfill_scn_, tablet_info.tablet_id_, share::ObStorageHACostItemName::TRANSFER_REPLACE_BEGIN); if (ctx_->is_failed()) { int tmp_ret = OB_SUCCESS; if (OB_SUCCESS != (tmp_ret = ctx_->get_result(ret))) { @@ -1739,6 +1744,12 @@ int ObTransferReplaceTableTask::do_replace_logical_tables_(ObLS *ls) "dest_ls_id", ctx_->dest_ls_id_.id(), "tablet_id", tablet_info.tablet_id_.id()); #endif + const int64_t end_ts = ObTimeUtility::current_time(); + process_transfer_perf_diagnose_(end_ts, start_ts, true/*is_report*/, ls->get_ls_id(), + ctx_->backfill_scn_, tablet_info.tablet_id_, share::ObStorageHACostItemName::TRANSFER_REPLACE_END); + } + if (OB_FAIL(ret)) { + ObTransferUtils::add_transfer_error_diagnose_in_backfill(ls->get_ls_id(), ctx_->backfill_scn_, ret, tablet_info.tablet_id_, diagnose_result_msg); } } } @@ -1820,5 +1831,21 @@ int ObTransferReplaceTableTask::build_migration_param_( return ret; } +void ObTransferReplaceTableTask::process_transfer_perf_diagnose_( + const int64_t timestamp, + const int64_t start_ts, + const bool is_report, + const share::ObLSID &dest_ls_id, + const share::SCN &log_sync_scn, + const common::ObTabletID &tablet_id, + const share::ObStorageHACostItemName name) const +{ + int ret = OB_SUCCESS; + ObStorageHAPerfDiagParams params; + ObTransferUtils::process_backfill_perf_diag_info(dest_ls_id, tablet_id, + ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, name, params); + ObTransferUtils::add_transfer_perf_diagnose_in_backfill(params, log_sync_scn, ret, timestamp, start_ts, is_report); +} + } } diff --git a/src/storage/high_availability/ob_transfer_backfill_tx.h b/src/storage/high_availability/ob_transfer_backfill_tx.h index 7ebcc6dd4..010e87352 100644 --- a/src/storage/high_availability/ob_transfer_backfill_tx.h +++ b/src/storage/high_availability/ob_transfer_backfill_tx.h @@ -16,6 +16,7 @@ #include "ob_storage_ha_struct.h" #include "ob_storage_ha_dag.h" #include "share/scn.h" +#include "share/ob_storage_ha_diagnose_struct.h" namespace oceanbase { @@ -86,6 +87,7 @@ public: virtual int fill_comment(char *buf, const int64_t buf_len) const override; virtual bool is_valid() const; virtual DagNetCtxType get_dag_net_ctx_type() { return ObIHADagNetCtx::TRANSFER_BACKFILL_TX; } + const share::ObLSID &get_ls_id() const { return dest_ls_id_; } public: uint64_t tenant_id_; #ifdef ERRSIM @@ -250,6 +252,14 @@ private: int check_major_sstable_( const ObTablet *tablet, const ObTabletMemberWrapper &table_store_wrapper); + void process_transfer_perf_diagnose_( + const int64_t timestamp, + const int64_t start_ts, + const bool is_report, + const share::ObLSID &dest_ls_id, + const share::SCN &log_sync_scn, + const common::ObTabletID &tablet_id, + const share::ObStorageHACostItemName name) const; private: bool is_inited_; ObTransferBackfillTXCtx *ctx_; diff --git a/src/storage/high_availability/ob_transfer_handler.cpp b/src/storage/high_availability/ob_transfer_handler.cpp index d15a15249..8e164fef4 100644 --- a/src/storage/high_availability/ob_transfer_handler.cpp +++ b/src/storage/high_availability/ob_transfer_handler.cpp @@ -28,6 +28,8 @@ #include "storage/compaction/ob_tenant_tablet_scheduler.h" #include "ob_rebuild_service.h" #include "storage/tablet/ob_tablet.h" +#include "share/ob_storage_ha_diagnose_struct.h" +#include "storage/high_availability/ob_storage_ha_diagnose_mgr.h" #include "storage/tx/wrs/ob_weak_read_util.h" using namespace oceanbase::transaction; @@ -56,6 +58,10 @@ ERRSIM_POINT_DEF(EN_UPDATE_TRANSFER_TASK_FAILED); ERRSIM_POINT_DEF(EN_START_CAN_NOT_RETRY); ERRSIM_POINT_DEF(EN_MAKE_SRC_LS_REBUILD); ERRSIM_POINT_DEF(EN_INSERT_TRANSFER_START_FAILED); +ERRSIM_POINT_DEF(EN_TRANSFER_DIAGNOSE_START_FAILED); +ERRSIM_POINT_DEF(EN_TRANSFER_DIAGNOSE_DOING_FAILED); +ERRSIM_POINT_DEF(EN_TRANSFER_DIAGNOSE_ABORT_FAILED); +ERRSIM_POINT_DEF(EN_TRANSFER_DIAGNOSE_BACKFILL_FAILED); ObTransferHandler::ObTransferHandler() : is_inited_(false), @@ -68,6 +74,9 @@ ObTransferHandler::ObTransferHandler() transfer_worker_mgr_(), round_(0), gts_seq_(share::SCN::base_scn()), + related_info_(), + task_info_(), + diagnose_result_msg_(share::ObStorageHACostItemName::MAX_NAME), transfer_handler_lock_(), transfer_handler_enabled_(true) { @@ -95,6 +104,8 @@ int ObTransferHandler::init( KP(svr_rpc_proxy), KP(storage_rpc), KP(sql_proxy)); } else if (OB_FAIL(transfer_worker_mgr_.init(ls))) { LOG_WARN("failed to init transfer worker manager", K(ret), KP(ls)); + } else if (OB_FAIL(related_info_.init())) { + LOG_WARN("failed to init related_info"); } else { ls_ = ls; bandwidth_throttle_ = bandwidth_throttle; @@ -272,6 +283,9 @@ void ObTransferHandler::destroy() svr_rpc_proxy_ = nullptr; storage_rpc_ = nullptr; sql_proxy_ = nullptr; + related_info_.destroy(); + task_info_.reset(); + diagnose_result_msg_ = share::ObStorageHACostItemName::MAX_NAME; is_inited_ = false; } } @@ -325,6 +339,7 @@ int ObTransferHandler::flush(share::SCN &scn) int ObTransferHandler::process() { int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; ObCurTraceId::init(GCONF.self_addr_); common::SpinRLockGuard guard(transfer_handler_lock_); if (!is_inited_) { @@ -337,6 +352,9 @@ int ObTransferHandler::process() } else if (OB_FAIL(do_worker_transfer_())) { LOG_WARN("failed to do worker transfer", K(ret)); } + if (OB_TMP_FAIL(do_clean_diagnose_info_())) { + LOG_WARN("failed to clean diagnose info", K(tmp_ret)); + } return ret; } @@ -359,10 +377,11 @@ int ObTransferHandler::do_leader_transfer_() } else { LOG_WARN("failed to get transfer task", K(ret), KPC(ls_)); } + } else if (OB_FAIL(task_info_.assign(task_info))) { + LOG_WARN("fail to assign task info", K(ret), K(task_info)); } else { round_++; ObCurTraceId::set(task_info.trace_id_); - switch (task_info.status_) { case ObTransferStatus::START: { if (OB_FAIL(do_with_start_status_(task_info))) { @@ -437,6 +456,10 @@ int ObTransferHandler::do_with_start_status_(const share::ObTransferTaskInfo &ta bool is_leader = true; bool succ_block_tx = false; int64_t tablet_stop_begin = 0; + int64_t tmp_round = round_; + diagnose_result_msg_ = share::ObStorageHACostItemName::MAX_NAME; + process_perf_diagnose_info_(ObStorageHACostItemName::TRANSFER_START_BEGIN, + ObStorageHADiagTaskType::TRANSFER_START, start_ts, round_, false/*is_report*/); bool commit_succ = false; if (!is_inited_) { @@ -531,8 +554,15 @@ int ObTransferHandler::do_with_start_status_(const share::ObTransferTaskInfo &ta LOG_WARN("failed to commit trans", K(tmp_ret), K(ret)); if (OB_SUCCESS == ret) { ret = tmp_ret; + diagnose_result_msg_ = share::ObStorageHACostItemName::START_TRANS_COMMIT; } commit_succ = false; + } else if (OB_SUCCESS == ret) { + round_ = 0; + } + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::START_TRANS_COMMIT, + ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, tmp_round, false/*is_report*/); } int64_t trans_commit_end = ObTimeUtil::current_time(); if (new_transfer) { @@ -551,9 +581,13 @@ int ObTransferHandler::do_with_start_status_(const share::ObTransferTaskInfo &ta } else if (can_retry_(task_info, ret)) { LOG_INFO("transfer task can retry", K(ret), K(task_info)); if (!new_transfer && OB_TMP_FAIL(unblock_tx_(task_info.tenant_id_, task_info.src_ls_id_, gts_seq_))) { + diagnose_result_msg_ = share::ObStorageHACostItemName::UNLOCK_MEMBER_LIST_IN_START; LOG_WARN("failed to unblock tx", K(ret)); } else if (OB_TMP_FAIL(unlock_src_and_dest_ls_member_list_(task_info))) { LOG_WARN("failed to unlock src and dest ls member list", K(tmp_ret), K(ret), K(task_info)); + } else { + process_perf_diagnose_info_(ObStorageHACostItemName::UNLOCK_MEMBER_LIST_IN_START, + ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, tmp_round, false/*is_report*/); } ob_usleep(INTERVAL_US); } else if (OB_SUCCESS != (tmp_ret = update_transfer_status_aborted_(task_info, ret))) { @@ -581,6 +615,26 @@ int ObTransferHandler::do_with_start_status_(const share::ObTransferTaskInfo &ta LOG_WARN("failed to wakeup dest ls leader", K(tmp_ret), K(task_info)); } } +#ifdef ERRSIM + if (OB_SUCC(ret)) { + ret = EN_TRANSFER_DIAGNOSE_START_FAILED ? : OB_SUCCESS; + if (OB_FAIL(ret)) { + STORAGE_LOG(WARN, "fake EN_TRANSFER_DIAGNOSE_START_FAILED", K(ret)); + } + } +#endif + if (OB_FAIL(ret) && task_info.is_valid()) { + if (OB_TMP_FAIL(ObStorageHADiagMgr::add_transfer_error_diagnose_info(task_info.task_id_, task_info.dest_ls_id_, + share::ObStorageHADiagTaskType::TRANSFER_START, tmp_round, ret, diagnose_result_msg_))) { + LOG_WARN("failed to add error diagnose info", K(tmp_ret), K(ret), + K(task_info.task_id_), K(task_info.dest_ls_id_), K(tmp_round)); + } + } + + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::TRANSFER_START_END, + ObStorageHADiagTaskType::TRANSFER_START, start_ts, tmp_round, true/*is_report*/); + } LOG_INFO("[TRANSFER] finish do with start status", K(ret), K(task_info), "cost_ts", ObTimeUtil::current_time() - start_ts); return ret; } @@ -653,7 +707,12 @@ int ObTransferHandler::lock_src_and_dest_ls_member_list_( "member_list_is_same", is_same); #endif DEBUG_SYNC(AFTER_TRANSFER_START_LOCK_MEMBER_LIST); - + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::LOCK_MEMBER_LIST, + ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/); + } else { + diagnose_result_msg_ = share::ObStorageHACostItemName::LOCK_MEMBER_LIST; + } return ret; } @@ -828,6 +887,12 @@ int ObTransferHandler::check_src_ls_has_active_trans_( #endif LOG_INFO("get active trans count", K(src_ls_id), K(active_trans_count)); } + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::CHECK_SRC_LS_HAS_ACTIVE_TRANS, + ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/); + } else { + diagnose_result_msg_ = share::ObStorageHACostItemName::CHECK_SRC_LS_HAS_ACTIVE_TRANS; + } return ret; } @@ -927,6 +992,12 @@ int ObTransferHandler::check_start_status_transfer_tablets_( #endif } + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::CHECK_START_STATUS_TRANSFER_TABLETS, + ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/); + } else { + diagnose_result_msg_ = share::ObStorageHACostItemName::CHECK_START_STATUS_TRANSFER_TABLETS; + } return ret; } @@ -1307,6 +1378,7 @@ int ObTransferHandler::do_tx_start_transfer_out_( start_transfer_out_info.data_end_scn_ = data_end_scn; // TODO lana optimise transfer_epoch value start_transfer_out_info.transfer_epoch_ = task_info.task_id_.id(); + start_transfer_out_info.task_id_ = task_info.task_id_; start_transfer_out_info.data_version_ = DEFAULT_MIN_DATA_VERSION; if (OB_FAIL(start_transfer_out_info.tablet_list_.assign(task_info.tablet_list_))) { LOG_WARN("failed to assign transfer tablet list", K(ret), K(task_info)); @@ -1347,6 +1419,12 @@ int ObTransferHandler::do_tx_start_transfer_out_( DEBUG_SYNC(AFTER_START_TRANSFER_OUT); } + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::REGISTER_TRANSFER_START_OUT, + ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/); + } else { + diagnose_result_msg_ = share::ObStorageHACostItemName::REGISTER_TRANSFER_START_OUT; + } return ret; } @@ -1401,6 +1479,12 @@ int ObTransferHandler::get_start_transfer_out_scn_( #endif DEBUG_SYNC(AFTER_START_TRANSFER_GET_START_SCN); } + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::DEST_LS_GET_START_SCN, + ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/); + } else { + diagnose_result_msg_ = share::ObStorageHACostItemName::DEST_LS_GET_START_SCN; + } return ret; } @@ -1512,7 +1596,12 @@ int ObTransferHandler::wait_src_ls_replay_to_start_scn_( } } #endif - + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::WAIT_SRC_LS_REPLAY_TO_START_SCN, + ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/); + } else { + diagnose_result_msg_ = share::ObStorageHACostItemName::WAIT_SRC_LS_REPLAY_TO_START_SCN; + } return ret; } @@ -1555,6 +1644,13 @@ int ObTransferHandler::precheck_ls_replay_scn_(const share::ObTransferTaskInfo & ret = OB_TRANSFER_CANNOT_START; LOG_WARN("transfer precheck timeout, cannot start transfer in", K(ret), K(task_info)); } + + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::PRECHECK_LS_REPALY_SCN, + ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/); + } else { + diagnose_result_msg_ = share::ObStorageHACostItemName::PRECHECK_LS_REPALY_SCN; + } return ret; } @@ -1680,7 +1776,12 @@ int ObTransferHandler::get_transfer_tablets_meta_( } DEBUG_SYNC(AFTER_START_TRANSFER_GET_TABLET_META); - + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::SRC_LS_GET_TABLET_META, + ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/); + } else { + diagnose_result_msg_ = share::ObStorageHACostItemName::SRC_LS_GET_TABLET_META; + } return ret; } @@ -1752,6 +1853,7 @@ int ObTransferHandler::do_tx_start_transfer_in_( start_transfer_in_info.src_ls_id_ = task_info.src_ls_id_; start_transfer_in_info.dest_ls_id_ = task_info.dest_ls_id_; start_transfer_in_info.start_scn_ = start_scn; + start_transfer_in_info.task_id_ = task_info.task_id_; start_transfer_in_info.data_version_ = DEFAULT_MIN_DATA_VERSION; if (timeout_ctx.is_timeouted()) { @@ -1796,6 +1898,12 @@ int ObTransferHandler::do_tx_start_transfer_in_( LOG_INFO("[TRANSFER_BLOCK_TX] do tx start transfer in", K(ret), "cost", ObTimeUtil::current_time() - start_ts); DEBUG_SYNC(AFTER_START_TRANSFER_IN); } + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::REGISTER_TRANSFER_START_IN, + ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/); + } else { + diagnose_result_msg_ = share::ObStorageHACostItemName::REGISTER_TRANSFER_START_IN; + } return ret; } @@ -1885,6 +1993,12 @@ int ObTransferHandler::update_all_tablet_to_ls_( } } + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::UPDATE_ALL_TABLET_TO_LS, + ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/); + } else { + diagnose_result_msg_ = share::ObStorageHACostItemName::UPDATE_ALL_TABLET_TO_LS; + } return ret; } @@ -1910,7 +2024,12 @@ int ObTransferHandler::lock_tablet_on_dest_ls_for_table_lock_( } else { LOG_INFO("[TRANSFER] success lock tablet on dest ls for table lock", "cost", ObTimeUtil::current_time() - start_ts); } - + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::LOCK_TABLET_ON_DEST_LS_FOR_TABLE_LOCK, + ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/); + } else { + diagnose_result_msg_ = share::ObStorageHACostItemName::LOCK_TABLET_ON_DEST_LS_FOR_TABLE_LOCK; + } return ret; } @@ -1974,6 +2093,7 @@ int ObTransferHandler::update_transfer_status_aborted_( { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; + int64_t tmp_round = round_; const share::ObTransferStatus next_status(ObTransferStatus::ABORTED); ObTimeoutCtx timeout_ctx; ObMySQLTransaction trans; @@ -2010,8 +2130,16 @@ int ObTransferHandler::update_transfer_status_aborted_( if (OB_SUCCESS == ret) { ret = tmp_ret; } + } else if (OB_SUCCESS == ret) { + round_ = 0; } } + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::TRANSFER_START_END, + ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, tmp_round, true/*is_report*/); + } else { + diagnose_result_msg_ = share::ObStorageHACostItemName::TRANSFER_START_END; + } return ret; } @@ -2128,6 +2256,10 @@ int ObTransferHandler::do_with_aborted_status_( ObTimeoutCtx timeout_ctx; ObMySQLTransaction trans; const int64_t tmp_round = round_; + const int64_t start_ts = ObTimeUtil::current_time(); + diagnose_result_msg_ = share::ObStorageHACostItemName::MAX_NAME; + process_perf_diagnose_info_(ObStorageHACostItemName::TRANSFER_ABORT_BEGIN, + ObStorageHADiagTaskType::TRANSFER_ABORT, start_ts, round_, false/*is_report*/); if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("transfer handler do not init", K(ret)); @@ -2145,17 +2277,26 @@ int ObTransferHandler::do_with_aborted_status_( } else if (OB_FAIL(update_transfer_status_(task_info, next_status, scn, result, trans))) { LOG_WARN("failed to update transfer status", K(ret), K(task_info), K(next_status)); } else if (OB_FAIL(unlock_src_and_dest_ls_member_list_(task_info))) { + diagnose_result_msg_ = share::ObStorageHACostItemName::UNLOCK_MEMBER_LIST_IN_ABORT; LOG_WARN("failed to unlock src and dest ls member list", K(ret), K(task_info)); + } else { + process_perf_diagnose_info_(ObStorageHACostItemName::UNLOCK_MEMBER_LIST_IN_ABORT, + ObStorageHADiagTaskType::TRANSFER_ABORT, start_ts, round_, false/*is_report*/); } if (OB_TMP_FAIL(commit_trans_(ret, trans))) { LOG_WARN("failed to commit trans", K(tmp_ret), K(ret)); if (OB_SUCCESS == ret) { ret = tmp_ret; + diagnose_result_msg_ = share::ObStorageHACostItemName::ABORT_TRANS_COMMIT; } } else if (OB_SUCCESS == ret) { round_ = 0; } + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::ABORT_TRANS_COMMIT, + ObStorageHADiagTaskType::TRANSFER_ABORT, start_ts, tmp_round, false/*is_report*/); + } } } @@ -2173,6 +2314,26 @@ int ObTransferHandler::do_with_aborted_status_( } else if (OB_SUCCESS != (tmp_ret = record_server_event_(ret, tmp_round, task_info))) { LOG_WARN("failed to record server event", K(tmp_ret), K(ret), K(retry_count_), K(task_info)); } +#ifdef ERRSIM + if (OB_SUCC(ret)) { + ret = EN_TRANSFER_DIAGNOSE_ABORT_FAILED ? : OB_SUCCESS; + if (OB_FAIL(ret)) { + STORAGE_LOG(WARN, "fake EN_TRANSFER_DIAGNOSE_ABORT_FAILED", K(ret)); + } + } +#endif + if (OB_FAIL(ret)) { + if (OB_TMP_FAIL(ObStorageHADiagMgr::add_transfer_error_diagnose_info(task_info.task_id_, task_info.dest_ls_id_, + share::ObStorageHADiagTaskType::TRANSFER_ABORT, tmp_round, ret, diagnose_result_msg_))) { + LOG_WARN("failed to add error diagnose info", K(tmp_ret), K(ret), + K(task_info.task_id_), K(task_info.dest_ls_id_), K(tmp_round)); + } + } + + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::TRANSFER_ABORT_END, + ObStorageHADiagTaskType::TRANSFER_ABORT, start_ts, tmp_round, true/*is_report*/); + } return ret; } @@ -2244,6 +2405,12 @@ int ObTransferHandler::block_and_kill_tx_( SERVER_EVENT_SYNC_ADD("TRANSFER", "AFTER_TRANSFER_BLOCK_AND_KILL_TX"); #endif DEBUG_SYNC(AFTER_TRANSFER_BLOCK_AND_KILL_TX); + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::BLOCK_AND_KILL_TX, + ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/); + } else { + diagnose_result_msg_ = share::ObStorageHACostItemName::BLOCK_AND_KILL_TX; + } return ret; } @@ -2337,6 +2504,12 @@ int ObTransferHandler::unblock_tx_( SERVER_EVENT_SYNC_ADD("TRANSFER", "AFTER_TRANSFER_UNBLOCK_TX"); #endif DEBUG_SYNC(AFTER_TRANSFER_UNBLOCK_TX); + if (OB_SUCC(ret)) { + process_perf_diagnose_info_(ObStorageHACostItemName::UNBLOCK_TX, + ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/); + } else { + diagnose_result_msg_ = share::ObStorageHACostItemName::UNBLOCK_TX; + } return ret; } @@ -2436,6 +2609,9 @@ int ObTransferHandler::stop_tablets_schedule_medium_(const ObIArray } else { succ_stop = true; } + if (OB_FAIL(ret)) { + diagnose_result_msg_ = share::ObStorageHACostItemName::STOP_LS_SCHEDULE_MEMDIUM; + } return ret; } @@ -2783,6 +2959,159 @@ int ObTransferHandler::check_task_exist_( return ret; } +int ObTransferHandler::set_related_info( + const share::ObTransferTaskID &task_id, + const share::SCN &start_scn) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ls transfer handler do not init", K(ret)); + } else if (!task_id.is_valid() || !start_scn.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(task_id), K(start_scn)); + } else if (OB_FAIL(related_info_.set_info(task_id, start_scn))) { + LOG_WARN("fail to set info", K(ret), K(task_id), K(start_scn)); + } + return ret; +} + +int ObTransferHandler::get_related_info_task_id(share::ObTransferTaskID &task_id) const +{ + int ret = OB_SUCCESS; + task_id.reset(); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ls transfer handler do not init", K(ret)); + } else if (OB_FAIL(related_info_.get_related_info_task_id(task_id))) { + LOG_WARN("failed to get task id", K(ret)); + } + return ret; +} + +int ObTransferHandler::record_error_diagnose_info_in_replay( + const share::ObTransferTaskID &task_id, + const share::ObLSID &dest_ls_id, + const int result_code, + const bool clean_related_info, + const share::ObStorageHADiagTaskType type, + const share::ObStorageHACostItemName result_msg) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ls transfer handler do not init", K(ret)); + } else if (!task_id.is_valid() + || !dest_ls_id.is_valid() + || type < share::ObStorageHADiagTaskType::TRANSFER_START + || type >= share::ObStorageHADiagTaskType::MAX_TYPE) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(task_id), K(dest_ls_id), K(type)); + } else if (OB_FAIL(related_info_.record_error_diagnose_info_in_replay( + task_id, dest_ls_id, result_code, clean_related_info, type, result_msg))) { + LOG_WARN("failed to record diagnose info in replay", K(ret), K(task_id), K(dest_ls_id), + K(result_code), K(clean_related_info), K(type), K(result_msg)); + } + return ret; +} + +int ObTransferHandler::record_error_diagnose_info_in_backfill( + const share::SCN &log_sync_scn, + const share::ObLSID &dest_ls_id, + const int result_code, + const ObTabletID &tablet_id, + const ObMigrationStatus &migration_status, + const share::ObStorageHACostItemName result_msg) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ls transfer handler do not init", K(ret)); + } else if (!log_sync_scn.is_valid() + || !dest_ls_id.is_valid() + || !tablet_id.is_valid() + || OB_SUCCESS == result_code + || migration_status < ObMigrationStatus::OB_MIGRATION_STATUS_NONE + || migration_status >= ObMigrationStatus::OB_MIGRATION_STATUS_MAX) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(log_sync_scn), K(dest_ls_id), K(result_code), K(tablet_id), K(migration_status)); + } else if (OB_FAIL(related_info_.record_error_diagnose_info_in_backfill( + log_sync_scn, dest_ls_id, result_code, tablet_id, migration_status, result_msg))) { + LOG_WARN("failed to record diagnose info in worker", K(ret), K(log_sync_scn), + K(dest_ls_id), K(result_code), K(tablet_id), K(migration_status), K(result_msg)); + } + return ret; +} + +void ObTransferHandler::reset_related_info() +{ + related_info_.reset(); +} + +int ObTransferHandler::reset_related_info(const share::ObTransferTaskID &task_id) +{ + int ret = OB_SUCCESS; + if (!task_id.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(task_id)); + } else { + related_info_.reset(task_id); + } + return ret; +} + +void ObTransferHandler::process_perf_diagnose_info_( + const ObStorageHACostItemName name, + const ObStorageHADiagTaskType task_type, + const int64_t start_ts, + const int64_t round, const bool is_report) const +{ + int ret = OB_SUCCESS; + ObArenaAllocator alloc; + ObTransferPerfDiagInfo info; + ObStorageHATimestampItem item; + item.name_ = name; + item.type_ = ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE; + item.retry_id_ = round; + item.timestamp_ = ObTimeUtil::current_time(); + common::ObTabletID tablet_id; + share::ObStorageHADiagTaskKey key; + if (OB_FAIL(info.init(&alloc, MTL_ID()))) { + LOG_WARN("fail to init info", K(ret)); + } else if (OB_FAIL(info.add_item(item))) { + LOG_WARN("fail to add item", K(ret), K(item)); + } else if (OB_FAIL(ObStorageHADiagMgr::construct_diagnose_info_key(task_info_.task_id_, ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE, + task_type, ObStorageHADiagType::PERF_DIAGNOSE, round, tablet_id, key))) { + LOG_WARN("failed to construct error diagnose info key", K(ret), K(task_info_.task_id_), K(round), K(tablet_id)); + } else if (OB_FAIL(ObStorageHADiagMgr::construct_diagnose_info(task_info_.task_id_, task_info_.dest_ls_id_, + task_type, round, OB_SUCCESS, ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE, info))) { + LOG_WARN("failed to construct diagnose info", K(ret), K(task_info_), K(round), K(task_type)); + } else if (OB_FAIL(ObStorageHADiagMgr::add_transfer_perf_diagnose_info(key, start_ts, task_info_.tablet_list_.count(), is_report, info))) { + LOG_WARN("failed to add perf diagnose info", K(ret), K(key), K(info), K(start_ts), K(task_info_), K(is_report)); + } +} + +int ObTransferHandler::record_perf_diagnose_info_in_replay( + const share::ObStorageHAPerfDiagParams ¶ms, + const int result, + const uint64_t timestamp, + const int64_t start_ts, + const bool is_report) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ls transfer handler do not init", K(ret)); + } else if (!params.is_valid() + || start_ts < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(params), K(start_ts)); + } else if (OB_FAIL(related_info_.record_perf_diagnose_info_in_replay(params, result, timestamp, start_ts, is_report))) { + LOG_WARN("failed to record diagnose info in replay", K(ret), K(params), K(result), K(start_ts), K(timestamp), K(is_report)); + } + return ret; +} + int ObTransferHandler::get_src_ls_member_list_( common::ObMemberList &member_list) { @@ -2807,6 +3136,34 @@ int ObTransferHandler::get_src_ls_member_list_( return ret; } +int ObTransferHandler::record_perf_diagnose_info_in_backfill( + const share::ObStorageHAPerfDiagParams ¶ms, + const share::SCN &log_sync_scn, + const int result_code, + const ObMigrationStatus &migration_status, + const uint64_t timestamp, + const int64_t start_ts, + const bool is_report) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ls transfer handler do not init", K(ret)); + } else if (!params.is_valid() + ||!log_sync_scn.is_valid() + || migration_status < ObMigrationStatus::OB_MIGRATION_STATUS_NONE + || migration_status >= ObMigrationStatus::OB_MIGRATION_STATUS_MAX + || start_ts < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(log_sync_scn), K(start_ts), K(params), K(migration_status)); + } else if (OB_FAIL(related_info_.record_perf_diagnose_info_in_backfill(params, log_sync_scn, + result_code, migration_status, timestamp, start_ts, is_report))) { + LOG_WARN("failed to record diagnose info in backfill", K(ret), K(params), K(log_sync_scn), + K(result_code), K(migration_status), K(start_ts), K(timestamp), K(is_report)); + } + return ret; +} + int ObTransferHandler::broadcast_tablet_location_(const ObTransferTaskInfo &task_info) { int ret = OB_SUCCESS; @@ -2848,6 +3205,28 @@ int ObTransferHandler::broadcast_tablet_location_(const ObTransferTaskInfo &task return ret; } -} +int ObTransferHandler::do_clean_diagnose_info_() +{ + int ret = OB_SUCCESS; + bool is_leader = false; + ObStorageHADiagMgr *mgr = nullptr; + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("transfer handler do not init", K(ret)); + } else if (OB_FAIL(check_self_is_leader_(is_leader))) { + LOG_WARN("failed to check self is leader", K(ret), KPC(ls_)); + } else if (is_leader) { + // clean follower diagnose info + } else if (!task_info_.is_valid()) { + // unset error ret code + } else if (OB_ISNULL(mgr = MTL(ObStorageHADiagMgr *))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get ObStorageHADiagMgr from MTL", K(ret)); + } else if (OB_FAIL(mgr->report_task(task_info_.task_id_))) { + LOG_WARN("fail to report task", K(ret), K(task_info_.task_id_)); + } + return ret; } +} +} \ No newline at end of file diff --git a/src/storage/high_availability/ob_transfer_handler.h b/src/storage/high_availability/ob_transfer_handler.h index 22edc46ba..553eb34a2 100644 --- a/src/storage/high_availability/ob_transfer_handler.h +++ b/src/storage/high_availability/ob_transfer_handler.h @@ -31,6 +31,7 @@ #include "ob_transfer_backfill_tx.h" #include "lib/thread/thread_mgr_interface.h" #include "logservice/ob_log_base_type.h" +#include "share/ob_storage_ha_diagnose_struct.h" namespace oceanbase { @@ -68,6 +69,40 @@ public: int safe_to_destroy(bool &is_safe); int offline(); void online(); + int set_related_info( + const share::ObTransferTaskID &task_id, + const share::SCN &start_scn); + int get_related_info_task_id(share::ObTransferTaskID &task_id) const; + int reset_related_info(const share::ObTransferTaskID &task_id); + int record_error_diagnose_info_in_replay( + const share::ObTransferTaskID &task_id, + const share::ObLSID &dest_ls_id, + const int result_code, + const bool clean_related_info, + const share::ObStorageHADiagTaskType type, + const share::ObStorageHACostItemName result_msg); + int record_error_diagnose_info_in_backfill( + const share::SCN &log_sync_scn, + const share::ObLSID &dest_ls_id, + const int result_code, + const ObTabletID &tablet_id, + const ObMigrationStatus &migration_status, + const share::ObStorageHACostItemName result_msg); + void reset_related_info(); + int record_perf_diagnose_info_in_replay( + const share::ObStorageHAPerfDiagParams ¶ms, + const int result, + const uint64_t timestamp, + const int64_t start_ts, + const bool is_report); + int record_perf_diagnose_info_in_backfill( + const share::ObStorageHAPerfDiagParams ¶ms, + const share::SCN &log_sync_scn, + const int result_code, + const ObMigrationStatus &migration_status, + const uint64_t timestamp, + const int64_t start_ts, + const bool is_report); private: int get_transfer_task_(share::ObTransferTaskInfo &task_info); int get_transfer_task_from_inner_table_( @@ -284,6 +319,12 @@ private: int get_src_ls_member_list_( common::ObMemberList &member_list); int broadcast_tablet_location_(const share::ObTransferTaskInfo &task_info); + void process_perf_diagnose_info_( + const ObStorageHACostItemName name, + const ObStorageHADiagTaskType task_type, + const int64_t start_ts, + const int64_t round, const bool is_report) const; + int do_clean_diagnose_info_(); int register_move_tx_ctx_batch_(const share::ObTransferTaskInfo &task_info, const SCN transfer_scn, @@ -306,6 +347,9 @@ private: ObTransferWorkerMgr transfer_worker_mgr_; int64_t round_; share::SCN gts_seq_; + ObTransferRelatedInfo related_info_; + ObTransferTaskInfo task_info_; + share::ObStorageHACostItemName diagnose_result_msg_; common::SpinRWLock transfer_handler_lock_; bool transfer_handler_enabled_; DISALLOW_COPY_AND_ASSIGN(ObTransferHandler); diff --git a/src/storage/high_availability/ob_transfer_struct.cpp b/src/storage/high_availability/ob_transfer_struct.cpp index af315637a..147e7bdfd 100644 --- a/src/storage/high_availability/ob_transfer_struct.cpp +++ b/src/storage/high_availability/ob_transfer_struct.cpp @@ -20,6 +20,7 @@ #include "storage/tx_storage/ob_ls_service.h" #include "storage/high_availability/ob_storage_ha_utils.h" #include "storage/tx/ob_ts_mgr.h" +#include "storage/high_availability/ob_storage_ha_diagnose_mgr.h" using namespace oceanbase; using namespace share; @@ -591,3 +592,418 @@ int ObTransferTaskLockInfo::set(const uint64_t tenant_id, const share::ObLSID &l } return ret; } + +ObTransferRelatedInfo::ObTransferRelatedInfo() + : is_inited_(false), + task_id_(), + start_scn_(), + start_out_log_replay_num_(0), + start_in_log_replay_num_(0), + finish_out_log_replay_num_(0), + finish_in_log_replay_num_(0), + map_(), + lock_() +{ +} + +ObTransferRelatedInfo::~ObTransferRelatedInfo() +{ + destroy(); +} + +int ObTransferRelatedInfo::init() +{ + int ret = OB_SUCCESS; + const int64_t BUCKET_NUM = 100; + ObMemAttr attr(MTL_ID(), "TXBFRetryId"); + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", K(ret)); + } else if (OB_FAIL(map_.create(BUCKET_NUM, attr))) { + LOG_WARN("fail to create hash map", K(ret), K(BUCKET_NUM)); + } else { + start_out_log_replay_num_ = 0; + start_in_log_replay_num_ = 0; + finish_in_log_replay_num_ = 0; + finish_out_log_replay_num_ = 0; + is_inited_ = true; + } + return ret; +} + +bool ObTransferRelatedInfo::is_valid() const +{ + return task_id_.is_valid() + && start_scn_.is_valid() + && start_out_log_replay_num_ >= 0 + && start_in_log_replay_num_ >= 0 + && finish_in_log_replay_num_ >= 0 + && finish_out_log_replay_num_ >= 0 + && is_inited_; +} + +void ObTransferRelatedInfo::reset_() +{ + task_id_.reset(); + start_scn_.reset(); + start_out_log_replay_num_ = 0; + start_in_log_replay_num_ = 0; + finish_in_log_replay_num_ = 0; + finish_out_log_replay_num_ = 0; +} + +void ObTransferRelatedInfo::destroy() +{ + common::SpinWLockGuard guard(lock_); + is_inited_ = false; + reset_(); + map_.destroy(); +} + +void ObTransferRelatedInfo::reset() +{ + common::SpinWLockGuard guard(lock_); + reset_(); + map_.reuse(); +} + +int ObTransferRelatedInfo::reset(const share::ObTransferTaskID &task_id) +{ + int ret = OB_SUCCESS; + common::SpinWLockGuard guard(lock_); + if (task_id != get_task_id_()) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("infos already are clean", K(ret), K(task_id), K(task_id_)); + } else { + reset_(); + map_.reuse(); + } + return ret; +} + +int ObTransferRelatedInfo::inc_tx_backfill_retry_num_(const ObTabletID &id, int64_t &retry_num) +{ + int ret = OB_SUCCESS; + retry_num = 0; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K(ret)); + } else if (!id.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument!", K(ret), K(id)); + } else { + if (OB_FAIL(map_.get_refactored(id, retry_num))) { + if (OB_HASH_NOT_EXIST == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to get retry id from map", K(ret), K(id)); + } + } + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(map_.set_refactored(id, ++retry_num, 1/*overwrite*/))) { + LOG_WARN("fail to set retry_num", K(ret), K(id), K(retry_num)); + } + } + return ret; +} + +int ObTransferRelatedInfo::set_info( + const share::ObTransferTaskID &task_id, + const share::SCN &start_scn) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K(ret)); + } else if (!task_id.is_valid() || !start_scn.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(task_id), K(start_scn)); + } else { + common::SpinWLockGuard guard(lock_); + reset_(); + map_.reuse(); + start_scn_ = start_scn; + task_id_ = task_id; + } + return ret; +} + +const share::ObTransferTaskID &ObTransferRelatedInfo::get_task_id_() const +{ + return task_id_; +} + +const share::SCN &ObTransferRelatedInfo::get_start_scn_() const +{ + return start_scn_; +} + +int ObTransferRelatedInfo::get_replay_retry_num_( + const share::ObStorageHADiagTaskType type, const bool inc_retry_num, int64_t &retry_num) +{ + int ret = OB_SUCCESS; + retry_num = 0; + switch (type) { + case ObStorageHADiagTaskType::TRANSFER_START_IN: { + if (inc_retry_num) { + start_out_log_replay_num_++; + } + retry_num = start_out_log_replay_num_; + break; + } + case ObStorageHADiagTaskType::TRANSFER_START_OUT: { + if (inc_retry_num) { + start_in_log_replay_num_++; + } + retry_num = start_in_log_replay_num_; + break; + } + case ObStorageHADiagTaskType::TRANSFER_FINISH_IN: { + if (inc_retry_num) { + finish_in_log_replay_num_++; + } + retry_num = finish_in_log_replay_num_; + break; + } + case ObStorageHADiagTaskType::TRANSFER_FINISH_OUT: { + if (inc_retry_num) { + finish_out_log_replay_num_++; + } + retry_num = finish_out_log_replay_num_; + break; + } + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected type", K(ret), K(type)); + } + } + return ret; +} + +int ObTransferRelatedInfo::record_error_diagnose_info_in_backfill( + const share::SCN &log_sync_scn, + const share::ObLSID &dest_ls_id, + const int result_code, + const ObTabletID &tablet_id, + const ObMigrationStatus &migration_status, + const share::ObStorageHACostItemName result_msg) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K(ret)); + } else if (!log_sync_scn.is_valid() + || !dest_ls_id.is_valid() + || !tablet_id.is_valid() + || OB_SUCCESS == result_code + || migration_status < ObMigrationStatus::OB_MIGRATION_STATUS_NONE + || migration_status >= ObMigrationStatus::OB_MIGRATION_STATUS_MAX) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(log_sync_scn), K(dest_ls_id), + K(result_code), K(tablet_id), K(migration_status)); + } else { + common::SpinWLockGuard guard(lock_); + if (log_sync_scn != get_start_scn_()) { + if (ObMigrationStatus::OB_MIGRATION_STATUS_NONE != migration_status) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("dest ls in migration", K(ret), K(dest_ls_id), K(migration_status)); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("related_info already change", K(ret), K(start_scn_), K(log_sync_scn)); + } + } else { + int64_t retry_num = 0; + if (OB_FAIL(inc_tx_backfill_retry_num_(tablet_id, retry_num))) { + LOG_WARN("fail to get retry id", K(ret), K(tablet_id)); + } else if (OB_FAIL(ObStorageHADiagMgr::add_transfer_error_diagnose_info(get_task_id_(), dest_ls_id, + share::ObStorageHADiagTaskType::TRANSFER_BACKFILLED, retry_num, result_code, tablet_id, result_msg))) { + LOG_WARN("failed to add error diagnose info", K(ret), K(get_task_id_()), K(dest_ls_id), + K(retry_num), K(result_code), K(tablet_id), K(result_msg)); + } + } + } + return ret; +} + +int ObTransferRelatedInfo::record_error_diagnose_info_in_replay( + const share::ObTransferTaskID &task_id, + const share::ObLSID &dest_ls_id, + const int result_code, + const bool clean_related_info, + const share::ObStorageHADiagTaskType type, + const share::ObStorageHACostItemName result_msg) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K(ret)); + } else if (!task_id.is_valid() + || !dest_ls_id.is_valid() + || type < share::ObStorageHADiagTaskType::TRANSFER_START + || type >= share::ObStorageHADiagTaskType::MAX_TYPE) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(task_id), K(dest_ls_id), K(type)); + } else { + common::SpinWLockGuard guard(lock_); + if (OB_SUCCESS == result_code) { + // do nothing + } else { + int64_t log_replay_num = 0; + if (OB_FAIL(get_replay_retry_num_(type, true/*inc_retry_num*/, log_replay_num))) { + LOG_WARN("fail to inc retry num", K(ret), K(type), K(log_replay_num)); + } else if (OB_FAIL(ObStorageHADiagMgr::add_transfer_error_diagnose_info(task_id, dest_ls_id, + type, log_replay_num, result_code, result_msg))) { + LOG_WARN("failed to add error diagnose info", K(ret), + K(task_id), K(dest_ls_id), K(type), K(log_replay_num), K(result_code), K(result_msg)); + } + } + if (OB_SUCCESS == result_code && clean_related_info) { + reset_(); + map_.reuse(); + } + } + return ret; +} + +int ObTransferRelatedInfo::construct_perf_diag_info_( + const share::ObStorageHAPerfDiagParams ¶ms, + const uint64_t timestamp, + const int64_t retry_num, + const share::ObTransferTaskID &task_id, + const int result, + const int64_t start_ts, + const bool is_report) +{ + int ret = OB_SUCCESS; + ObArenaAllocator alloc; + ObTransferPerfDiagInfo info; + ObStorageHACostAccumItem accum_item; + ObStorageHATimestampItem ts_item; + ObIStorageHACostItem *item = nullptr; + if (ObStorageHACostItemType::ACCUM_COST_TYPE == params.item_type_) { + item = &accum_item; + static_cast(item)->cost_time_ = timestamp; + } else { + item = &ts_item; + static_cast(item)->timestamp_ = timestamp; + } + item->name_ = params.name_; + item->type_ = params.item_type_; + item->retry_id_ = retry_num + 1; + share::ObStorageHADiagTaskKey key; + if (OB_FAIL(info.init(&alloc, MTL_ID()))) { + LOG_WARN("fail to init info", K(ret)); + } else if (OB_FAIL(info.add_item(*item))) { + LOG_WARN("fail to add item", K(ret), KPC(item)); + } else if (OB_FAIL(ObStorageHADiagMgr::construct_diagnose_info_key( + task_id, ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE, + params.task_type_, ObStorageHADiagType::PERF_DIAGNOSE, retry_num + 1, params.tablet_id_, key))) { + LOG_WARN("failed to construct error diagnose info key", + K(ret), K(task_id), K(retry_num), K(params.tablet_id_)); + } else if (OB_FAIL(ObStorageHADiagMgr::construct_diagnose_info(task_id, params.dest_ls_id_, + params.task_type_, retry_num + 1, result, ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE, info))) { + LOG_WARN("failed to construct diagnose info", K(ret), K(params), K(retry_num), K(result)); + } else if (OB_FAIL(ObStorageHADiagMgr::append_perf_diagnose_info(params.tablet_id_, info))) { + LOG_WARN("fail to append diagnose info", K(ret), K(params.tablet_id_)); + } else if (OB_FAIL(ObStorageHADiagMgr::add_transfer_perf_diagnose_info( + key, start_ts, params.tablet_count_, is_report, info))) { + LOG_WARN("failed to add perf diagnose info", K(ret), + K(key), K(info), K(timestamp), K(start_ts), K(is_report)); + } + return ret; +} + +int ObTransferRelatedInfo::record_perf_diagnose_info_in_replay( + const share::ObStorageHAPerfDiagParams ¶ms, + const int result, + const uint64_t timestamp, + const int64_t start_ts, + const bool is_report) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K(ret)); + } else if (!params.is_valid() + || start_ts < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(params), K(start_ts)); + } else { + common::SpinRLockGuard guard(lock_); + int64_t log_replay_num = 0; + if (OB_FAIL(get_replay_retry_num_(params.task_type_, false/*inc_retry_num*/, log_replay_num))) { + LOG_WARN("fail to inc retry num", K(ret), K(params.task_type_), K(log_replay_num)); + } else if (OB_FAIL(construct_perf_diag_info_(params, timestamp, + log_replay_num, params.task_id_, result, start_ts, is_report))) { + LOG_WARN("fail to construct perf diag info", K(ret), K(params), K(timestamp), + K(log_replay_num), K(result), K(start_ts), K(is_report)); + } + } + return ret; +} + +int ObTransferRelatedInfo::record_perf_diagnose_info_in_backfill( + const share::ObStorageHAPerfDiagParams ¶ms, + const share::SCN &log_sync_scn, + const int result_code, + const ObMigrationStatus &migration_status, + const uint64_t timestamp, + const int64_t start_ts, + const bool is_report) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K(ret)); + } else if (!params.is_valid() + ||!log_sync_scn.is_valid() + || migration_status < ObMigrationStatus::OB_MIGRATION_STATUS_NONE + || migration_status >= ObMigrationStatus::OB_MIGRATION_STATUS_MAX + || start_ts < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(log_sync_scn), K(start_ts), K(params), K(migration_status)); + } else { + common::SpinWLockGuard guard(lock_); + if (log_sync_scn != get_start_scn_()) { + if (ObMigrationStatus::OB_MIGRATION_STATUS_NONE != migration_status) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("dest ls in migration", K(ret), K(migration_status)); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("related_info already change", K(ret), K(start_scn_), K(log_sync_scn)); + } + } else { + int64_t retry_num = 0; + if (OB_FAIL(map_.get_refactored(params.tablet_id_, retry_num))) { + if (OB_HASH_NOT_EXIST == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to get retry id from map", K(ret), K(params.tablet_id_)); + } + } + if (OB_FAIL(ret)) { + //do nothing + } else if (OB_FAIL(construct_perf_diag_info_(params, timestamp, + retry_num, get_task_id_(), result_code, start_ts, is_report))) { + LOG_WARN("fail to construct perf diag info", K(ret), K(params), K(timestamp), + K(retry_num), K(result_code), K(start_ts), K(is_report)); + } + } + } + return ret; +} + +int ObTransferRelatedInfo::get_related_info_task_id(share::ObTransferTaskID &task_id) const +{ + int ret = OB_SUCCESS; + task_id.reset(); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ls transfer handler do not init", K(ret)); + } else { + common::SpinRLockGuard guard(lock_); + task_id = get_task_id_(); + } + return ret; +} \ No newline at end of file diff --git a/src/storage/high_availability/ob_transfer_struct.h b/src/storage/high_availability/ob_transfer_struct.h index 762e200ab..c9a5f8014 100644 --- a/src/storage/high_availability/ob_transfer_struct.h +++ b/src/storage/high_availability/ob_transfer_struct.h @@ -21,6 +21,7 @@ #include "storage/memtable/ob_memtable.h" #include "storage/tablet/ob_tablet_meta.h" #include "storage/tablet/ob_tablet_create_delete_mds_user_data.h" +#include "share/ob_storage_ha_diagnose_struct.h" namespace oceanbase { @@ -230,6 +231,84 @@ public: ObSqlString comment_; }; +class ObTransferRelatedInfo final +{ +public: + ObTransferRelatedInfo(); + ~ObTransferRelatedInfo(); + int init(); + bool is_valid() const; + void reset(); + void destroy(); + int set_info( + const share::ObTransferTaskID &task_id, + const share::SCN &start_scn); + int record_error_diagnose_info_in_replay( + const share::ObTransferTaskID &task_id, + const share::ObLSID &dest_ls_id, + const int result_code, + const bool clean_related_info, + const share::ObStorageHADiagTaskType type, + const share::ObStorageHACostItemName result_msg); + int record_error_diagnose_info_in_backfill( + const share::SCN &log_sync_scn, + const share::ObLSID &dest_ls_id, + const int result_code, + const ObTabletID &tablet_id, + const ObMigrationStatus &migration_status, + const share::ObStorageHACostItemName result_msg); + + int record_perf_diagnose_info_in_replay( + const share::ObStorageHAPerfDiagParams ¶ms, + const int result, + const uint64_t timestamp, + const int64_t start_ts, + const bool is_report); + + int record_perf_diagnose_info_in_backfill( + const share::ObStorageHAPerfDiagParams ¶ms, + const share::SCN &log_sync_scn, + const int result_code, + const ObMigrationStatus &migration_status, + const uint64_t timestamp, + const int64_t start_ts, + const bool is_report); + int get_related_info_task_id(share::ObTransferTaskID &task_id) const; + int reset(const share::ObTransferTaskID &task_id); + TO_STRING_KV(K_(task_id), K_(start_scn), K_(start_out_log_replay_num), + K_(start_in_log_replay_num), K_(finish_out_log_replay_num), K_(finish_in_log_replay_num)); + typedef hash::ObHashMap TxBackfillStatMap; + +private: + void reset_(); + int inc_tx_backfill_retry_num_(const ObTabletID &id, int64_t &retry_num); + const share::ObTransferTaskID &get_task_id_() const; + const share::SCN &get_start_scn_() const; + int get_replay_retry_num_( + const share::ObStorageHADiagTaskType type, const bool inc_retry_num, int64_t &retry_num); + int construct_perf_diag_info_( + const share::ObStorageHAPerfDiagParams ¶ms, + const uint64_t timestamp, + const int64_t retry_num, + const share::ObTransferTaskID &task_id, + const int result, + const int64_t start_ts, + const bool is_report); + +private: + bool is_inited_; + share::ObTransferTaskID task_id_; + share::SCN start_scn_; + int64_t start_out_log_replay_num_; + int64_t start_in_log_replay_num_; + int64_t finish_out_log_replay_num_; + int64_t finish_in_log_replay_num_; + TxBackfillStatMap map_; + SpinRWLock lock_; + + DISALLOW_COPY_AND_ASSIGN(ObTransferRelatedInfo); +}; + } } #endif diff --git a/src/storage/tablet/ob_tablet_finish_transfer_mds_helper.cpp b/src/storage/tablet/ob_tablet_finish_transfer_mds_helper.cpp index eef3f2cfa..5c1a65a4b 100644 --- a/src/storage/tablet/ob_tablet_finish_transfer_mds_helper.cpp +++ b/src/storage/tablet/ob_tablet_finish_transfer_mds_helper.cpp @@ -30,6 +30,8 @@ #include "storage/high_availability/ob_rebuild_service.h" #include "storage/high_availability/ob_storage_ha_utils.h" #include "storage/high_availability/ob_transfer_service.h" +#include "share/ob_storage_ha_diagnose_struct.h" +#include "storage/high_availability/ob_storage_ha_diagnose_mgr.h" #define USING_LOG_PREFIX MDS @@ -39,6 +41,7 @@ namespace storage { ERRSIM_POINT_DEF(EN_TRANSFER_NEED_REBUILD); +ERRSIM_POINT_DEF(EN_TRANSFER_DIAGNOSE_FINISH_REPLAY_FAILED); int ObTabletFinishTransferUtil::check_transfer_table_replaced( ObTabletHandle &tablet_handle, @@ -283,7 +286,8 @@ int ObTabletFinishTransferOutHelper::on_register( ObTXFinishTransferOutInfo tx_finish_transfer_out_info; int64_t pos = 0; ObTransferUtils::set_transfer_module(); - + const int64_t start_ts = ObTimeUtility::current_time(); + share::ObStorageHACostItemName diagnose_result_msg = share::ObStorageHACostItemName::MAX_NAME; if (OB_ISNULL(buf) || len < 0) { ret = OB_INVALID_ARGUMENT; LOG_WARN("on register finish transfer out get invalid argument", K(ret), KP(buf), K(len)); @@ -293,11 +297,32 @@ int ObTabletFinishTransferOutHelper::on_register( ret = OB_ERR_UNEXPECTED; LOG_WARN("tx finish transfer out info is unexpected", K(ret), K(tx_finish_transfer_out_info)); } else if (CLICK_FAIL(on_register_success_(tx_finish_transfer_out_info, ctx))) { + diagnose_result_msg = share::ObStorageHACostItemName::ON_REGISTER_SUCCESS; LOG_WARN("failed to on register", K(ret), K(tx_finish_transfer_out_info)); } else if (CLICK_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_empty_shell_trigger(tx_finish_transfer_out_info.src_ls_id_))) { LOG_WARN("failed to set_tablet_empty_shell_trigger", K(ret), K(tx_finish_transfer_out_info)); } ObTransferUtils::clear_transfer_module(); +#ifdef ERRSIM + if (OB_SUCC(ret)) { + ret = EN_TRANSFER_DIAGNOSE_FINISH_REPLAY_FAILED ? : OB_SUCCESS; + if (OB_FAIL(ret)) { + STORAGE_LOG(WARN, "fake EN_TRANSFER_DIAGNOSE_FINISH_REPLAY_FAILED", K(ret)); + } + } +#endif + const int64_t end_ts = ObTimeUtility::current_time(); + bool is_report = OB_SUCCESS == ret ? true : false; + ObTransferUtils::process_finish_out_perf_diag_info(tx_finish_transfer_out_info, + ObStorageHACostItemType::ACCUM_COST_TYPE, ObStorageHACostItemName::FINISH_TRANSFER_OUT_REPLAY_COST, + ret, end_ts - start_ts, start_ts, is_report); + ObTransferUtils::add_transfer_error_diagnose_in_replay( + tx_finish_transfer_out_info.task_id_, + tx_finish_transfer_out_info.dest_ls_id_, + ret, + false/*clean_related_info*/, + ObStorageHADiagTaskType::TRANSFER_FINISH_OUT, + diagnose_result_msg); return ret; } @@ -312,6 +337,9 @@ int ObTabletFinishTransferOutHelper::on_register_success_( ObLS *ls = NULL; const share::ObLSID &src_ls_id = tx_finish_transfer_out_info.src_ls_id_; const int64_t start_ts = ObTimeUtil::current_time(); + ObTransferUtils::process_finish_out_perf_diag_info(tx_finish_transfer_out_info, + ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::FINISH_TRANSFER_OUT_FIRST_REPALY_LOG_TIMESTAMP, + ret, start_ts, start_ts, false/*is_report*/); LOG_INFO("[TRANSFER] start tx finish transfer out on_register_success_", K(tx_finish_transfer_out_info)); #ifdef ERRSIM SERVER_EVENT_ADD("transfer", "tx_finish_transfer_out", @@ -464,7 +492,8 @@ int ObTabletFinishTransferOutHelper::on_replay( ObTXFinishTransferOutInfo tx_finish_transfer_out_info; int64_t pos = 0; ObTransferUtils::set_transfer_module(); - + const int64_t start_ts = ObTimeUtility::current_time(); + share::ObStorageHACostItemName diagnose_result_msg = share::ObStorageHACostItemName::MAX_NAME; if (OB_ISNULL(buf) || len < 0 || !scn.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("on replay finish transfer out get invalid argument", K(ret), KP(buf), K(len), K(scn)); @@ -474,11 +503,32 @@ int ObTabletFinishTransferOutHelper::on_replay( ret = OB_ERR_UNEXPECTED; LOG_WARN("tx finish transfer out info is unexpected", K(ret), K(tx_finish_transfer_out_info)); } else if (CLICK_FAIL(on_replay_success_(scn, tx_finish_transfer_out_info, ctx))) { + diagnose_result_msg = share::ObStorageHACostItemName::ON_REPLAY_SUCCESS; LOG_WARN("failed to do on_replay_success_", K(ret), K(tx_finish_transfer_out_info)); } else if (CLICK_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_empty_shell_trigger(tx_finish_transfer_out_info.src_ls_id_))) { LOG_WARN("failed to set_tablet_empty_shell_trigger", K(ret), K(tx_finish_transfer_out_info)); } ObTransferUtils::clear_transfer_module(); +#ifdef ERRSIM + if (OB_SUCC(ret)) { + ret = EN_TRANSFER_DIAGNOSE_FINISH_REPLAY_FAILED ? : OB_SUCCESS; + if (OB_FAIL(ret)) { + STORAGE_LOG(WARN, "fake EN_TRANSFER_DIAGNOSE_FINISH_REPLAY_FAILED", K(ret)); + } + } +#endif + const int64_t end_ts = ObTimeUtility::current_time(); + bool is_report = OB_SUCCESS == ret ? true : false; + ObTransferUtils::process_finish_out_perf_diag_info(tx_finish_transfer_out_info, + ObStorageHACostItemType::ACCUM_COST_TYPE, ObStorageHACostItemName::FINISH_TRANSFER_OUT_REPLAY_COST, + ret, end_ts - start_ts, start_ts, is_report); + ObTransferUtils::add_transfer_error_diagnose_in_replay( + tx_finish_transfer_out_info.task_id_, + tx_finish_transfer_out_info.dest_ls_id_, + ret, + true/*clean_related_info*/, + ObStorageHADiagTaskType::TRANSFER_FINISH_OUT, + diagnose_result_msg); return ret; } @@ -493,7 +543,12 @@ int ObTabletFinishTransferOutHelper::on_replay_success_( ObLSService *ls_svr = NULL; ObLSHandle ls_handle; ObLS *ls = NULL; - + ObTransferUtils::process_finish_out_perf_diag_info(tx_finish_transfer_out_info, + ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::FINISH_TRANSFER_OUT_LOG_SCN, + ret, scn.get_val_for_logservice(), start_ts, false/*is_report*/); + ObTransferUtils::process_finish_out_perf_diag_info(tx_finish_transfer_out_info, + ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::FINISH_TRANSFER_OUT_FIRST_REPALY_LOG_TIMESTAMP, + ret, start_ts, start_ts, false/*is_report*/); FLOG_INFO("[TRANSFER] start tx finish transfer out on_replay_success_", K(scn), K(tx_finish_transfer_out_info)); #ifdef ERRSIM SERVER_EVENT_ADD("transfer", "tx_finish_transfer_out", @@ -867,7 +922,8 @@ int ObTabletFinishTransferInHelper::on_register( int64_t pos = 0; const bool for_replay = false; ObTransferUtils::set_transfer_module(); - + const int64_t start_ts = ObTimeUtility::current_time(); + share::ObStorageHACostItemName diagnose_result_msg = ObStorageHACostItemName::MAX_NAME; if (OB_ISNULL(buf) || len < 0) { ret = OB_INVALID_ARGUMENT; LOG_WARN("on register finish transfer in get invalid argument", K(ret), KP(buf), K(len)); @@ -877,10 +933,31 @@ int ObTabletFinishTransferInHelper::on_register( ret = OB_ERR_UNEXPECTED; LOG_WARN("tx finish transfer in info is unexpected", K(ret), K(tx_finish_transfer_in_info)); } else if (CLICK_FAIL(on_register_success_(tx_finish_transfer_in_info, ctx))) { + diagnose_result_msg = share::ObStorageHACostItemName::ON_REGISTER_SUCCESS; LOG_WARN("failed to do on register success", K(ret), K(tx_finish_transfer_in_info)); } ObTransferUtils::clear_transfer_module(); +#ifdef ERRSIM + if (OB_SUCC(ret)) { + ret = EN_TRANSFER_DIAGNOSE_FINISH_REPLAY_FAILED ? : OB_SUCCESS; + if (OB_FAIL(ret)) { + STORAGE_LOG(WARN, "fake EN_TRANSFER_DIAGNOSE_FINISH_REPLAY_FAILED", K(ret)); + } + } +#endif + const int64_t end_ts = ObTimeUtility::current_time(); + bool is_report = OB_SUCCESS == ret ? true : false; + ObTransferUtils::process_finish_in_perf_diag_info(tx_finish_transfer_in_info, + ObStorageHACostItemType::ACCUM_COST_TYPE, ObStorageHACostItemName::FINISH_TRANSFER_IN_REPLAY_COST, + ret, end_ts - start_ts, 0/*start_ts*/, is_report); + ObTransferUtils::add_transfer_error_diagnose_in_replay( + tx_finish_transfer_in_info.task_id_, + tx_finish_transfer_in_info.dest_ls_id_, + ret, + false/*clean_related_info*/, + ObStorageHADiagTaskType::TRANSFER_FINISH_IN, + diagnose_result_msg); return ret; } @@ -895,6 +972,9 @@ int ObTabletFinishTransferInHelper::on_register_success_( ObLS *ls = NULL; const share::ObLSID &dest_ls_id = tx_finish_transfer_in_info.dest_ls_id_; const int64_t start_ts = ObTimeUtil::current_time(); + ObTransferUtils::process_finish_in_perf_diag_info(tx_finish_transfer_in_info, + ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::FINISH_TRANSFER_IN_FIRST_REPALY_LOG_TIMESTAMP, + ret, start_ts, start_ts, false/*is_report*/); LOG_INFO("[TRANSFER] start tx finish transfer in on_register_success_", K(tx_finish_transfer_in_info)); #ifdef ERRSIM SERVER_EVENT_ADD("transfer", "tx_finish_transfer_in", @@ -1083,7 +1163,8 @@ int ObTabletFinishTransferInHelper::on_replay( bool skip_replay = false; const bool for_replay = true; ObTransferUtils::set_transfer_module(); - + const int64_t start_ts = ObTimeUtility::current_time(); + share::ObStorageHACostItemName diagnose_result_msg = share::ObStorageHACostItemName::MAX_NAME; if (OB_ISNULL(buf) || len < 0 || !scn.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("on_replay_success_ finish transfer in get invalid argument", K(ret), KP(buf), K(len), K(scn)); @@ -1093,9 +1174,30 @@ int ObTabletFinishTransferInHelper::on_replay( ret = OB_ERR_UNEXPECTED; LOG_WARN("tx finish transfer in info is unexpected", K(ret), K(tx_finish_transfer_in_info)); } else if (CLICK_FAIL(on_replay_success_(scn, tx_finish_transfer_in_info, ctx))) { + diagnose_result_msg = share::ObStorageHACostItemName::ON_REPLAY_SUCCESS; LOG_WARN("failed to do on_replay_success_", K(ret), K(tx_finish_transfer_in_info)); } ObTransferUtils::clear_transfer_module(); +#ifdef ERRSIM + if (OB_SUCC(ret)) { + ret = EN_TRANSFER_DIAGNOSE_FINISH_REPLAY_FAILED ? : OB_SUCCESS; + if (OB_FAIL(ret)) { + STORAGE_LOG(WARN, "fake EN_TRANSFER_DIAGNOSE_FINISH_REPLAY_FAILED", K(ret)); + } + } +#endif + const int64_t end_ts = ObTimeUtility::current_time(); + bool is_report = OB_SUCCESS == ret ? true : false; + ObTransferUtils::process_finish_in_perf_diag_info(tx_finish_transfer_in_info, + ObStorageHACostItemType::ACCUM_COST_TYPE, ObStorageHACostItemName::FINISH_TRANSFER_IN_REPLAY_COST, + ret, end_ts - start_ts, 0/*start_ts*/, is_report); + ObTransferUtils::add_transfer_error_diagnose_in_replay( + tx_finish_transfer_in_info.task_id_, + tx_finish_transfer_in_info.dest_ls_id_, + ret, + false/*clean_related_info*/, + ObStorageHADiagTaskType::TRANSFER_FINISH_IN, + diagnose_result_msg); return ret; } @@ -1107,6 +1209,12 @@ int ObTabletFinishTransferInHelper::on_replay_success_( MDS_TG(1_s); int ret = OB_SUCCESS; const int64_t start_ts = ObTimeUtil::current_time(); + ObTransferUtils::process_finish_in_perf_diag_info(tx_finish_transfer_in_info, + ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::FINISH_TRANSFER_IN_LOG_SCN, + ret, scn.get_val_for_logservice(), start_ts, false/*is_report*/); + ObTransferUtils::process_finish_in_perf_diag_info(tx_finish_transfer_in_info, + ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::FINISH_TRANSFER_IN_FIRST_REPALY_LOG_TIMESTAMP, + ret, start_ts, start_ts, false/*is_report*/); LOG_INFO("[TRANSFER] start tx finish transfer in on_replay_success_", K(scn), K(tx_finish_transfer_in_info)); #ifdef ERRSIM SERVER_EVENT_ADD("transfer", "tx_finish_transfer_in", diff --git a/src/storage/tablet/ob_tablet_start_transfer_mds_helper.cpp b/src/storage/tablet/ob_tablet_start_transfer_mds_helper.cpp index 7d516d417..1d047a2ff 100644 --- a/src/storage/tablet/ob_tablet_start_transfer_mds_helper.cpp +++ b/src/storage/tablet/ob_tablet_start_transfer_mds_helper.cpp @@ -30,6 +30,8 @@ #include "storage/high_availability/ob_transfer_service.h" #include "storage/high_availability/ob_rebuild_service.h" #include "storage/high_availability/ob_storage_ha_utils.h" +#include "share/ob_storage_ha_diagnose_struct.h" +#include "storage/high_availability/ob_storage_ha_diagnose_mgr.h" #include "storage/tx/ob_multi_data_source.h" #define USING_LOG_PREFIX MDS @@ -41,6 +43,7 @@ using namespace oceanbase::transaction; ERRSIM_POINT_DEF(EN_CREATE_TRANSFER_IN_TABLET_FAILED); +ERRSIM_POINT_DEF(EN_TRANSFER_DIAGNOSE_START_REPLAY_FAILED); /******************ObTabletStartTransferOutReplayExecutor*********************/ class ObTabletStartTransferOutReplayExecutor final : public logservice::ObTabletReplayExecutor { @@ -219,7 +222,8 @@ int ObTabletStartTransferOutHelper::on_register( int64_t pos = 0; const bool for_replay = false; ObTransferUtils::set_transfer_module(); - + const int64_t start_ts = ObTimeUtility::current_time(); + share::ObStorageHACostItemName diagnose_result_msg = share::ObStorageHACostItemName::MAX_NAME; if (OB_ISNULL(buf) || len < 0) { ret = OB_INVALID_ARGUMENT; LOG_WARN("on register start transfer out get invalid argument", K(ret), KP(buf), K(len)); @@ -229,9 +233,30 @@ int ObTabletStartTransferOutHelper::on_register( ret = OB_ERR_UNEXPECTED; LOG_WARN("tx start transfer out info is unexpected", K(ret), K(tx_start_transfer_out_info)); } else if (CLICK_FAIL(on_register_success_(tx_start_transfer_out_info, ctx))) { + diagnose_result_msg = share::ObStorageHACostItemName::ON_REGISTER_SUCCESS; LOG_WARN("failed to on register", K(ret), K(tx_start_transfer_out_info)); } ObTransferUtils::clear_transfer_module(); +#ifdef ERRSIM + if (OB_SUCC(ret)) { + ret = EN_TRANSFER_DIAGNOSE_START_REPLAY_FAILED ? : OB_SUCCESS; + if (OB_FAIL(ret)) { + STORAGE_LOG(WARN, "fake EN_TRANSFER_DIAGNOSE_START_REPLAY_FAILED", K(ret)); + } + } +#endif + const int64_t end_ts = ObTimeUtility::current_time(); + bool is_report = OB_SUCCESS == ret ? true : false; + ObTransferUtils::process_start_out_perf_diag_info(tx_start_transfer_out_info, + ObStorageHACostItemType::ACCUM_COST_TYPE, ObStorageHACostItemName::START_TRANSFER_OUT_REPLAY_COST, + ret, end_ts - start_ts, start_ts, is_report); + ObTransferUtils::add_transfer_error_diagnose_in_replay( + tx_start_transfer_out_info.task_id_, + tx_start_transfer_out_info.dest_ls_id_, + ret, + false/*clean_related_info*/, + ObStorageHADiagTaskType::TRANSFER_START_OUT, + diagnose_result_msg); return ret; } @@ -246,6 +271,9 @@ int ObTabletStartTransferOutHelper::on_register_success_( ObLS *ls = nullptr; const int64_t start_ts = ObTimeUtil::current_time(); LOG_INFO("[TRANSFER] start tx start transfer out on_register_success_", K(tx_start_transfer_out_info)); + ObTransferUtils::process_start_out_perf_diag_info(tx_start_transfer_out_info, + ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::START_TRANSFER_OUT_FIRST_REPALY_LOG_TIMESTAMP, + ret, start_ts, start_ts, false/*is_report*/); #ifdef ERRSIM SERVER_EVENT_ADD("transfer", "tx_start_transfer_out", "stage", "on_register_success", @@ -463,7 +491,8 @@ int ObTabletStartTransferOutHelper::on_replay( ObTxDataSourceType mds_op_type = ObTxDataSourceType::START_TRANSFER_OUT; ObTabletStartTransferOutCommonHelper transfer_out_helper(mds_op_type); ObTransferUtils::set_transfer_module(); - + const int64_t start_ts = ObTimeUtility::current_time(); + share::ObStorageHACostItemName diagnose_result_msg = share::ObStorageHACostItemName::MAX_NAME; if (OB_ISNULL(buf) || len < 0 || !scn.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("on replay start transfer out get invalid argument", K(ret), KP(buf), K(len), K(scn)); @@ -484,6 +513,9 @@ int ObTabletStartTransferOutHelper::on_replay( } #endif } + ObTransferUtils::process_start_out_perf_diag_info(tx_start_transfer_out_info, + ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::START_TRANSFER_OUT_LOG_SCN, + ret, scn.get_val_for_logservice(), start_ts, false/*is_report*/); #ifdef ERRSIM SERVER_EVENT_SYNC_ADD("TRANSFER", "BEFORE_ON_REDO_START_TRANSFER_OUT", "tenant_id", MTL_ID(), @@ -494,6 +526,7 @@ int ObTabletStartTransferOutHelper::on_replay( #endif DEBUG_SYNC(BEFORE_ON_REDO_START_TRANSFER_OUT); if (CLICK() && FAILEDx(transfer_out_helper.on_replay_success_(scn, tx_start_transfer_out_info, ctx))) { + diagnose_result_msg = share::ObStorageHACostItemName::ON_REPLAY_SUCCESS; LOG_WARN("failed to on register_success_", K(ret), K(scn), K(tx_start_transfer_out_info)); } #ifdef ERRSIM @@ -506,6 +539,26 @@ int ObTabletStartTransferOutHelper::on_replay( #endif DEBUG_SYNC(AFTER_ON_REDO_START_TRANSFER_OUT); ObTransferUtils::clear_transfer_module(); +#ifdef ERRSIM + if (OB_SUCC(ret)) { + ret = EN_TRANSFER_DIAGNOSE_START_REPLAY_FAILED ? : OB_SUCCESS; + if (OB_FAIL(ret)) { + STORAGE_LOG(WARN, "fake EN_TRANSFER_DIAGNOSE_START_REPLAY_FAILED", K(ret)); + } + } +#endif + const int64_t end_ts = ObTimeUtility::current_time(); + bool is_report = OB_SUCCESS == ret ? true : false; + ObTransferUtils::process_start_out_perf_diag_info(tx_start_transfer_out_info, + ObStorageHACostItemType::ACCUM_COST_TYPE, ObStorageHACostItemName::START_TRANSFER_OUT_REPLAY_COST, + ret, end_ts - start_ts, start_ts, is_report); + ObTransferUtils::add_transfer_error_diagnose_in_replay( + tx_start_transfer_out_info.task_id_, + tx_start_transfer_out_info.dest_ls_id_, + ret, + false/*clean_related_info*/, + ObStorageHADiagTaskType::TRANSFER_START_OUT, + diagnose_result_msg); return ret; } @@ -623,6 +676,9 @@ int ObTabletStartTransferOutCommonHelper::on_replay_success_( ObLS *ls = nullptr; const int64_t start_ts = ObTimeUtil::current_time(); LOG_INFO("[TRANSFER] start tx start transfer out on_replay_success_", K(scn), K(tx_start_transfer_out_info)); + ObTransferUtils::process_start_out_perf_diag_info(tx_start_transfer_out_info, + ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::START_TRANSFER_OUT_FIRST_REPALY_LOG_TIMESTAMP, + ret, start_ts, start_ts, false/*is_report*/); #ifdef ERRSIM SERVER_EVENT_ADD("transfer", "tx_start_transfer_out", "stage", "on_replay_success", @@ -922,8 +978,9 @@ int ObTabletStartTransferInHelper::on_register( int ret = OB_SUCCESS; ObTXStartTransferInInfo tx_start_transfer_in_info; int64_t pos = 0; - + share::ObStorageHACostItemName diagnose_result_msg = share::ObStorageHACostItemName::MAX_NAME; ObTransferUtils::set_transfer_module(); + const int64_t start_ts = ObTimeUtility::current_time(); if (OB_ISNULL(buf) || len < 0) { ret = OB_INVALID_ARGUMENT; LOG_WARN("on register start transfer in get invalid argument", K(ret), KP(buf), K(len)); @@ -932,12 +989,39 @@ int ObTabletStartTransferInHelper::on_register( } else if (!tx_start_transfer_in_info.is_valid()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tx start transfer in info is unexpected", K(ret), K(tx_start_transfer_in_info)); - } else if (CLICK_FAIL(on_register_success_(tx_start_transfer_in_info, ctx))) { - LOG_WARN("failed to do on register success", K(ret), K(tx_start_transfer_in_info)); - } else if (CLICK_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_gc_trigger(tx_start_transfer_in_info.dest_ls_id_))) { - LOG_WARN("failed to set_tablet_gc_trigger", K(ret), K(tx_start_transfer_in_info)); + } else { + ObTransferUtils::set_transfer_related_info( + tx_start_transfer_in_info.dest_ls_id_, + tx_start_transfer_in_info.task_id_, + tx_start_transfer_in_info.start_scn_); + if (CLICK_FAIL(on_register_success_(tx_start_transfer_in_info, ctx))) { + diagnose_result_msg = share::ObStorageHACostItemName::ON_REGISTER_SUCCESS; + LOG_WARN("failed to do on register success", K(ret), K(tx_start_transfer_in_info)); + } else if (CLICK_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_gc_trigger(tx_start_transfer_in_info.dest_ls_id_))) { + LOG_WARN("failed to set_tablet_gc_trigger", K(ret), K(tx_start_transfer_in_info)); + } } ObTransferUtils::clear_transfer_module(); +#ifdef ERRSIM + if (OB_SUCC(ret)) { + ret = EN_TRANSFER_DIAGNOSE_START_REPLAY_FAILED ? : OB_SUCCESS; + if (OB_FAIL(ret)) { + STORAGE_LOG(WARN, "fake EN_TRANSFER_DIAGNOSE_START_REPLAY_FAILED", K(ret)); + } + } +#endif + const int64_t end_ts = ObTimeUtility::current_time(); + bool is_report = OB_SUCCESS == ret ? true : false; + ObTransferUtils::process_start_in_perf_diag_info(tx_start_transfer_in_info, + ObStorageHACostItemType::ACCUM_COST_TYPE, ObStorageHACostItemName::START_TRANSFER_IN_REPLAY_COST, + ret, end_ts - start_ts, start_ts, is_report); + ObTransferUtils::add_transfer_error_diagnose_in_replay( + tx_start_transfer_in_info.task_id_, + tx_start_transfer_in_info.dest_ls_id_, + ret, + false/*clean_related_info*/, + ObStorageHADiagTaskType::TRANSFER_START_IN, + diagnose_result_msg); return ret; } @@ -955,6 +1039,9 @@ int ObTabletStartTransferInHelper::on_register_success_( const int64_t start_ts = ObTimeUtil::current_time(); LOG_INFO("[TRANSFER] start tx start transfer in on_register_success_", K(tx_start_transfer_in_info)); + ObTransferUtils::process_start_in_perf_diag_info(tx_start_transfer_in_info, + ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::START_TRANSFER_IN_FIRST_REPALY_LOG_TIMESTAMP, + ret, start_ts, start_ts, false/*is_report*/); if (!tx_start_transfer_in_info.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("on register success get invalid argument", K(ret), K(tx_start_transfer_in_info)); @@ -1103,7 +1190,6 @@ int ObTabletStartTransferInHelper::check_can_skip_replay_( ObLSHandle ls_handle; ObLSService *ls_service = nullptr; ObLS *ls = nullptr; - if (!scn.is_valid() || !tx_start_transfer_in_info.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("check can skip replay start transfer in get invalid argument", K(ret), K(scn), K(tx_start_transfer_in_info)); @@ -1494,7 +1580,8 @@ int ObTabletStartTransferInHelper::on_replay( bool skip_replay = false; ObTransferService *transfer_service = nullptr; ObTransferUtils::set_transfer_module(); - + const int64_t start_ts = ObTimeUtility::current_time(); + share::ObStorageHACostItemName diagnose_result_msg = share::ObStorageHACostItemName::MAX_NAME; if (OB_ISNULL(buf) || len < 0) { ret = OB_INVALID_ARGUMENT; LOG_WARN("on replay start transfer in get invalid argument", K(ret), KP(buf), K(len)); @@ -1510,18 +1597,45 @@ int ObTabletStartTransferInHelper::on_replay( LOG_WARN("failed to check can skip replay", K(ret), K(tx_start_transfer_in_info)); } else if (skip_replay) { FLOG_INFO("skip replay start transfer in", K(scn), K(tx_start_transfer_in_info)); - } else if (CLICK_FAIL(on_replay_success_(scn, tx_start_transfer_in_info, ctx))) { - LOG_WARN("failed to do on_replay_success_", K(ret), K(tx_start_transfer_in_info)); - } else if (CLICK_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_gc_trigger(tx_start_transfer_in_info.dest_ls_id_))) { - LOG_WARN("failed to set_tablet_gc_trigger", K(ret), K(tx_start_transfer_in_info)); } else { - transfer_service->wakeup(); + ObTransferUtils::set_transfer_related_info( + tx_start_transfer_in_info.dest_ls_id_, + tx_start_transfer_in_info.task_id_, + tx_start_transfer_in_info.start_scn_); + if (CLICK_FAIL(on_replay_success_(scn, tx_start_transfer_in_info, ctx))) { + diagnose_result_msg = share::ObStorageHACostItemName::ON_REPLAY_SUCCESS; + LOG_WARN("failed to do on_replay_success_", K(ret), K(tx_start_transfer_in_info)); + } else if (CLICK_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_gc_trigger(tx_start_transfer_in_info.dest_ls_id_))) { + LOG_WARN("failed to set_tablet_gc_trigger", K(ret), K(tx_start_transfer_in_info)); + } else { + transfer_service->wakeup(); + } } #ifdef ERRSIM SERVER_EVENT_SYNC_ADD("TRANSFER", "AFTER_ON_REDO_START_TRANSFER_IN"); #endif DEBUG_SYNC(AFTER_ON_REDO_START_TRANSFER_IN); ObTransferUtils::clear_transfer_module(); +#ifdef ERRSIM + if (OB_SUCC(ret)) { + ret = EN_TRANSFER_DIAGNOSE_START_REPLAY_FAILED ? : OB_SUCCESS; + if (OB_FAIL(ret)) { + STORAGE_LOG(WARN, "fake EN_TRANSFER_DIAGNOSE_START_REPLAY_FAILED", K(ret)); + } + } +#endif + const int64_t end_ts = ObTimeUtility::current_time(); + bool is_report = OB_SUCCESS == ret ? true : false; + ObTransferUtils::process_start_in_perf_diag_info(tx_start_transfer_in_info, + ObStorageHACostItemType::ACCUM_COST_TYPE, ObStorageHACostItemName::START_TRANSFER_IN_REPLAY_COST, + ret, end_ts - start_ts, start_ts, is_report); + ObTransferUtils::add_transfer_error_diagnose_in_replay( + tx_start_transfer_in_info.task_id_, + tx_start_transfer_in_info.dest_ls_id_, + ret, + false/*clean_related_info*/, + ObStorageHADiagTaskType::TRANSFER_START_IN, + diagnose_result_msg); return ret; } @@ -1541,6 +1655,12 @@ int ObTabletStartTransferInHelper::on_replay_success_( const int64_t start_ts = ObTimeUtil::current_time(); LOG_INFO("[TRANSFER] start tx start transfer in on_replay_success_", K(tx_start_transfer_in_info)); + ObTransferUtils::process_start_in_perf_diag_info(tx_start_transfer_in_info, + ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::START_TRANSFER_IN_LOG_SCN, + ret, scn.get_val_for_logservice(), start_ts, false/*is_report*/); + ObTransferUtils::process_start_in_perf_diag_info(tx_start_transfer_in_info, + ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::START_TRANSFER_IN_FIRST_REPALY_LOG_TIMESTAMP, + ret, start_ts, start_ts, false/*is_report*/); #ifdef ERRSIM SERVER_EVENT_ADD("transfer", "tx_start_transfer_in", "stage", "on_replay_success", diff --git a/tools/deploy/mysql_test/r/mysql/information_schema.result b/tools/deploy/mysql_test/r/mysql/information_schema.result index 7312a5956..a5bd60789 100644 --- a/tools/deploy/mysql_test/r/mysql/information_schema.result +++ b/tools/deploy/mysql_test/r/mysql/information_schema.result @@ -2886,6 +2886,12 @@ select * from information_schema.statistics where table_schema in ('oceanbase', | def | oceanbase | __all_spatial_reference_systems | 0 | oceanbase | PRIMARY | 1 | srs_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_spm_config | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_spm_config | 0 | oceanbase | PRIMARY | 2 | name | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_storage_ha_error_diagnose_history | 0 | oceanbase | PRIMARY | 1 | gmt_create | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_storage_ha_error_diagnose_history | 0 | oceanbase | PRIMARY | 2 | svr_ip | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_storage_ha_error_diagnose_history | 0 | oceanbase | PRIMARY | 3 | svr_port | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_storage_ha_perf_diagnose_history | 0 | oceanbase | PRIMARY | 1 | gmt_create | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_storage_ha_perf_diagnose_history | 0 | oceanbase | PRIMARY | 2 | svr_ip | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | +| def | oceanbase | __all_storage_ha_perf_diagnose_history | 0 | oceanbase | PRIMARY | 3 | svr_port | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_sub_part | 0 | oceanbase | PRIMARY | 1 | tenant_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_sub_part | 0 | oceanbase | PRIMARY | 2 | table_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | | def | oceanbase | __all_sub_part | 0 | oceanbase | PRIMARY | 3 | part_id | A | NULL | NULL | NULL | | BTREE | VALID | | YES | NULL | diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result index 9b0f1c74e..32189566e 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result @@ -336,6 +336,7 @@ _force_skip_encoding_partition_id _global_enable_rich_vector_format _hash_area_size _hash_join_enabled +_ha_diagnose_history_recycle_interval _ha_rpc_timeout _ha_tablet_info_batch_count _hidden_sys_tenant_memory diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result index fedd05dda..721433cc6 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result @@ -9011,6 +9011,49 @@ is_deleted bigint(20) NO NULL select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_column_group_mapping_history; IF(count(*) >= 0, 1, 0) 1 +desc oceanbase.__all_virtual_storage_ha_error_diagnose; +Field Type Null Key Default Extra +tenant_id bigint(20) NO NULL +ls_id bigint(20) NO NULL +module varchar(64) NO NULL +type varchar(64) NO NULL +task_id varchar(64) NO NULL +svr_ip varchar(46) NO NULL +svr_port bigint(20) NO NULL +retry_id bigint(20) NO NULL +create_time timestamp(6) NO NULL +result_code bigint(20) NO NULL +result_msg varchar(1024) NO NULL +info longtext NO NULL +select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_storage_ha_error_diagnose; +IF(count(*) >= 0, 1, 0) +1 +"oceanbase.__all_virtual_storage_ha_error_diagnose runs in single server" +IF(count(*) >= 0, 1, 0) +1 +desc oceanbase.__all_virtual_storage_ha_perf_diagnose; +Field Type Null Key Default Extra +tenant_id bigint(20) NO NULL +ls_id bigint(20) NO NULL +module varchar(64) NO NULL +type varchar(64) NO NULL +task_id varchar(64) NO NULL +svr_ip varchar(46) NO NULL +svr_port bigint(20) NO NULL +retry_id bigint(20) NO NULL +start_timestamp timestamp(6) NO NULL +end_timestamp timestamp(6) NO NULL +tablet_id bigint(20) NO NULL +tablet_count bigint(20) NO NULL +result_code bigint(20) NO NULL +result_msg varchar(1024) NO NULL +info longtext NO NULL +select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_storage_ha_perf_diagnose; +IF(count(*) >= 0, 1, 0) +1 +"oceanbase.__all_virtual_storage_ha_perf_diagnose runs in single server" +IF(count(*) >= 0, 1, 0) +1 desc oceanbase.__all_virtual_clone_job; Field Type Null Key Default Extra tenant_id bigint(20) NO PRI NULL diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result index ad939146c..f4e52346c 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result @@ -275,6 +275,8 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 478 __all_import_table_job_history 0 201001 1 479 __all_import_table_task 0 201001 1 480 __all_import_table_task_history 0 201001 1 +483 __all_storage_ha_error_diagnose_history 0 201001 1 +484 __all_storage_ha_perf_diagnose_history 0 201001 1 485 __all_clone_job 0 201001 1 486 __all_clone_job_history 0 201001 1 493 __all_ncomp_dll 0 201001 1 @@ -704,6 +706,8 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 12430 __all_virtual_column_group_mapping 2 201001 1 12431 __all_virtual_column_group_history 2 201001 1 12432 __all_virtual_column_group_mapping_history 2 201001 1 +12433 __all_virtual_storage_ha_error_diagnose 2 201001 1 +12434 __all_virtual_storage_ha_perf_diagnose 2 201001 1 12435 __all_virtual_clone_job 2 201001 1 12436 __all_virtual_clone_job_history 2 201001 1 12437 __all_virtual_checkpoint_diagnose_memtable_info 2 201001 1