[FEAT MERGE] transfer_diagnose 422 patch 431

src/observer/CMakeLists.txt

@@ -408,6 +408,8 @@ ob_set_subtarget(ob_server virtual_table
   virtual_table/ob_all_virtual_checkpoint_diagnose_info.cpp
   virtual_table/ob_all_virtual_checkpoint_diagnose_memtable_info.cpp
   virtual_table/ob_all_virtual_nic_info.cpp
+  virtual_table/ob_all_virtual_storage_ha_error_diagnose.cpp
+  virtual_table/ob_all_virtual_storage_ha_perf_diagnose.cpp
 )
 
 ob_server_add_target(ob_server)

src/observer/ob_server.cpp

@@ -110,6 +110,7 @@
 #include "observer/ob_startup_accel_task_handler.h"
 #include "share/detect/ob_detect_manager.h"
 #include "observer/table/ttl/ob_table_ttl_task.h"
+#include "storage/high_availability/ob_storage_ha_diagnose_service.h"
 #ifdef OB_BUILD_ARBITRATION
 #include "logservice/arbserver/palf_env_lite_mgr.h"
 #include "logservice/arbserver/ob_arb_srv_network_frame.h"
@@ -506,6 +507,8 @@ int ObServer::init(const ObServerOptions &opts, const ObPLogWriterCfg &log_cfg)
       LOG_WARN("init ObDetectManagerThread failed", KR(ret));
     } else if (OB_FAIL(wr_service_.init())) {
       LOG_WARN("failed to init wr service", K(ret));
+    } else if (OB_FAIL(ObStorageHADiagService::instance().init(GCTX.sql_proxy_))) {
+      LOG_WARN("init storage ha diagnose service failed", K(ret));
     } else {
       GDS.set_rpc_proxy(&rs_rpc_proxy_);
     }
@@ -771,6 +774,9 @@ void ObServer::destroy()
     palf::election::GLOBAL_REPORT_TIMER.destroy();
     FLOG_INFO("global election report timer destroyed");
 
+    ObStorageHADiagService::instance().destroy();
+    FLOG_INFO("storage ha diagnose destroy");
+
     FLOG_INFO("begin to destroy virtual tenant manager");
     ObVirtualTenantManager::get_instance().destroy();
     FLOG_INFO("virtual tenant manager destroyed");
@@ -964,6 +970,12 @@ int ObServer::start()
     }
 #endif
 
+    if (FAILEDx(ObStorageHADiagService::instance().start())) {
+      LOG_ERROR("fail to start storage ha diagnose service", KR(ret));
+    } else {
+      FLOG_INFO("success to start storage ha diagnose service");
+    }
+
     if (FAILEDx(unix_domain_listener_.start())) {
       LOG_ERROR("fail to start unix domain listener", KR(ret));
     } else {
@@ -1446,6 +1458,10 @@ int ObServer::stop()
       FLOG_INFO("rpc network shutdowned");
     }
 
+    FLOG_INFO("begin to stop storage ha diagnose service");
+    ObStorageHADiagService::instance().stop();
+    FLOG_INFO("storage ha diagnose service stopped");
+
     FLOG_INFO("begin to shutdown high priority rpc");
     if (OB_FAIL(net_frame_.high_prio_rpc_shutdown())) {
       FLOG_WARN("fail to shutdown high priority rpc", KR(ret));
@@ -1759,6 +1775,10 @@ int ObServer::wait()
     ObClockGenerator::get_instance().wait();
     FLOG_INFO("wait clock generator success");
 
+    FLOG_INFO("begin to wait storage ha diagnose");
+    ObStorageHADiagService::instance().wait();
+    FLOG_INFO("wait storage ha diagnose success");
+
     gctx_.status_ = SS_STOPPED;
     FLOG_INFO("[OBSERVER_NOTICE] wait observer end", KR(ret));
     if (OB_SUCCESS != fail_ret) {

src/observer/omt/ob_multi_tenant.cpp

@@ -150,6 +150,7 @@
 #include "observer/table/ttl/ob_ttl_service.h"
 #include "sql/dtl/ob_dtl_interm_result_manager.h"
 #include "storage/tablet/ob_tablet_memtable_mgr.h"
+#include "storage/high_availability/ob_storage_ha_diagnose_mgr.h"
 #ifdef ERRSIM
 #include "share/errsim_module/ob_tenant_errsim_module_mgr.h"
 #include "share/errsim_module/ob_tenant_errsim_event_mgr.h"
@@ -559,7 +560,7 @@ int ObMultiTenant::init(ObAddr myaddr,
     MTL_BIND2(mtl_new_default, rootserver::ObHeartbeatService::mtl_init, nullptr, rootserver::ObHeartbeatService::mtl_stop, rootserver::ObHeartbeatService::mtl_wait, mtl_destroy_default);
     MTL_BIND2(mtl_new_default, table::ObTTLService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
     MTL_BIND2(mtl_new_default, ObEmptyReadBucket::mtl_init, nullptr, nullptr, nullptr, ObEmptyReadBucket::mtl_destroy);
-
+    MTL_BIND2(mtl_new_default, storage::ObStorageHADiagMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
 #ifdef ERRSIM
     MTL_BIND2(mtl_new_default, ObTenantErrsimModuleMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
     MTL_BIND2(mtl_new_default, ObTenantErrsimEventMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);

src/observer/virtual_table/ob_all_virtual_storage_ha_error_diagnose.cpp

@@ -0,0 +1,204 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include "ob_all_virtual_storage_ha_error_diagnose.h"
+
+namespace oceanbase
+{
+using namespace storage;
+using namespace common;
+using namespace share;
+namespace observer
+{
+ObAllVirtualStorageHAErrorDiagnose::ObAllVirtualStorageHAErrorDiagnose()
+    : info_(nullptr),
+      iter_()
+{
+  memset(ip_buf_, 0, sizeof(ip_buf_));
+  memset(info_str_, 0, sizeof(info_str_));
+  memset(task_id_, 0, sizeof(task_id_));
+}
+
+ObAllVirtualStorageHAErrorDiagnose::~ObAllVirtualStorageHAErrorDiagnose()
+{
+  reset();
+}
+
+int ObAllVirtualStorageHAErrorDiagnose::get_info_from_type_(ObStorageHADiagInfo *&info, ObTransferErrorDiagInfo &transfer_err_diag)
+{
+  int ret = OB_SUCCESS;
+  info = nullptr;
+  ObStorageHADiagTaskKey key;
+  if (OB_FAIL(iter_.get_cur_key(key))) {
+    if (ret != OB_ITER_END) {
+      SERVER_LOG(WARN, "failed to get cur key", K(ret), K(iter_));
+    }
+  } else {
+    switch(key.module_) {
+      case ObStorageHADiagModule::TRANSFER_ERROR_DIAGNOSE: {
+        info = &transfer_err_diag;
+        break;
+      }
+      case ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE:
+        ret = OB_NOT_SUPPORTED;
+        SERVER_LOG(WARN, "not supported perf diagnose", K(ret), K(key.module_));
+        break;
+      default: {
+        ret = OB_INVALID_ARGUMENT;
+        SERVER_LOG(WARN, "invalid module", K(ret), K(key.module_));
+        break;
+      }
+    }
+  }
+
+  return ret;
+}
+
+int ObAllVirtualStorageHAErrorDiagnose::inner_get_next_row(ObNewRow *&row)
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(execute(row))) {
+    SERVER_LOG(WARN, "fail to execute", K(ret));
+  }
+  return ret;
+}
+
+void ObAllVirtualStorageHAErrorDiagnose::release_last_tenant()
+{
+  memset(ip_buf_, 0, sizeof(ip_buf_));
+  memset(info_str_, 0, sizeof(info_str_));
+  memset(task_id_, 0, sizeof(task_id_));
+  info_ = nullptr;
+  iter_.reset();
+}
+
+bool ObAllVirtualStorageHAErrorDiagnose::is_need_process(uint64_t tenant_id)
+{
+  if (!is_virtual_tenant_id(tenant_id)
+      && (is_sys_tenant(effective_tenant_id_) || tenant_id == effective_tenant_id_)) {
+    return true;
+  }
+  return false;
+}
+
+int ObAllVirtualStorageHAErrorDiagnose::process_curr_tenant(common::ObNewRow *&row)
+{
+  int ret = OB_SUCCESS;
+  const int64_t col_count = output_column_ids_.count();
+  ObObj *cells = cur_row_.cells_;
+  ObTransferErrorDiagInfo transfer_err_diag;
+  if (!iter_.is_opened() && OB_FAIL(iter_.open(ObStorageHADiagType::ERROR_DIAGNOSE))) {
+    SERVER_LOG(WARN, "Fail to open storage ha diagnose iter", K(ret), K(iter_));
+  } else if (OB_ISNULL(cur_row_.cells_)) {
+    ret = OB_ERR_UNEXPECTED;
+    SERVER_LOG(ERROR, "cur row cell is NULL", K(ret));
+  } else if (OB_FAIL(get_info_from_type_(info_, transfer_err_diag))) {
+    if (OB_ITER_END != ret) {
+      SERVER_LOG(WARN, "fail to get info");
+    }
+  } else if (OB_FAIL(iter_.get_next_info(*info_))) {
+    if (OB_ITER_END != ret) {
+      STORAGE_LOG(WARN, "Fail to get next diagnose info", K(ret));
+    }
+  } else {
+    for (int64_t i = 0; OB_SUCC(ret) && i < col_count; ++i) {
+      uint64_t col_id = output_column_ids_.at(i);
+      switch (col_id) {
+      case TENANT_ID:
+        cells[i].set_int(MTL_ID());
+        break;
+      case LS_ID:
+        cells[i].set_int(info_->ls_id_.id());
+        break;
+      case MODULE: {
+        cells[i].set_varchar(info_->get_module_str());
+        cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+        break;
+      }
+      case TYPE: {
+        cells[i].set_varchar(info_->get_type_str());
+        cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+        break;
+      }
+      case TASK_ID: {
+        if (OB_FAIL(info_->get_task_id(task_id_, sizeof(task_id_)))) {
+          SERVER_LOG(WARN, "failed to get task id", K(ret));
+        }
+        cells[i].set_varchar(task_id_);
+        cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+        break;
+      }
+      case SVR_IP:
+        //svr_ip
+        if (ObServerConfig::get_instance().self_addr_.ip_to_string(ip_buf_, sizeof(ip_buf_))) {
+          cells[i].set_varchar(ip_buf_);
+          cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+        } else {
+          ret = OB_ERR_UNEXPECTED;
+          SERVER_LOG(WARN, "fail to execute ip_to_string", K(ret));
+        }
+        break;
+      case SVR_PORT:
+        //svr_port
+        cells[i].set_int(ObServerConfig::get_instance().self_addr_.get_port());
+        break;
+      case RETRY_ID:
+        cells[i].set_int(info_->retry_id_);
+        break;
+      case CREATE_TIME:
+        cells[i].set_timestamp(info_->timestamp_);
+        break;
+      case RESULT_CODE:
+        cells[i].set_int(info_->result_code_);
+        break;
+      case RESULT_MSG: {
+        cells[i].set_varchar(info_->get_transfer_error_diagnose_msg());
+        cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+        break;
+      }
+      case INFO: {
+        if (OB_FAIL(info_->get_info(info_str_, sizeof(info_str_)))) {
+          SERVER_LOG(WARN, "failed to get info str", K(ret));
+        }
+        cells[i].set_lob_value(ObLongTextType, info_str_, strlen(info_str_) + 1);
+        cells[i].set_collation_type(ObCharset::get_default_collation(
+                                      ObCharset::get_default_charset()));
+        break;
+      }
+      default:
+        ret = OB_ERR_UNEXPECTED;
+        SERVER_LOG(WARN, "invalid column id", K(ret), K(col_id));
+      }
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    row = &cur_row_;
+  }
+
+  return ret;
+}
+
+void ObAllVirtualStorageHAErrorDiagnose::reset()
+{
+  omt::ObMultiTenantOperator::reset();
+  ObVirtualTableScannerIterator::reset();
+  memset(ip_buf_, 0, sizeof(ip_buf_));
+  memset(info_str_, 0, sizeof(info_str_));
+  memset(task_id_, 0, sizeof(task_id_));
+  info_ = nullptr;
+  iter_.reset();
+}
+
+
+} /* namespace observer */
+} /* namespace oceanbase */

src/observer/virtual_table/ob_all_virtual_storage_ha_error_diagnose.h

@@ -0,0 +1,71 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OB_ALL_VIRTUAL_STORAGE_HA_ERROR_DIAGNOSE_H_
+#define OB_ALL_VIRTUAL_STORAGE_HA_ERROR_DIAGNOSE_H_
+#include "share/ob_virtual_table_scanner_iterator.h"
+#include "share/ob_storage_ha_diagnose_struct.h"
+#include "storage/high_availability/ob_storage_ha_diagnose_mgr.h"
+#include "observer/omt/ob_multi_tenant_operator.h"
+
+namespace oceanbase
+{
+namespace observer
+{
+
+class ObAllVirtualStorageHAErrorDiagnose : public common::ObVirtualTableScannerIterator,
+                                           public omt::ObMultiTenantOperator
+{
+public:
+  enum COLUMN_ID_LIST
+  {
+    TENANT_ID = common::OB_APP_MIN_COLUMN_ID,
+    LS_ID,
+    MODULE,
+    TYPE,
+    TASK_ID,
+    SVR_IP,
+    SVR_PORT,
+    RETRY_ID,
+    CREATE_TIME,
+    RESULT_CODE,
+    RESULT_MSG,
+    INFO,
+  };
+  ObAllVirtualStorageHAErrorDiagnose();
+  virtual ~ObAllVirtualStorageHAErrorDiagnose();
+  int init();
+  virtual int inner_get_next_row(common::ObNewRow *&row);
+  virtual void reset();
+protected:
+  int get_info_from_type_(ObStorageHADiagInfo *&info, ObTransferErrorDiagInfo &transfer_err_diag);
+
+private:
+  // Filter out tenants that need to be processed
+  virtual bool is_need_process(uint64_t tenant_id) override;
+  // Process tenants for the current iteration
+  virtual int process_curr_tenant(common::ObNewRow *&row) override;
+  // Release the resources of the previous tenant
+  virtual void release_last_tenant() override;
+
+private:
+  char ip_buf_[common::OB_IP_STR_BUFF];
+  ObStorageHADiagInfo *info_;
+  ObStorageHADiagIterator iter_;
+  char info_str_[common::OB_DIAGNOSE_INFO_LENGTH];
+  char task_id_[common::OB_MAX_TRACE_ID_BUFFER_SIZE];
+  DISALLOW_COPY_AND_ASSIGN(ObAllVirtualStorageHAErrorDiagnose);
+};
+
+} /* namespace observer */
+} /* namespace oceanbase */
+#endif

src/observer/virtual_table/ob_all_virtual_storage_ha_perf_diagnose.cpp

@@ -0,0 +1,225 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include "ob_all_virtual_storage_ha_perf_diagnose.h"
+
+namespace oceanbase
+{
+using namespace storage;
+using namespace common;
+using namespace share;
+namespace observer
+{
+ObAllVirtualStorageHAPerfDiagnose::ObAllVirtualStorageHAPerfDiagnose()
+  : info_(nullptr),
+    iter_()
+{
+  memset(ip_buf_, 0, sizeof(ip_buf_));
+  memset(info_str_, 0, sizeof(info_str_));
+  memset(task_id_, 0, sizeof(task_id_));
+}
+
+ObAllVirtualStorageHAPerfDiagnose::~ObAllVirtualStorageHAPerfDiagnose()
+{
+  reset();
+}
+
+int ObAllVirtualStorageHAPerfDiagnose::get_info_from_type_(ObStorageHADiagInfo *&info, ObTransferPerfDiagInfo &transfer_diagnose_info)
+{
+  int ret = OB_SUCCESS;
+  info = nullptr;
+  ObStorageHADiagTaskKey key;
+  if (OB_FAIL(iter_.get_cur_key(key))) {
+    if (ret != OB_ITER_END) {
+      SERVER_LOG(WARN, "failed to get cur key", K(ret), K(iter_));
+    }
+  } else {
+    switch(key.module_) {
+      case ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE: {
+        info = &transfer_diagnose_info;
+        break;
+      }
+      case ObStorageHADiagModule::TRANSFER_ERROR_DIAGNOSE:
+        ret = OB_NOT_SUPPORTED;
+        SERVER_LOG(WARN, "not supported perf diagnose", K(ret), K(key.module_));
+        break;
+      default: {
+        ret = OB_INVALID_ARGUMENT;
+        SERVER_LOG(WARN, "invalid module", K(ret), K(key.module_));
+        break;
+      }
+    }
+  }
+
+  return ret;
+}
+
+int ObAllVirtualStorageHAPerfDiagnose::inner_get_next_row(common::ObNewRow *&row)
+{
+  int ret = OB_SUCCESS;
+  if (OB_UNLIKELY(NULL == allocator_)) {
+    ret = OB_NOT_INIT;
+    SERVER_LOG(WARN, "allocator is NULL", K(ret));
+  } else if (OB_FAIL(execute(row))) {
+    SERVER_LOG(WARN, "fail to execute", K(ret));
+  }
+  return ret;
+}
+
+void ObAllVirtualStorageHAPerfDiagnose::release_last_tenant()
+{
+  memset(ip_buf_, 0, sizeof(ip_buf_));
+  memset(info_str_, 0, sizeof(info_str_));
+  memset(task_id_, 0, sizeof(task_id_));
+  info_ = nullptr;
+  iter_.reset();
+}
+
+bool ObAllVirtualStorageHAPerfDiagnose::is_need_process(uint64_t tenant_id)
+{
+  if (!is_virtual_tenant_id(tenant_id)
+      && (is_sys_tenant(effective_tenant_id_) || tenant_id == effective_tenant_id_)) {
+    return true;
+  }
+  return false;
+}
+
+int ObAllVirtualStorageHAPerfDiagnose::process_curr_tenant(common::ObNewRow *&row)
+{
+  int ret = OB_SUCCESS;
+  const int64_t col_count = output_column_ids_.count();
+  ObObj *cells = cur_row_.cells_;
+  ObArenaAllocator alloc;
+  ObTransferPerfDiagInfo transfer_diagnose_info;
+  ObTransferPerfDiagInfo * transfer_perf_info = nullptr;
+  if (OB_FAIL(transfer_diagnose_info.init(&alloc, MTL_ID()))) {
+    SERVER_LOG(WARN, "fail to init info", K(ret));
+  } else if (!iter_.is_opened() && OB_FAIL(iter_.open(ObStorageHADiagType::PERF_DIAGNOSE))) {
+    SERVER_LOG(WARN, "Fail to open storage ha diagnose iter", K(ret), K(iter_));
+  } else if (OB_ISNULL(cur_row_.cells_)) {
+    ret = OB_ERR_UNEXPECTED;
+    SERVER_LOG(ERROR, "cur row cell is NULL", K(ret));
+  } else if (OB_FAIL(get_info_from_type_(info_, transfer_diagnose_info))) {
+    if (OB_ITER_END != ret) {
+      SERVER_LOG(WARN, "fail to get info");
+    }
+  } else if (OB_FAIL(iter_.get_next_info(*info_))) {
+    if (OB_ITER_END != ret) {
+      STORAGE_LOG(WARN, "Fail to get next diagnose info", K(ret));
+    }
+  } else if (ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE != info_->module_) {
+    ret = OB_ERR_UNEXPECTED;
+    STORAGE_LOG(WARN, "unexpected info module", K(ret), K(info_->module_));
+  } else {
+    transfer_perf_info = static_cast<ObTransferPerfDiagInfo *>(info_);
+    for (int64_t i = 0; OB_SUCC(ret) && i < col_count; ++i) {
+      uint64_t col_id = output_column_ids_.at(i);
+      switch (col_id) {
+      case TENANT_ID:
+        cells[i].set_int(MTL_ID());
+        break;
+      case LS_ID:
+        cells[i].set_int(info_->ls_id_.id());
+        break;
+      case MODULE: {
+        cells[i].set_varchar(info_->get_module_str());
+        cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+        break;
+      }
+      case TYPE: {
+        cells[i].set_varchar(info_->get_type_str());
+        cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+        break;
+      }
+      case TASK_ID: {
+        if (OB_FAIL(info_->get_task_id(task_id_, sizeof(task_id_)))) {
+          SERVER_LOG(WARN, "failed to get task id", K(ret));
+        }
+        cells[i].set_varchar(task_id_);
+        cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+        break;
+      }
+      case SVR_IP:
+        //svr_ip
+        if (ObServerConfig::get_instance().self_addr_.ip_to_string(ip_buf_, sizeof(ip_buf_))) {
+          cells[i].set_varchar(ip_buf_);
+          cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+        } else {
+          ret = OB_ERR_UNEXPECTED;
+          SERVER_LOG(WARN, "fail to execute ip_to_string", K(ret));
+        }
+        break;
+      case SVR_PORT:
+        //svr_port
+        cells[i].set_int(ObServerConfig::get_instance().self_addr_.get_port());
+        break;
+      case RETRY_ID:
+        cells[i].set_int(info_->retry_id_);
+        break;
+      case START_TIMESTAMP:
+        cells[i].set_timestamp(info_->timestamp_);
+        break;
+      case END_TIMESTAMP: {
+        cells[i].set_timestamp(transfer_perf_info->end_timestamp_);
+        break;
+      }
+      case TABLET_ID: {
+        cells[i].set_int(info_->tablet_id_.id());
+        break;
+      }
+      case TABLET_COUNT: {
+        cells[i].set_int(transfer_perf_info->tablet_count_);
+        break;
+      }
+      case RESULT_CODE:
+        cells[i].set_int(info_->result_code_);
+        break;
+      case RESULT_MSG: {
+        cells[i].set_varchar(info_->get_transfer_error_diagnose_msg());
+        cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+        break;
+      }
+      case INFO: {
+        if (OB_FAIL(info_->get_info(info_str_, sizeof(info_str_)))) {
+          SERVER_LOG(WARN, "failed to get info str", K(ret));
+        }
+        cells[i].set_lob_value(ObLongTextType, info_str_, strlen(info_str_) + 1);
+        cells[i].set_collation_type(ObCharset::get_default_collation(
+                                      ObCharset::get_default_charset()));
+        break;
+      }
+      default:
+        ret = OB_ERR_UNEXPECTED;
+        SERVER_LOG(WARN, "invalid column id", K(ret), K(col_id));
+      }
+    }
+  }
+  if (OB_SUCC(ret)) {
+    row = &cur_row_;
+  }
+
+  return ret;
+}
+
+void ObAllVirtualStorageHAPerfDiagnose::reset()
+{
+  omt::ObMultiTenantOperator::reset();
+  ObVirtualTableScannerIterator::reset();
+  memset(ip_buf_, 0, sizeof(ip_buf_));
+  memset(info_str_, 0, sizeof(info_str_));
+  memset(task_id_, 0, sizeof(task_id_));
+  info_ = nullptr;
+  iter_.reset();
+}
+
+} /* namespace observer */
+} /* namespace oceanbase */

src/observer/virtual_table/ob_all_virtual_storage_ha_perf_diagnose.h

@@ -0,0 +1,74 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OB_ALL_VIRTUAL_STORAGE_HA_PERF_DIAGNOSE_H_
+#define OB_ALL_VIRTUAL_STORAGE_HA_PERF_DIAGNOSE_H_
+#include "share/ob_virtual_table_scanner_iterator.h"
+#include "share/ob_storage_ha_diagnose_struct.h"
+#include "storage/high_availability/ob_storage_ha_diagnose_mgr.h"
+#include "observer/omt/ob_multi_tenant_operator.h"
+
+namespace oceanbase
+{
+namespace observer
+{
+
+class ObAllVirtualStorageHAPerfDiagnose : public common::ObVirtualTableScannerIterator,
+                                          public omt::ObMultiTenantOperator
+{
+public:
+  enum COLUMN_ID_LIST
+  {
+    TENANT_ID = common::OB_APP_MIN_COLUMN_ID,
+    LS_ID,
+    MODULE,
+    TYPE,
+    TASK_ID,
+    SVR_IP,
+    SVR_PORT,
+    RETRY_ID,
+    START_TIMESTAMP,
+    END_TIMESTAMP,
+    TABLET_ID,
+    TABLET_COUNT,
+    RESULT_CODE,
+    RESULT_MSG,
+    INFO,
+  };
+  ObAllVirtualStorageHAPerfDiagnose();
+  virtual ~ObAllVirtualStorageHAPerfDiagnose();
+  virtual int inner_get_next_row(common::ObNewRow *&row);
+  virtual void reset();
+protected:
+  int get_info_from_type_(ObStorageHADiagInfo *&info, ObTransferPerfDiagInfo &transfer_diagnose_info);
+
+private:
+  // 过滤得到需要处理的租户
+  virtual bool is_need_process(uint64_t tenant_id) override;
+  // 处理当前迭代的租户
+  virtual int process_curr_tenant(common::ObNewRow *&row) override;
+  // 释放上一个租户的资源
+  virtual void release_last_tenant() override;
+  static constexpr int64_t OB_STORAGE_HA_DIAGNOSE_INFO_LENGTH = (1 << 12);// 4k
+
+private:
+  char ip_buf_[common::OB_IP_STR_BUFF];
+  ObStorageHADiagInfo *info_;
+  ObStorageHADiagIterator iter_;
+  char info_str_[OB_STORAGE_HA_DIAGNOSE_INFO_LENGTH];
+  char task_id_[common::OB_MAX_TRACE_ID_BUFFER_SIZE];
+  DISALLOW_COPY_AND_ASSIGN(ObAllVirtualStorageHAPerfDiagnose);
+};
+
+} /* namespace observer */
+} /* namespace oceanbase */
+#endif

src/observer/virtual_table/ob_virtual_table_iterator_factory.cpp

@@ -219,7 +219,9 @@
 #include "observer/virtual_table/ob_all_virtual_activity_metrics.h"
 #include "observer/virtual_table/ob_all_virtual_checkpoint_diagnose_info.h"
 #include "observer/virtual_table/ob_all_virtual_checkpoint_diagnose_memtable_info.h"
-
+#include "observer/virtual_table/ob_all_virtual_storage_ha_error_diagnose.h"
+#include "observer/virtual_table/ob_all_virtual_storage_ha_perf_diagnose.h"
+#include "observer/virtual_table/ob_tenant_show_restore_preview.h"
 #include "observer/virtual_table/ob_all_virtual_kv_connection.h"
 #include "observer/virtual_table/ob_tenant_show_restore_preview.h"
 #include "observer/virtual_table/ob_all_virtual_nic_info.h"
@@ -2658,6 +2660,24 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam &params,
             }
             break;
           }
+          case OB_ALL_VIRTUAL_STORAGE_HA_ERROR_DIAGNOSE_TID: {
+            ObAllVirtualStorageHAErrorDiagnose *storage_ha_error_diagnose = NULL;
+            if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualStorageHAErrorDiagnose, storage_ha_error_diagnose))) {
+              SERVER_LOG(ERROR, "ObAllVirtualStorageHAErrorDiagnose construct failed", K(ret));
+            } else {
+              vt_iter = static_cast<ObVirtualTableIterator *>(storage_ha_error_diagnose);
+            }
+            break;
+          }
+          case OB_ALL_VIRTUAL_STORAGE_HA_PERF_DIAGNOSE_TID: {
+            ObAllVirtualStorageHAPerfDiagnose *storage_ha_perf_diagnose = NULL;
+            if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualStorageHAPerfDiagnose, storage_ha_perf_diagnose))) {
+              SERVER_LOG(ERROR, "ObAllVirtualStorageHAPerfDiagnose construct failed", K(ret));
+            } else {
+              vt_iter = static_cast<ObVirtualTableIterator *>(storage_ha_perf_diagnose);
+            }
+            break;
+          }
         END_CREATE_VT_ITER_SWITCH_LAMBDA
 
 #define AGENT_VIRTUAL_TABLE_CREATE_ITER

src/share/CMakeLists.txt

@@ -184,6 +184,8 @@ ob_set_subtarget(ob_share common
   ob_cluster_event_history_table_operator.cpp
   scn.cpp
   ob_throttling_utils.cpp
+  ob_storage_ha_diagnose_struct.cpp
+  ob_storage_ha_diagnose_operator.cpp
 )
 
 ob_set_subtarget(ob_share common_mixed

src/share/inner_table/ob_inner_table_schema.12401_12450.cpp

@@ -9593,6 +9593,522 @@ int ObInnerTableSchema::all_virtual_column_group_mapping_history_schema(ObTableS
   return ret;
 }
 
+int ObInnerTableSchema::all_virtual_storage_ha_error_diagnose_schema(ObTableSchema &table_schema)
+{
+  int ret = OB_SUCCESS;
+  uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1;
+
+  //generated fields:
+  table_schema.set_tenant_id(OB_SYS_TENANT_ID);
+  table_schema.set_tablegroup_id(OB_INVALID_ID);
+  table_schema.set_database_id(OB_SYS_DATABASE_ID);
+  table_schema.set_table_id(OB_ALL_VIRTUAL_STORAGE_HA_ERROR_DIAGNOSE_TID);
+  table_schema.set_rowkey_split_pos(0);
+  table_schema.set_is_use_bloomfilter(false);
+  table_schema.set_progressive_merge_num(0);
+  table_schema.set_rowkey_column_num(0);
+  table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK);
+  table_schema.set_table_type(VIRTUAL_TABLE);
+  table_schema.set_index_type(INDEX_TYPE_IS_NOT);
+  table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL);
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_STORAGE_HA_ERROR_DIAGNOSE_TNAME))) {
+      LOG_ERROR("fail to set table_name", K(ret));
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) {
+      LOG_ERROR("fail to set compress_func_name", K(ret));
+    }
+  }
+  table_schema.set_part_level(PARTITION_LEVEL_ZERO);
+  table_schema.set_charset_type(ObCharset::get_default_charset());
+  table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("tenant_id", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("ls_id", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("module", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      OB_MODULE_NAME_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("type", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      OB_SYS_TASK_TYPE_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("task_id", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      OB_MAX_TRACE_ID_BUFFER_SIZE, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("svr_ip", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      1, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      MAX_IP_ADDR_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("svr_port", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      2, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("retry_id", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA_TS("create_time", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObTimestampType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(ObPreciseDateTime), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      false); //is_on_update_for_timestamp
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("result_code", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("result_msg", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      OB_DIAGNOSE_INFO_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("info", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObLongTextType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      0, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+  if (OB_SUCC(ret)) {
+    table_schema.get_part_option().set_part_num(1);
+    table_schema.set_part_level(PARTITION_LEVEL_ONE);
+    table_schema.get_part_option().set_part_func_type(PARTITION_FUNC_TYPE_LIST_COLUMNS);
+    if (OB_FAIL(table_schema.get_part_option().set_part_expr("svr_ip, svr_port"))) {
+      LOG_WARN("set_part_expr failed", K(ret));
+    } else if (OB_FAIL(table_schema.mock_list_partition_array())) {
+      LOG_WARN("mock list partition array failed", K(ret));
+    }
+  }
+  table_schema.set_index_using_type(USING_HASH);
+  table_schema.set_row_store_type(ENCODING_ROW_STORE);
+  table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL);
+  table_schema.set_progressive_merge_round(1);
+  table_schema.set_storage_format_version(3);
+  table_schema.set_tablet_id(0);
+
+  table_schema.set_max_used_column_id(column_id);
+  return ret;
+}
+
+int ObInnerTableSchema::all_virtual_storage_ha_perf_diagnose_schema(ObTableSchema &table_schema)
+{
+  int ret = OB_SUCCESS;
+  uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1;
+
+  //generated fields:
+  table_schema.set_tenant_id(OB_SYS_TENANT_ID);
+  table_schema.set_tablegroup_id(OB_INVALID_ID);
+  table_schema.set_database_id(OB_SYS_DATABASE_ID);
+  table_schema.set_table_id(OB_ALL_VIRTUAL_STORAGE_HA_PERF_DIAGNOSE_TID);
+  table_schema.set_rowkey_split_pos(0);
+  table_schema.set_is_use_bloomfilter(false);
+  table_schema.set_progressive_merge_num(0);
+  table_schema.set_rowkey_column_num(0);
+  table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK);
+  table_schema.set_table_type(VIRTUAL_TABLE);
+  table_schema.set_index_type(INDEX_TYPE_IS_NOT);
+  table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL);
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_STORAGE_HA_PERF_DIAGNOSE_TNAME))) {
+      LOG_ERROR("fail to set table_name", K(ret));
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) {
+      LOG_ERROR("fail to set compress_func_name", K(ret));
+    }
+  }
+  table_schema.set_part_level(PARTITION_LEVEL_ZERO);
+  table_schema.set_charset_type(ObCharset::get_default_charset());
+  table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("tenant_id", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("ls_id", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("module", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      OB_MODULE_NAME_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("type", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      OB_SYS_TASK_TYPE_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("task_id", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      OB_MAX_TRACE_ID_BUFFER_SIZE, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("svr_ip", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      1, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      MAX_IP_ADDR_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("svr_port", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      2, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("retry_id", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA_TS("start_timestamp", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObTimestampType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(ObPreciseDateTime), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      false); //is_on_update_for_timestamp
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA_TS("end_timestamp", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObTimestampType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(ObPreciseDateTime), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      false); //is_on_update_for_timestamp
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("tablet_id", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("tablet_count", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("result_code", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("result_msg", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      OB_DIAGNOSE_INFO_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("info", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObLongTextType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      0, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+  if (OB_SUCC(ret)) {
+    table_schema.get_part_option().set_part_num(1);
+    table_schema.set_part_level(PARTITION_LEVEL_ONE);
+    table_schema.get_part_option().set_part_func_type(PARTITION_FUNC_TYPE_LIST_COLUMNS);
+    if (OB_FAIL(table_schema.get_part_option().set_part_expr("svr_ip, svr_port"))) {
+      LOG_WARN("set_part_expr failed", K(ret));
+    } else if (OB_FAIL(table_schema.mock_list_partition_array())) {
+      LOG_WARN("mock list partition array failed", K(ret));
+    }
+  }
+  table_schema.set_index_using_type(USING_HASH);
+  table_schema.set_row_store_type(ENCODING_ROW_STORE);
+  table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL);
+  table_schema.set_progressive_merge_round(1);
+  table_schema.set_storage_format_version(3);
+  table_schema.set_tablet_id(0);
+
+  table_schema.set_max_used_column_id(column_id);
+  return ret;
+}
+
 int ObInnerTableSchema::all_virtual_clone_job_schema(ObTableSchema &table_schema)
 {
   int ret = OB_SUCCESS;

src/share/inner_table/ob_inner_table_schema.451_500.cpp

@@ -9947,6 +9947,538 @@ int ObInnerTableSchema::all_import_table_task_history_schema(ObTableSchema &tabl
   return ret;
 }
 
+int ObInnerTableSchema::all_storage_ha_error_diagnose_history_schema(ObTableSchema &table_schema)
+{
+  int ret = OB_SUCCESS;
+  uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1;
+
+  //generated fields:
+  table_schema.set_tenant_id(OB_SYS_TENANT_ID);
+  table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID);
+  table_schema.set_database_id(OB_SYS_DATABASE_ID);
+  table_schema.set_table_id(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_TID);
+  table_schema.set_rowkey_split_pos(0);
+  table_schema.set_is_use_bloomfilter(false);
+  table_schema.set_progressive_merge_num(0);
+  table_schema.set_rowkey_column_num(3);
+  table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK);
+  table_schema.set_table_type(SYSTEM_TABLE);
+  table_schema.set_index_type(INDEX_TYPE_IS_NOT);
+  table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL);
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_table_name(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_TNAME))) {
+      LOG_ERROR("fail to set table_name", K(ret));
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) {
+      LOG_ERROR("fail to set compress_func_name", K(ret));
+    }
+  }
+  table_schema.set_part_level(PARTITION_LEVEL_ZERO);
+  table_schema.set_charset_type(ObCharset::get_default_charset());
+  table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA_TS("gmt_create", //column_name
+      ++column_id, //column_id
+      1, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObTimestampType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(ObPreciseDateTime), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      false); //is_on_update_for_timestamp
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("svr_ip", //column_name
+      ++column_id, //column_id
+      2, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      MAX_IP_ADDR_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("svr_port", //column_name
+      ++column_id, //column_id
+      3, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("tenant_id", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("ls_id", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("module", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      OB_MODULE_NAME_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("type", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      OB_SYS_TASK_TYPE_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("task_id", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      OB_MAX_TRACE_ID_BUFFER_SIZE, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("retry_id", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA_TS("create_time", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObTimestampType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(ObPreciseDateTime), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      false); //is_on_update_for_timestamp
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("result_code", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("result_msg", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      OB_DIAGNOSE_INFO_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("info", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObLongTextType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      0, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+  table_schema.set_index_using_type(USING_BTREE);
+  table_schema.set_row_store_type(ENCODING_ROW_STORE);
+  table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL);
+  table_schema.set_progressive_merge_round(1);
+  table_schema.set_storage_format_version(3);
+  table_schema.set_tablet_id(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_TID);
+  table_schema.set_aux_lob_meta_tid(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_META_TID);
+  table_schema.set_aux_lob_piece_tid(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TID);
+
+  table_schema.set_max_used_column_id(column_id);
+  return ret;
+}
+
+int ObInnerTableSchema::all_storage_ha_perf_diagnose_history_schema(ObTableSchema &table_schema)
+{
+  int ret = OB_SUCCESS;
+  uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1;
+
+  //generated fields:
+  table_schema.set_tenant_id(OB_SYS_TENANT_ID);
+  table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID);
+  table_schema.set_database_id(OB_SYS_DATABASE_ID);
+  table_schema.set_table_id(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_TID);
+  table_schema.set_rowkey_split_pos(0);
+  table_schema.set_is_use_bloomfilter(false);
+  table_schema.set_progressive_merge_num(0);
+  table_schema.set_rowkey_column_num(3);
+  table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK);
+  table_schema.set_table_type(SYSTEM_TABLE);
+  table_schema.set_index_type(INDEX_TYPE_IS_NOT);
+  table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL);
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_table_name(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_TNAME))) {
+      LOG_ERROR("fail to set table_name", K(ret));
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) {
+      LOG_ERROR("fail to set compress_func_name", K(ret));
+    }
+  }
+  table_schema.set_part_level(PARTITION_LEVEL_ZERO);
+  table_schema.set_charset_type(ObCharset::get_default_charset());
+  table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA_TS("gmt_create", //column_name
+      ++column_id, //column_id
+      1, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObTimestampType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(ObPreciseDateTime), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      false); //is_on_update_for_timestamp
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("svr_ip", //column_name
+      ++column_id, //column_id
+      2, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      MAX_IP_ADDR_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("svr_port", //column_name
+      ++column_id, //column_id
+      3, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("tenant_id", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("ls_id", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("module", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      OB_MODULE_NAME_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("type", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      OB_SYS_TASK_TYPE_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("task_id", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      OB_MAX_TRACE_ID_BUFFER_SIZE, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("retry_id", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA_TS("start_timestamp", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObTimestampType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(ObPreciseDateTime), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      false); //is_on_update_for_timestamp
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA_TS("end_timestamp", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObTimestampType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(ObPreciseDateTime), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      false); //is_on_update_for_timestamp
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("tablet_id", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("tablet_count", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("result_code", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("result_msg", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      OB_DIAGNOSE_INFO_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("info", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObLongTextType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      0, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+  table_schema.set_index_using_type(USING_BTREE);
+  table_schema.set_row_store_type(ENCODING_ROW_STORE);
+  table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL);
+  table_schema.set_progressive_merge_round(1);
+  table_schema.set_storage_format_version(3);
+  table_schema.set_tablet_id(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_TID);
+  table_schema.set_aux_lob_meta_tid(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_META_TID);
+  table_schema.set_aux_lob_piece_tid(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TID);
+
+  table_schema.set_max_used_column_id(column_id);
+  return ret;
+}
+
 int ObInnerTableSchema::all_clone_job_schema(ObTableSchema &table_schema)
 {
   int ret = OB_SUCCESS;

src/share/inner_table/ob_inner_table_schema.50451_50500.cpp

@@ -3940,6 +3940,276 @@ int ObInnerTableSchema::all_import_table_task_history_aux_lob_meta_schema(ObTabl
   return ret;
 }
 
+int ObInnerTableSchema::all_storage_ha_error_diagnose_history_aux_lob_meta_schema(ObTableSchema &table_schema)
+{
+  int ret = OB_SUCCESS;
+  uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1;
+
+  //generated fields:
+  table_schema.set_tenant_id(OB_SYS_TENANT_ID);
+  table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID);
+  table_schema.set_database_id(OB_SYS_DATABASE_ID);
+  table_schema.set_table_id(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_META_TID);
+  table_schema.set_rowkey_split_pos(0);
+  table_schema.set_is_use_bloomfilter(false);
+  table_schema.set_progressive_merge_num(0);
+  table_schema.set_rowkey_column_num(2);
+  table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK);
+  table_schema.set_table_type(AUX_LOB_META);
+  table_schema.set_index_type(INDEX_TYPE_IS_NOT);
+  table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL);
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_table_name(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_META_TNAME))) {
+      LOG_ERROR("fail to set table_name", K(ret));
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) {
+      LOG_ERROR("fail to set compress_func_name", K(ret));
+    }
+  }
+  table_schema.set_part_level(PARTITION_LEVEL_ZERO);
+  table_schema.set_charset_type(ObCharset::get_default_charset());
+  table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("lob_id", //column_name
+      ++column_id, //column_id
+      1, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_BINARY, //column_collation_type
+      16, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("seq_id", //column_name
+      ++column_id, //column_id
+      2, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_BINARY, //column_collation_type
+      8192, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("binary_len", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObUInt32Type, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(uint32_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("char_len", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObUInt32Type, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(uint32_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("piece_id", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObUInt64Type, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(uint64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("lob_data", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_BINARY, //column_collation_type
+      262144, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+  table_schema.set_index_using_type(USING_BTREE);
+  table_schema.set_row_store_type(ENCODING_ROW_STORE);
+  table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL);
+  table_schema.set_progressive_merge_round(1);
+  table_schema.set_storage_format_version(3);
+  table_schema.set_tablet_id(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_META_TID);
+  table_schema.set_data_table_id(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_TID);
+
+  table_schema.set_max_used_column_id(column_id);
+  return ret;
+}
+
+int ObInnerTableSchema::all_storage_ha_perf_diagnose_history_aux_lob_meta_schema(ObTableSchema &table_schema)
+{
+  int ret = OB_SUCCESS;
+  uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1;
+
+  //generated fields:
+  table_schema.set_tenant_id(OB_SYS_TENANT_ID);
+  table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID);
+  table_schema.set_database_id(OB_SYS_DATABASE_ID);
+  table_schema.set_table_id(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_META_TID);
+  table_schema.set_rowkey_split_pos(0);
+  table_schema.set_is_use_bloomfilter(false);
+  table_schema.set_progressive_merge_num(0);
+  table_schema.set_rowkey_column_num(2);
+  table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK);
+  table_schema.set_table_type(AUX_LOB_META);
+  table_schema.set_index_type(INDEX_TYPE_IS_NOT);
+  table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL);
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_table_name(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_META_TNAME))) {
+      LOG_ERROR("fail to set table_name", K(ret));
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) {
+      LOG_ERROR("fail to set compress_func_name", K(ret));
+    }
+  }
+  table_schema.set_part_level(PARTITION_LEVEL_ZERO);
+  table_schema.set_charset_type(ObCharset::get_default_charset());
+  table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("lob_id", //column_name
+      ++column_id, //column_id
+      1, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_BINARY, //column_collation_type
+      16, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("seq_id", //column_name
+      ++column_id, //column_id
+      2, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_BINARY, //column_collation_type
+      8192, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("binary_len", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObUInt32Type, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(uint32_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("char_len", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObUInt32Type, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(uint32_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("piece_id", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObUInt64Type, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(uint64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("lob_data", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_BINARY, //column_collation_type
+      262144, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+  table_schema.set_index_using_type(USING_BTREE);
+  table_schema.set_row_store_type(ENCODING_ROW_STORE);
+  table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL);
+  table_schema.set_progressive_merge_round(1);
+  table_schema.set_storage_format_version(3);
+  table_schema.set_tablet_id(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_META_TID);
+  table_schema.set_data_table_id(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_TID);
+
+  table_schema.set_max_used_column_id(column_id);
+  return ret;
+}
+
 int ObInnerTableSchema::all_clone_job_aux_lob_meta_schema(ObTableSchema &table_schema)
 {
   int ret = OB_SUCCESS;

src/share/inner_table/ob_inner_table_schema.60451_60500.cpp

@@ -2635,6 +2635,186 @@ int ObInnerTableSchema::all_import_table_task_history_aux_lob_piece_schema(ObTab
   return ret;
 }
 
+int ObInnerTableSchema::all_storage_ha_error_diagnose_history_aux_lob_piece_schema(ObTableSchema &table_schema)
+{
+  int ret = OB_SUCCESS;
+  uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1;
+
+  //generated fields:
+  table_schema.set_tenant_id(OB_SYS_TENANT_ID);
+  table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID);
+  table_schema.set_database_id(OB_SYS_DATABASE_ID);
+  table_schema.set_table_id(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TID);
+  table_schema.set_rowkey_split_pos(0);
+  table_schema.set_is_use_bloomfilter(false);
+  table_schema.set_progressive_merge_num(0);
+  table_schema.set_rowkey_column_num(1);
+  table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK);
+  table_schema.set_table_type(AUX_LOB_PIECE);
+  table_schema.set_index_type(INDEX_TYPE_IS_NOT);
+  table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL);
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_table_name(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TNAME))) {
+      LOG_ERROR("fail to set table_name", K(ret));
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) {
+      LOG_ERROR("fail to set compress_func_name", K(ret));
+    }
+  }
+  table_schema.set_part_level(PARTITION_LEVEL_ZERO);
+  table_schema.set_charset_type(ObCharset::get_default_charset());
+  table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("piece_id", //column_name
+      ++column_id, //column_id
+      1, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObUInt64Type, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(uint64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("data_len", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObUInt32Type, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(uint32_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("lob_data", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_BINARY, //column_collation_type
+      32, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+  table_schema.set_index_using_type(USING_BTREE);
+  table_schema.set_row_store_type(ENCODING_ROW_STORE);
+  table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL);
+  table_schema.set_progressive_merge_round(1);
+  table_schema.set_storage_format_version(3);
+  table_schema.set_tablet_id(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TID);
+  table_schema.set_data_table_id(OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_TID);
+
+  table_schema.set_max_used_column_id(column_id);
+  return ret;
+}
+
+int ObInnerTableSchema::all_storage_ha_perf_diagnose_history_aux_lob_piece_schema(ObTableSchema &table_schema)
+{
+  int ret = OB_SUCCESS;
+  uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1;
+
+  //generated fields:
+  table_schema.set_tenant_id(OB_SYS_TENANT_ID);
+  table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID);
+  table_schema.set_database_id(OB_SYS_DATABASE_ID);
+  table_schema.set_table_id(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TID);
+  table_schema.set_rowkey_split_pos(0);
+  table_schema.set_is_use_bloomfilter(false);
+  table_schema.set_progressive_merge_num(0);
+  table_schema.set_rowkey_column_num(1);
+  table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK);
+  table_schema.set_table_type(AUX_LOB_PIECE);
+  table_schema.set_index_type(INDEX_TYPE_IS_NOT);
+  table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL);
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_table_name(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TNAME))) {
+      LOG_ERROR("fail to set table_name", K(ret));
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) {
+      LOG_ERROR("fail to set compress_func_name", K(ret));
+    }
+  }
+  table_schema.set_part_level(PARTITION_LEVEL_ZERO);
+  table_schema.set_charset_type(ObCharset::get_default_charset());
+  table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("piece_id", //column_name
+      ++column_id, //column_id
+      1, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObUInt64Type, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(uint64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("data_len", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObUInt32Type, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(uint32_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("lob_data", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_BINARY, //column_collation_type
+      32, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+  table_schema.set_index_using_type(USING_BTREE);
+  table_schema.set_row_store_type(ENCODING_ROW_STORE);
+  table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL);
+  table_schema.set_progressive_merge_round(1);
+  table_schema.set_storage_format_version(3);
+  table_schema.set_tablet_id(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TID);
+  table_schema.set_data_table_id(OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_TID);
+
+  table_schema.set_max_used_column_id(column_id);
+  return ret;
+}
+
 int ObInnerTableSchema::all_clone_job_aux_lob_piece_schema(ObTableSchema &table_schema)
 {
   int ret = OB_SUCCESS;

src/share/inner_table/ob_inner_table_schema.h

@@ -567,6 +567,8 @@ public:
   static int all_import_table_job_history_schema(share::schema::ObTableSchema &table_schema);
   static int all_import_table_task_schema(share::schema::ObTableSchema &table_schema);
   static int all_import_table_task_history_schema(share::schema::ObTableSchema &table_schema);
+  static int all_storage_ha_error_diagnose_history_schema(share::schema::ObTableSchema &table_schema);
+  static int all_storage_ha_perf_diagnose_history_schema(share::schema::ObTableSchema &table_schema);
   static int all_clone_job_schema(share::schema::ObTableSchema &table_schema);
   static int all_clone_job_history_schema(share::schema::ObTableSchema &table_schema);
   static int all_ncomp_dll_schema(share::schema::ObTableSchema &table_schema);
@@ -996,6 +998,8 @@ public:
   static int all_virtual_column_group_mapping_schema(share::schema::ObTableSchema &table_schema);
   static int all_virtual_column_group_history_schema(share::schema::ObTableSchema &table_schema);
   static int all_virtual_column_group_mapping_history_schema(share::schema::ObTableSchema &table_schema);
+  static int all_virtual_storage_ha_error_diagnose_schema(share::schema::ObTableSchema &table_schema);
+  static int all_virtual_storage_ha_perf_diagnose_schema(share::schema::ObTableSchema &table_schema);
   static int all_virtual_clone_job_schema(share::schema::ObTableSchema &table_schema);
   static int all_virtual_clone_job_history_schema(share::schema::ObTableSchema &table_schema);
   static int all_virtual_checkpoint_diagnose_memtable_info_schema(share::schema::ObTableSchema &table_schema);
@@ -2401,6 +2405,8 @@ public:
   static int all_import_table_job_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema);
   static int all_import_table_task_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema);
   static int all_import_table_task_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema);
+  static int all_storage_ha_error_diagnose_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema);
+  static int all_storage_ha_perf_diagnose_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema);
   static int all_clone_job_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema);
   static int all_clone_job_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema);
   static int all_ncomp_dll_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema);
@@ -2686,6 +2692,8 @@ public:
   static int all_import_table_job_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema);
   static int all_import_table_task_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema);
   static int all_import_table_task_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema);
+  static int all_storage_ha_error_diagnose_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema);
+  static int all_storage_ha_perf_diagnose_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema);
   static int all_clone_job_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema);
   static int all_clone_job_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema);
   static int all_ncomp_dll_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema);
@@ -3180,6 +3188,8 @@ const schema_create_func sys_table_schema_creators [] = {
   ObInnerTableSchema::all_import_table_job_history_schema,
   ObInnerTableSchema::all_import_table_task_schema,
   ObInnerTableSchema::all_import_table_task_history_schema,
+  ObInnerTableSchema::all_storage_ha_error_diagnose_history_schema,
+  ObInnerTableSchema::all_storage_ha_perf_diagnose_history_schema,
   ObInnerTableSchema::all_clone_job_schema,
   ObInnerTableSchema::all_clone_job_history_schema,
   ObInnerTableSchema::all_ncomp_dll_schema,
@@ -3612,6 +3622,8 @@ const schema_create_func virtual_table_schema_creators [] = {
   ObInnerTableSchema::all_virtual_column_group_mapping_schema,
   ObInnerTableSchema::all_virtual_column_group_history_schema,
   ObInnerTableSchema::all_virtual_column_group_mapping_history_schema,
+  ObInnerTableSchema::all_virtual_storage_ha_error_diagnose_schema,
+  ObInnerTableSchema::all_virtual_storage_ha_perf_diagnose_schema,
   ObInnerTableSchema::all_virtual_clone_job_schema,
   ObInnerTableSchema::all_virtual_clone_job_history_schema,
   ObInnerTableSchema::all_virtual_checkpoint_diagnose_memtable_info_schema,
@@ -9683,7 +9695,9 @@ const uint64_t cluster_distributed_vtables [] = {
   OB_ALL_VIRTUAL_HA_DIAGNOSE_TID,
   OB_ALL_VIRTUAL_IO_SCHEDULER_TID,
   OB_ALL_VIRTUAL_TX_DATA_TID,
-  OB_ALL_VIRTUAL_STORAGE_LEAK_INFO_TID,  };
+  OB_ALL_VIRTUAL_STORAGE_LEAK_INFO_TID,
+  OB_ALL_VIRTUAL_STORAGE_HA_ERROR_DIAGNOSE_TID,
+  OB_ALL_VIRTUAL_STORAGE_HA_PERF_DIAGNOSE_TID,  };
 
 const uint64_t tenant_distributed_vtables [] = {
   OB_ALL_VIRTUAL_PROCESSLIST_TID,
@@ -12318,6 +12332,22 @@ LOBMapping const lob_aux_table_mappings [] = {
     ObInnerTableSchema::all_import_table_task_history_aux_lob_piece_schema
   },
 
+  {
+    OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_TID,
+    OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_META_TID,
+    OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TID,
+    ObInnerTableSchema::all_storage_ha_error_diagnose_history_aux_lob_meta_schema,
+    ObInnerTableSchema::all_storage_ha_error_diagnose_history_aux_lob_piece_schema
+  },
+
+  {
+    OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_TID,
+    OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_META_TID,
+    OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TID,
+    ObInnerTableSchema::all_storage_ha_perf_diagnose_history_aux_lob_meta_schema,
+    ObInnerTableSchema::all_storage_ha_perf_diagnose_history_aux_lob_piece_schema
+  },
+
   {
     OB_ALL_CLONE_JOB_TID,
     OB_ALL_CLONE_JOB_AUX_LOB_META_TID,
@@ -12435,12 +12465,12 @@ static inline int get_sys_table_lob_aux_schema(const uint64_t tid,
 }
 
 const int64_t OB_CORE_TABLE_COUNT = 4;
-const int64_t OB_SYS_TABLE_COUNT = 282;
-const int64_t OB_VIRTUAL_TABLE_COUNT = 786;
+const int64_t OB_SYS_TABLE_COUNT = 284;
+const int64_t OB_VIRTUAL_TABLE_COUNT = 788;
 const int64_t OB_SYS_VIEW_COUNT = 854;
-const int64_t OB_SYS_TENANT_TABLE_COUNT = 1927;
+const int64_t OB_SYS_TENANT_TABLE_COUNT = 1931;
 const int64_t OB_CORE_SCHEMA_VERSION = 1;
-const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 1930;
+const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 1934;
 
 } // end namespace share
 } // end namespace oceanbase

src/share/inner_table/ob_inner_table_schema.lob.cpp

@@ -21,7 +21,7 @@ inner_lob_map_t inner_lob_map;
 bool lob_mapping_init()
 {
   int ret = OB_SUCCESS;
-  if (OB_FAIL(inner_lob_map.create(285, ObModIds::OB_INNER_LOB_HASH_SET))) {
+  if (OB_FAIL(inner_lob_map.create(287, ObModIds::OB_INNER_LOB_HASH_SET))) {
     SERVER_LOG(WARN, "fail to create inner lob map", K(ret));
   } else {
     for (int64_t i = 0; OB_SUCC(ret) && i < ARRAYSIZEOF(lob_aux_table_mappings); ++i) {

src/share/inner_table/ob_inner_table_schema_constants.h

@@ -303,6 +303,8 @@ const uint64_t OB_ALL_IMPORT_TABLE_JOB_TID = 477; // "__all_import_table_job"
 const uint64_t OB_ALL_IMPORT_TABLE_JOB_HISTORY_TID = 478; // "__all_import_table_job_history"
 const uint64_t OB_ALL_IMPORT_TABLE_TASK_TID = 479; // "__all_import_table_task"
 const uint64_t OB_ALL_IMPORT_TABLE_TASK_HISTORY_TID = 480; // "__all_import_table_task_history"
+const uint64_t OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_TID = 483; // "__all_storage_ha_error_diagnose_history"
+const uint64_t OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_TID = 484; // "__all_storage_ha_perf_diagnose_history"
 const uint64_t OB_ALL_CLONE_JOB_TID = 485; // "__all_clone_job"
 const uint64_t OB_ALL_CLONE_JOB_HISTORY_TID = 486; // "__all_clone_job_history"
 const uint64_t OB_ALL_NCOMP_DLL_TID = 493; // "__all_ncomp_dll"
@@ -732,6 +734,8 @@ const uint64_t OB_ALL_VIRTUAL_DATA_ACTIVITY_METRICS_TID = 12429; // "__all_virtu
 const uint64_t OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_TID = 12430; // "__all_virtual_column_group_mapping"
 const uint64_t OB_ALL_VIRTUAL_COLUMN_GROUP_HISTORY_TID = 12431; // "__all_virtual_column_group_history"
 const uint64_t OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_HISTORY_TID = 12432; // "__all_virtual_column_group_mapping_history"
+const uint64_t OB_ALL_VIRTUAL_STORAGE_HA_ERROR_DIAGNOSE_TID = 12433; // "__all_virtual_storage_ha_error_diagnose"
+const uint64_t OB_ALL_VIRTUAL_STORAGE_HA_PERF_DIAGNOSE_TID = 12434; // "__all_virtual_storage_ha_perf_diagnose"
 const uint64_t OB_ALL_VIRTUAL_CLONE_JOB_TID = 12435; // "__all_virtual_clone_job"
 const uint64_t OB_ALL_VIRTUAL_CLONE_JOB_HISTORY_TID = 12436; // "__all_virtual_clone_job_history"
 const uint64_t OB_ALL_VIRTUAL_CHECKPOINT_DIAGNOSE_MEMTABLE_INFO_TID = 12437; // "__all_virtual_checkpoint_diagnose_memtable_info"
@@ -2137,6 +2141,8 @@ const uint64_t OB_ALL_IMPORT_TABLE_JOB_AUX_LOB_META_TID = 50477; // "__all_impor
 const uint64_t OB_ALL_IMPORT_TABLE_JOB_HISTORY_AUX_LOB_META_TID = 50478; // "__all_import_table_job_history_aux_lob_meta"
 const uint64_t OB_ALL_IMPORT_TABLE_TASK_AUX_LOB_META_TID = 50479; // "__all_import_table_task_aux_lob_meta"
 const uint64_t OB_ALL_IMPORT_TABLE_TASK_HISTORY_AUX_LOB_META_TID = 50480; // "__all_import_table_task_history_aux_lob_meta"
+const uint64_t OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_META_TID = 50483; // "__all_storage_ha_error_diagnose_history_aux_lob_meta"
+const uint64_t OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_META_TID = 50484; // "__all_storage_ha_perf_diagnose_history_aux_lob_meta"
 const uint64_t OB_ALL_CLONE_JOB_AUX_LOB_META_TID = 50485; // "__all_clone_job_aux_lob_meta"
 const uint64_t OB_ALL_CLONE_JOB_HISTORY_AUX_LOB_META_TID = 50486; // "__all_clone_job_history_aux_lob_meta"
 const uint64_t OB_ALL_NCOMP_DLL_AUX_LOB_META_TID = 50493; // "__all_ncomp_dll_aux_lob_meta"
@@ -2422,6 +2428,8 @@ const uint64_t OB_ALL_IMPORT_TABLE_JOB_AUX_LOB_PIECE_TID = 60477; // "__all_impo
 const uint64_t OB_ALL_IMPORT_TABLE_JOB_HISTORY_AUX_LOB_PIECE_TID = 60478; // "__all_import_table_job_history_aux_lob_piece"
 const uint64_t OB_ALL_IMPORT_TABLE_TASK_AUX_LOB_PIECE_TID = 60479; // "__all_import_table_task_aux_lob_piece"
 const uint64_t OB_ALL_IMPORT_TABLE_TASK_HISTORY_AUX_LOB_PIECE_TID = 60480; // "__all_import_table_task_history_aux_lob_piece"
+const uint64_t OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TID = 60483; // "__all_storage_ha_error_diagnose_history_aux_lob_piece"
+const uint64_t OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TID = 60484; // "__all_storage_ha_perf_diagnose_history_aux_lob_piece"
 const uint64_t OB_ALL_CLONE_JOB_AUX_LOB_PIECE_TID = 60485; // "__all_clone_job_aux_lob_piece"
 const uint64_t OB_ALL_CLONE_JOB_HISTORY_AUX_LOB_PIECE_TID = 60486; // "__all_clone_job_history_aux_lob_piece"
 const uint64_t OB_ALL_NCOMP_DLL_AUX_LOB_PIECE_TID = 60493; // "__all_ncomp_dll_aux_lob_piece"
@@ -2903,6 +2911,8 @@ const char *const OB_ALL_IMPORT_TABLE_JOB_TNAME = "__all_import_table_job";
 const char *const OB_ALL_IMPORT_TABLE_JOB_HISTORY_TNAME = "__all_import_table_job_history";
 const char *const OB_ALL_IMPORT_TABLE_TASK_TNAME = "__all_import_table_task";
 const char *const OB_ALL_IMPORT_TABLE_TASK_HISTORY_TNAME = "__all_import_table_task_history";
+const char *const OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_TNAME = "__all_storage_ha_error_diagnose_history";
+const char *const OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_TNAME = "__all_storage_ha_perf_diagnose_history";
 const char *const OB_ALL_CLONE_JOB_TNAME = "__all_clone_job";
 const char *const OB_ALL_CLONE_JOB_HISTORY_TNAME = "__all_clone_job_history";
 const char *const OB_ALL_NCOMP_DLL_TNAME = "__all_ncomp_dll";
@@ -3332,6 +3342,8 @@ const char *const OB_ALL_VIRTUAL_DATA_ACTIVITY_METRICS_TNAME = "__all_virtual_da
 const char *const OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_TNAME = "__all_virtual_column_group_mapping";
 const char *const OB_ALL_VIRTUAL_COLUMN_GROUP_HISTORY_TNAME = "__all_virtual_column_group_history";
 const char *const OB_ALL_VIRTUAL_COLUMN_GROUP_MAPPING_HISTORY_TNAME = "__all_virtual_column_group_mapping_history";
+const char *const OB_ALL_VIRTUAL_STORAGE_HA_ERROR_DIAGNOSE_TNAME = "__all_virtual_storage_ha_error_diagnose";
+const char *const OB_ALL_VIRTUAL_STORAGE_HA_PERF_DIAGNOSE_TNAME = "__all_virtual_storage_ha_perf_diagnose";
 const char *const OB_ALL_VIRTUAL_CLONE_JOB_TNAME = "__all_virtual_clone_job";
 const char *const OB_ALL_VIRTUAL_CLONE_JOB_HISTORY_TNAME = "__all_virtual_clone_job_history";
 const char *const OB_ALL_VIRTUAL_CHECKPOINT_DIAGNOSE_MEMTABLE_INFO_TNAME = "__all_virtual_checkpoint_diagnose_memtable_info";
@@ -4737,6 +4749,8 @@ const char *const OB_ALL_IMPORT_TABLE_JOB_AUX_LOB_META_TNAME = "__all_import_tab
 const char *const OB_ALL_IMPORT_TABLE_JOB_HISTORY_AUX_LOB_META_TNAME = "__all_import_table_job_history_aux_lob_meta";
 const char *const OB_ALL_IMPORT_TABLE_TASK_AUX_LOB_META_TNAME = "__all_import_table_task_aux_lob_meta";
 const char *const OB_ALL_IMPORT_TABLE_TASK_HISTORY_AUX_LOB_META_TNAME = "__all_import_table_task_history_aux_lob_meta";
+const char *const OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_META_TNAME = "__all_storage_ha_error_diagnose_history_aux_lob_meta";
+const char *const OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_META_TNAME = "__all_storage_ha_perf_diagnose_history_aux_lob_meta";
 const char *const OB_ALL_CLONE_JOB_AUX_LOB_META_TNAME = "__all_clone_job_aux_lob_meta";
 const char *const OB_ALL_CLONE_JOB_HISTORY_AUX_LOB_META_TNAME = "__all_clone_job_history_aux_lob_meta";
 const char *const OB_ALL_NCOMP_DLL_AUX_LOB_META_TNAME = "__all_ncomp_dll_aux_lob_meta";
@@ -5022,6 +5036,8 @@ const char *const OB_ALL_IMPORT_TABLE_JOB_AUX_LOB_PIECE_TNAME = "__all_import_ta
 const char *const OB_ALL_IMPORT_TABLE_JOB_HISTORY_AUX_LOB_PIECE_TNAME = "__all_import_table_job_history_aux_lob_piece";
 const char *const OB_ALL_IMPORT_TABLE_TASK_AUX_LOB_PIECE_TNAME = "__all_import_table_task_aux_lob_piece";
 const char *const OB_ALL_IMPORT_TABLE_TASK_HISTORY_AUX_LOB_PIECE_TNAME = "__all_import_table_task_history_aux_lob_piece";
+const char *const OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TNAME = "__all_storage_ha_error_diagnose_history_aux_lob_piece";
+const char *const OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_AUX_LOB_PIECE_TNAME = "__all_storage_ha_perf_diagnose_history_aux_lob_piece";
 const char *const OB_ALL_CLONE_JOB_AUX_LOB_PIECE_TNAME = "__all_clone_job_aux_lob_piece";
 const char *const OB_ALL_CLONE_JOB_HISTORY_AUX_LOB_PIECE_TNAME = "__all_clone_job_history_aux_lob_piece";
 const char *const OB_ALL_NCOMP_DLL_AUX_LOB_PIECE_TNAME = "__all_ncomp_dll_aux_lob_piece";

src/share/inner_table/ob_inner_table_schema_def.py

@@ -6725,8 +6725,60 @@ def_table_schema(
 
 # 481 : __all_import_stmt_exec_history
 # 482 : __all_tablet_reorganize_history
-# 483 : __all_storage_ha_error_diagnose_history
-# 484 : __all_storage_ha_perf_diagnose_history
+def_table_schema(
+  owner = 'zhixing.yh',
+  table_name    = '__all_storage_ha_error_diagnose_history',
+  table_id      = '483',
+  table_type = 'SYSTEM_TABLE',
+  in_tenant_space = False,
+  gm_columns    = [],
+  rowkey_columns = [
+    ('gmt_create', 'timestamp', 'false'),
+    ('svr_ip', 'varchar:MAX_IP_ADDR_LENGTH'),
+    ('svr_port', 'int')
+  ],
+  normal_columns = [
+    ('tenant_id', 'int'),
+    ('ls_id', 'int'),
+    ('module', 'varchar:OB_MODULE_NAME_LENGTH'),
+    ('type', 'varchar:OB_SYS_TASK_TYPE_LENGTH'),
+    ('task_id', 'varchar:OB_MAX_TRACE_ID_BUFFER_SIZE'),
+    ('retry_id', 'int'),
+    ('create_time', 'timestamp'),
+    ('result_code', 'int'),
+    ('result_msg', 'varchar:OB_DIAGNOSE_INFO_LENGTH'),
+    ('info', 'longtext')
+  ],
+)
+
+def_table_schema(
+  owner = 'zhixing.yh',
+  table_name    = '__all_storage_ha_perf_diagnose_history',
+  table_id      = '484',
+  table_type = 'SYSTEM_TABLE',
+  in_tenant_space = False,
+  gm_columns    = [],
+  rowkey_columns = [
+    ('gmt_create', 'timestamp', 'false'),
+    ('svr_ip', 'varchar:MAX_IP_ADDR_LENGTH'),
+    ('svr_port', 'int')
+  ],
+  normal_columns = [
+    ('tenant_id', 'int'),
+    ('ls_id', 'int'),
+    ('module', 'varchar:OB_MODULE_NAME_LENGTH'),
+    ('type', 'varchar:OB_SYS_TASK_TYPE_LENGTH'),
+    ('task_id', 'varchar:OB_MAX_TRACE_ID_BUFFER_SIZE'),
+    ('retry_id', 'int'),
+    ('start_timestamp', 'timestamp'),
+    ('end_timestamp', 'timestamp'),
+    ('tablet_id', 'int'),
+    ('tablet_count', 'int'),
+    ('result_code', 'int'),
+    ('result_msg', 'varchar:OB_DIAGNOSE_INFO_LENGTH'),
+    ('info', 'longtext'),
+  ],
+)
 
 def_table_schema(
   owner = 'chensen.cs',
@@ -13737,8 +13789,60 @@ def_table_schema(**gen_iterate_virtual_table_def(
     keywords = all_def_keywords['__all_column_group_mapping_history'],
     in_tenant_space = True))
 
-# 12433: __all_virtual_storage_ha_error_diagnose
-# 12434: __all_virtual_storage_ha_perf_diagnose
+def_table_schema(
+  owner = 'zhixing.yh',
+  table_name    = '__all_virtual_storage_ha_error_diagnose',
+  table_id      = '12433',
+  table_type = 'VIRTUAL_TABLE',
+  in_tenant_space = False,
+  gm_columns    = [],
+  rowkey_columns = [],
+  normal_columns = [
+    ('tenant_id', 'int'),
+    ('ls_id', 'int'),
+    ('module', 'varchar:OB_MODULE_NAME_LENGTH'),
+    ('type', 'varchar:OB_SYS_TASK_TYPE_LENGTH'),
+    ('task_id', 'varchar:OB_MAX_TRACE_ID_BUFFER_SIZE'),
+    ('svr_ip', 'varchar:MAX_IP_ADDR_LENGTH'),
+    ('svr_port', 'int'),
+    ('retry_id', 'int'),
+    ('create_time', 'timestamp'),
+    ('result_code', 'int'),
+    ('result_msg', 'varchar:OB_DIAGNOSE_INFO_LENGTH'),
+    ('info', 'longtext')
+  ],
+  partition_columns = ['svr_ip', 'svr_port'],
+  vtable_route_policy = 'distributed',
+)
+
+def_table_schema(
+  owner = 'zhixing.yh',
+  table_name    = '__all_virtual_storage_ha_perf_diagnose',
+  table_id      = '12434',
+  table_type = 'VIRTUAL_TABLE',
+  in_tenant_space = False,
+  gm_columns    = [],
+  rowkey_columns = [],
+  normal_columns = [
+    ('tenant_id', 'int'),
+    ('ls_id', 'int'),
+    ('module', 'varchar:OB_MODULE_NAME_LENGTH'),
+    ('type', 'varchar:OB_SYS_TASK_TYPE_LENGTH'),
+    ('task_id', 'varchar:OB_MAX_TRACE_ID_BUFFER_SIZE'),
+    ('svr_ip', 'varchar:MAX_IP_ADDR_LENGTH'),
+    ('svr_port', 'int'),
+    ('retry_id', 'int'),
+    ('start_timestamp', 'timestamp'),
+    ('end_timestamp', 'timestamp'),
+    ('tablet_id', 'int'),
+    ('tablet_count', 'int'),
+    ('result_code', 'int'),
+    ('result_msg', 'varchar:OB_DIAGNOSE_INFO_LENGTH'),
+    ('info', 'longtext'),
+  ],
+  partition_columns = ['svr_ip', 'svr_port'],
+  vtable_route_policy = 'distributed',
+)
 
 def_table_schema(**gen_iterate_private_virtual_table_def(
   table_id = '12435',

src/share/inner_table/table_id_to_name

@@ -329,6 +329,8 @@
 # 478: __all_import_table_job_history
 # 479: __all_import_table_task
 # 480: __all_import_table_task_history
+# 483: __all_storage_ha_error_diagnose_history
+# 484: __all_storage_ha_perf_diagnose_history
 # 485: __all_clone_job
 # 486: __all_clone_job_history
 # 493: __all_ncomp_dll
@@ -1049,6 +1051,8 @@
 # 12432: __all_virtual_column_group_mapping_history
 # 12432: __all_column_group_mapping  # BASE_TABLE_NAME
 # 12432: __all_column_group_mapping_history  # BASE_TABLE_NAME1
+# 12433: __all_virtual_storage_ha_error_diagnose
+# 12434: __all_virtual_storage_ha_perf_diagnose
 # 12435: __all_virtual_clone_job
 # 12435: __all_clone_job  # BASE_TABLE_NAME
 # 12436: __all_virtual_clone_job_history

src/share/ob_storage_ha_diagnose_operator.cpp

@@ -0,0 +1,369 @@
+/**
+ * Copyright (c) 2022 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX SHARE
+
+#include "lib/mysqlclient/ob_mysql_proxy.h" // ObISqlClient, SMART_VAR
+#include "share/inner_table/ob_inner_table_schema.h"
+#include "share/ob_storage_ha_diagnose_operator.h"
+#include "share/ob_dml_sql_splicer.h" // ObDMLSqlSplicer
+#include "share/config/ob_server_config.h"
+
+namespace oceanbase
+{
+
+namespace share
+{
+
+ObStorageHADiagOperator::ObStorageHADiagOperator()
+  : is_inited_(false),
+    last_event_ts_(0) {}
+
+int ObStorageHADiagOperator::init()
+{
+  int ret = OB_SUCCESS;
+  if (is_inited_) {
+    ret = OB_INIT_TWICE;
+    LOG_WARN("ObStorageHADiagOperator is already init", K(ret));
+  } else {
+    last_event_ts_ = 0;
+    is_inited_ = true;
+  }
+  return ret;
+}
+
+void ObStorageHADiagOperator::reset()
+{
+  is_inited_ = false;
+  last_event_ts_ = 0;
+}
+
+int ObStorageHADiagOperator::get_batch_row_keys(
+        common::ObISQLClient &sql_proxy,
+        const uint64_t tenant_id,
+        const ObStorageHADiagModule &module,
+        const int64_t last_end_timestamp,
+        ObIArray<int64_t> &timestamp_array) const
+{
+  int ret = OB_SUCCESS;
+  ObSqlString sql;
+  const char* table_name = nullptr;
+  timestamp_array.reset();
+  if (!is_inited_) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not init", K(ret));
+  } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || ObStorageHADiagModule::MAX_MODULE == module)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(module));
+  } else if (OB_FAIL(get_table_name_(module, table_name))) {
+    LOG_WARN("fail to get table name", KR(ret), K(module));
+  } else {
+    SMART_VAR(ObMySQLProxy::MySQLResult, res) {
+      sqlclient::ObMySQLResult *result = nullptr;
+      if (OB_FAIL(sql.assign_fmt(
+          "select time_to_usec(gmt_create) as create_ts from %s where gmt_create > usec_to_time(%ld) order by gmt_create limit 100",
+          table_name, last_end_timestamp))) {
+        LOG_WARN("assign_fmt failed", K(ret), K(last_end_timestamp));
+      } else if (OB_FAIL(sql_proxy.read(res, sql.ptr()))) {
+        LOG_WARN("fail to execute sql", K(sql), K(ret), K(tenant_id));
+      } else if (OB_ISNULL(result = res.get_result())) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_WARN("result is NULL", K(sql), K(ret));
+      } else {
+        while (OB_SUCC(ret)) {
+          if (OB_FAIL(result->next())) {
+            if (OB_ITER_END == ret) {
+              ret = OB_SUCCESS;
+              break;
+            } else {
+              LOG_WARN("fail to get next result", K(sql), K(ret));
+            }
+          } else if (OB_ISNULL(result)) {
+            ret = OB_ERR_UNEXPECTED;
+            LOG_WARN("result is NULL", K(sql),K(ret));
+          } else {
+            int64_t timestamp = 0;
+            EXTRACT_INT_FIELD_MYSQL(*result, "create_ts", timestamp, int64_t);
+            if (OB_FAIL(ret)) {
+              LOG_WARN("extract mysql failed", K(ret), K(sql));
+            } else if (OB_FAIL(timestamp_array.push_back(timestamp))) {
+              LOG_WARN("failed to push to array", K(ret), K(timestamp_array), K(timestamp));
+            }
+          }
+        }
+      }
+    }
+  }
+  return ret;
+}
+
+int ObStorageHADiagOperator::do_batch_delete(
+        common::ObISQLClient &sql_proxy,
+        const uint64_t tenant_id,
+        const ObStorageHADiagModule &module,
+        const ObIArray<int64_t> &timestamp_array,
+        const int64_t delete_timestamp,
+        int64_t &delete_index) const
+{
+  int ret = OB_SUCCESS;
+  const char* table_name = nullptr;
+  ObSqlString sql;
+  int64_t affected_rows = 0;
+  delete_index = 0;
+  if (!is_inited_) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not init", K(ret));
+  } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || ObStorageHADiagModule::MAX_MODULE == module)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(module));
+  } else if (OB_FAIL(get_table_name_(module, table_name))) {
+    LOG_WARN("fail to get table name", KR(ret), K(module));
+  } else if (OB_FAIL(sql.assign_fmt("delete from %s where ",
+                              table_name))) {
+    LOG_WARN("assign_fmt failed", K(ret));
+  } else {
+    for (int64_t i = 0; OB_SUCC(ret) && i < timestamp_array.count(); i++) {
+      int64_t timestamp = timestamp_array.at(i);
+      if (timestamp <= delete_timestamp) {
+        if (i > 0 && OB_FAIL(sql.append_fmt(" or "))) {
+          LOG_WARN("append_fmt failed", K(ret), K(i));
+        } else if (OB_FAIL(sql.append_fmt("gmt_create = usec_to_time(%ld)", timestamp))) {
+          LOG_WARN("append_fmt failed", K(ret), K(i), K(timestamp));
+        }
+      } else if (0 == i) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_WARN("unexpected timestamp", K(ret), K(i), K(timestamp), K(delete_timestamp));
+      } else {
+        delete_index = i - 1;
+        break;
+      }
+    }
+    if (OB_FAIL(ret)) {
+      //do nothing
+      LOG_WARN("fail to splice sql", K(ret), K(sql));
+    } else if (OB_FAIL(sql_proxy.write(tenant_id, sql.ptr(), affected_rows))) {
+      LOG_WARN("fail to execute sql", K(sql), K(ret), K(tenant_id));
+    }
+  }
+  return ret;
+}
+
+//TODO(zhixing.yh) batch insert row
+int ObStorageHADiagOperator::insert_row(
+    common::ObISQLClient &sql_proxy,
+    const uint64_t tenant_id,
+    const uint64_t report_tenant_id,
+    const ObStorageHADiagInfo &info,
+    const ObStorageHADiagModule &module)
+{
+  int ret = OB_SUCCESS;
+  if (!is_inited_) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not init", K(ret));
+  } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id)
+      || !info.is_valid()
+      || ObStorageHADiagModule::MAX_MODULE == module)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(info));
+  } else {
+    ObSqlString sql;
+    ObDMLSqlSplicer dml_splicer;
+    int64_t affected_rows = 0;
+    const char* table_name = nullptr;
+    if (OB_FAIL(fill_dml_(tenant_id, report_tenant_id, info, dml_splicer))) {
+      LOG_WARN("fail to fill dml splicer", KR(ret), K(tenant_id), K(info));
+    } else if (OB_FAIL(get_table_name_(module, table_name))) {
+      LOG_WARN("fail to get table name", KR(ret), K(tenant_id), K(info));
+    } else if (OB_FAIL(dml_splicer.splice_insert_sql(table_name, sql))) {
+      LOG_WARN("fail to splice insert sql", KR(ret), K(module));
+    } else if (OB_FAIL(sql_proxy.write(tenant_id, sql.ptr(), affected_rows))) {
+      LOG_WARN("fail to execute sql", K(sql), K(ret), K(tenant_id));
+    } else {
+      LOG_DEBUG("insert storage ha diagnose info history success",
+          K(tenant_id), K(affected_rows), K(info));
+    }
+  }
+  return ret;
+}
+
+int ObStorageHADiagOperator::get_table_name_(const ObStorageHADiagModule &module, const char *&table_name) const
+{
+  int ret = OB_SUCCESS;
+  switch(module) {
+    case ObStorageHADiagModule::TRANSFER_ERROR_DIAGNOSE:
+      table_name = OB_ALL_STORAGE_HA_ERROR_DIAGNOSE_HISTORY_TNAME;
+      break;
+    case ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE:
+      table_name = OB_ALL_STORAGE_HA_PERF_DIAGNOSE_HISTORY_TNAME;
+      break;
+    default:
+      ret = OB_INVALID_ARGUMENT;
+      LOG_WARN("invalid module", K(ret), K(module));
+      break;
+  }
+  return ret;
+}
+
+int ObStorageHADiagOperator::gen_event_ts_(int64_t &event_ts)
+{
+  int ret = OB_SUCCESS;
+  event_ts = 0;
+  if (!is_inited_) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not init", K(ret));
+  } else {
+    const int64_t now = ObTimeUtility::current_time();
+    event_ts = (last_event_ts_ >= now ? last_event_ts_ + 1 : now);
+    last_event_ts_ = event_ts;
+  }
+  return ret;
+}
+
+int ObStorageHADiagOperator::fill_dml_(
+    const uint64_t tenant_id,
+    const uint64_t report_tenant_id,
+    const ObStorageHADiagInfo &input_info,
+    ObDMLSqlSplicer &dml)
+{
+  int ret = OB_SUCCESS;
+  char diagnose_info[common::OB_DIAGNOSE_INFO_LENGTH] = { 0 };
+  char task_id[common::OB_MAX_TRACE_ID_BUFFER_SIZE] = { 0 };
+  char ip_buf[common::OB_IP_STR_BUFF] = { 0 };
+  int64_t timestamp = 0;
+  if (OB_FAIL(gen_event_ts_(timestamp))) {
+    LOG_WARN("failed to gen event ts", K(ret), K(timestamp), K(last_event_ts_));
+  } else if (OB_FAIL(dml.add_gmt_create(timestamp))) {
+    LOG_WARN("failed to add gmt_create", K(ret), K(input_info));
+  } else if (!ObServerConfig::get_instance().self_addr_.ip_to_string(ip_buf, sizeof(ip_buf))) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("failed to set svr_ip", K(ret));
+  } else if (OB_FAIL(dml.add_column("svr_ip", ip_buf))) {
+    LOG_WARN("failed to add svr_ip", K(ret));
+  } else if (OB_FAIL(dml.add_column("svr_port", ObServerConfig::get_instance().self_addr_.get_port()))) {
+    LOG_WARN("failed to add svr_port", K(ret));
+  } else if (OB_FAIL(dml.add_column("tenant_id", report_tenant_id))) {
+    LOG_WARN("failed to add tenant id", K(ret), K(report_tenant_id));
+  } else if (OB_FAIL(dml.add_column("ls_id", input_info.ls_id_.id()))) {
+    LOG_WARN("failed to add ls id", K(ret), K(input_info));
+  } else if (OB_FAIL(dml.add_column("module", input_info.get_module_str()))) {
+    LOG_WARN("failed to add module", K(ret), K(input_info));
+  } else if (OB_FAIL(dml.add_column("type", input_info.get_type_str()))) {
+    LOG_WARN("failed to add type", K(ret), K(input_info));
+  } else if (OB_FAIL(input_info.get_task_id(task_id, sizeof(task_id)))) {
+    LOG_WARN("failed to get task id", K(ret), K(input_info));
+  } else if (OB_FAIL(dml.add_column("task_id", task_id))) {
+    LOG_WARN("failed to add task id", K(ret), K(input_info));
+  } else if (OB_FAIL(dml.add_column("retry_id", input_info.retry_id_))) {
+    LOG_WARN("failed to add retry id", K(ret), K(input_info));
+  } else if (OB_FAIL(dml.add_column("result_code", input_info.result_code_))) {
+    LOG_WARN("failed to add result code", K(ret), K(input_info));
+  } else if (OB_FAIL(dml.add_column("result_msg", input_info.get_transfer_error_diagnose_msg()))) {
+    LOG_WARN("failed to add result msg", K(ret), K(input_info));
+  } else if (OB_FAIL(input_info.get_info(diagnose_info, sizeof(diagnose_info)))) {
+    LOG_WARN("failed to get diagnose info", K(ret), K(input_info));
+  } else if (OB_FAIL(dml.add_column("info", diagnose_info))) {
+    LOG_WARN("failed to add info", K(ret), K(diagnose_info));
+  } else {
+    switch (input_info.module_) {
+      case ObStorageHADiagModule::TRANSFER_ERROR_DIAGNOSE: {
+        if (OB_FAIL(dml.add_time_column("create_time", input_info.timestamp_))) {
+          LOG_WARN("failed to add create_time", K(ret), K(input_info));
+        }
+        break;
+      }
+      case ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE: {
+        const ObTransferPerfDiagInfo &transfer_perf_info = static_cast<const ObTransferPerfDiagInfo&>(input_info);
+        if (OB_FAIL(dml.add_time_column("start_timestamp", input_info.timestamp_))) {
+          LOG_WARN("failed to add start timestamp", K(ret), K(transfer_perf_info));
+        } else if (OB_FAIL(dml.add_time_column("end_timestamp", transfer_perf_info.end_timestamp_))) {
+          LOG_WARN("failed to add end timestamp", K(ret), K(transfer_perf_info));
+        } else if (OB_FAIL(dml.add_column("tablet_id", transfer_perf_info.tablet_id_.id()))) {
+          LOG_WARN("failed to add tablet id", K(ret), K(transfer_perf_info));
+        } else if (OB_FAIL(dml.add_column("tablet_count", transfer_perf_info.tablet_count_))) {
+          LOG_WARN("failed to add tablet count", K(ret), K(transfer_perf_info));
+        }
+        break;
+      }
+      default: {
+        ret = OB_INVALID_ARGUMENT;
+        LOG_WARN("unknow module", K(ret), K(input_info.module_));
+        break;
+      }
+    }
+  }
+  return ret;
+}
+
+int ObStorageHADiagOperator::parse_result_(
+    common::sqlclient::ObMySQLResult &result,
+    int64_t &result_count) const
+{
+  int ret = OB_SUCCESS;
+  result_count = 0;
+  EXTRACT_INT_FIELD_MYSQL(result, "count", result_count, int64_t);
+  if (OB_FAIL(ret)) {
+    LOG_WARN("extract mysql failed", KR(ret));
+  }
+  return ret;
+}
+
+int ObStorageHADiagOperator::check_transfer_task_exist(
+    common::ObISQLClient &sql_proxy,
+    const uint64_t tenant_id,
+    const share::ObTransferTaskID &task_id,
+    int64_t &result_count) const
+{
+  int ret = OB_SUCCESS;
+  result_count = 0;
+  if (!is_valid_tenant_id(tenant_id) || !task_id.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(task_id));
+  } else {
+    ObSqlString sql;
+    SMART_VAR(ObISQLClient::ReadResult, res) {
+      sqlclient::ObMySQLResult *result = nullptr;
+      if (OB_FAIL(sql.assign_fmt(
+          "SELECT COUNT(*) as count FROM %s WHERE task_id = %ld",
+          OB_ALL_TRANSFER_TASK_TNAME, task_id.id()))) {
+        LOG_WARN("fail to assign sql", KR(ret), K(task_id));
+      } else if (OB_FAIL(sql_proxy.read(res, tenant_id, sql.ptr()))) {
+        LOG_WARN("execute sql failed", KR(ret), K(tenant_id), K(sql));
+      } else if (OB_ISNULL(result = res.get_result())) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_WARN("get mysql result failed", KR(ret), K(tenant_id), K(sql));
+      } else {
+        if (OB_FAIL(result->next())) {
+          if (OB_ITER_END == ret) {
+            ret = OB_ENTRY_NOT_EXIST;
+          } else {
+            LOG_WARN("fail to get next result", KR(ret), K(sql));
+          }
+        } else if (OB_FAIL(parse_result_(*result, result_count))) {
+          LOG_WARN("failed to parse result", KR(ret), K(result_count));
+        } else if (OB_FAIL(result->next())) {
+          if (OB_ITER_END == ret) {
+            ret = OB_SUCCESS;
+          } else {
+            LOG_WARN("get next result failed", KR(ret), K(result_count));
+          }
+        } else {
+          ret = OB_ERR_UNEXPECTED;
+          LOG_WARN("read more than one row", KR(ret), K(result_count));
+        }
+      }
+    }
+  }
+  return ret;
+}
+
+} // end namespace share
+} // end namespace oceanbase

src/share/ob_storage_ha_diagnose_operator.h

@@ -0,0 +1,91 @@
+/**
+ * Copyright (c) 2022 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_SHARE_HA_DIAGNOSE_OPERATOR_
+#define OCEANBASE_SHARE_HA_DIAGNOSE_OPERATOR_
+
+#include "share/ob_storage_ha_diagnose_struct.h"
+
+namespace oceanbase
+{
+namespace common
+{
+class ObISQLClient;
+
+namespace sqlclient
+{
+class ObMySQLResult;
+}
+}
+
+namespace share
+{
+class ObDMLSqlSplicer;
+
+class ObStorageHADiagOperator final
+{
+public:
+  ObStorageHADiagOperator();
+  ~ObStorageHADiagOperator() {}
+
+  int init();
+
+  void reset();
+  int get_batch_row_keys(
+          common::ObISQLClient &sql_proxy,
+          const uint64_t tenant_id,
+          const ObStorageHADiagModule &module,
+          const int64_t last_end_timestamp,
+          ObIArray<int64_t> &timestamp_array) const;
+
+  int insert_row(
+          common::ObISQLClient &sql_proxy,
+          const uint64_t tenant_id,
+          const uint64_t report_tenant_id,
+          const ObStorageHADiagInfo &info_row,
+          const ObStorageHADiagModule &module);
+  int do_batch_delete(
+          common::ObISQLClient &sql_proxy,
+          const uint64_t tenant_id,
+          const ObStorageHADiagModule &module,
+          const ObIArray<int64_t> &timestamp_array,
+          const int64_t delete_timestamp,
+          int64_t &delete_index) const;
+  int check_transfer_task_exist(
+          common::ObISQLClient &sql_proxy,
+          const uint64_t tenant_id,
+          const share::ObTransferTaskID &task_id,
+          int64_t &result_count) const;
+
+private:
+  int fill_dml_(
+          const uint64_t tenant_id,
+          const uint64_t report_tenant_id,
+          const ObStorageHADiagInfo &input_info,
+          ObDMLSqlSplicer &dml);
+
+  int get_table_name_(const ObStorageHADiagModule &module, const char *&table_name) const;
+  int gen_event_ts_(int64_t &event_ts);
+  int parse_result_(
+    common::sqlclient::ObMySQLResult &result,
+    int64_t &result_count) const;
+
+private:
+  bool is_inited_;
+  int64_t last_event_ts_;
+
+  DISALLOW_COPY_AND_ASSIGN(ObStorageHADiagOperator);
+};
+
+} // end namespace share
+} // end namespace oceanbase
+#endif

src/share/ob_storage_ha_diagnose_struct.h

@@ -0,0 +1,395 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEABASE_SHARE_HA_MONITOR_STRUCT_
+#define OCEABASE_SHARE_HA_MONITOR_STRUCT_
+
+#include "share/transfer/ob_transfer_info.h"
+#include "share/ob_ls_id.h"
+
+namespace oceanbase
+{
+namespace share
+{
+enum class ObStorageHADiagType
+{
+  ERROR_DIAGNOSE = 0,
+  PERF_DIAGNOSE = 1,
+  MAX_TYPE
+};
+
+enum class ObStorageHADiagModule
+{
+  TRANSFER_ERROR_DIAGNOSE = 0,
+  TRANSFER_PERF_DIAGNOSE = 1,
+  MAX_MODULE
+};
+
+enum class ObStorageHADiagTaskType
+{
+  TRANSFER_START = 0,
+  TRANSFER_DOING = 1,
+  TRANSFER_ABORT = 2,
+  TRANSFER_START_IN = 3,
+  TRANSFER_START_OUT = 4,
+  TRANSFER_FINISH_IN = 5,
+  TRANSFER_FINISH_OUT = 6,
+  TRANSFER_BACKFILLED = 7,
+  MAX_TYPE
+};
+
+enum class ObStorageHACostItemName : int
+{
+  // start stage
+  TRANSFER_START_BEGIN = 0,
+  LOCK_MEMBER_LIST = 1,
+  PRECHECK_LS_REPALY_SCN = 2,
+  CHECK_START_STATUS_TRANSFER_TABLETS = 3,
+  CHECK_SRC_LS_HAS_ACTIVE_TRANS = 4,
+  UPDATE_ALL_TABLET_TO_LS = 5,
+  LOCK_TABLET_ON_DEST_LS_FOR_TABLE_LOCK = 6,
+  BLOCK_AND_KILL_TX = 7,
+  REGISTER_TRANSFER_START_OUT = 8,
+  DEST_LS_GET_START_SCN = 9,
+  CHECK_SRC_LS_HAS_ACTIVE_TRANS_LATER = 10,
+  UNBLOCK_TX = 11,
+  WAIT_SRC_LS_REPLAY_TO_START_SCN = 12,
+  SRC_LS_GET_TABLET_META = 13,
+  REGISTER_TRANSFER_START_IN = 14,
+  START_TRANS_COMMIT = 15,
+  TRANSFER_START_END = 16,
+  // doing stage
+  TRANSFER_FINISH_BEGIN = 17,
+  UNLOCK_SRC_AND_DEST_LS_MEMBER_LIST = 18,
+  CHECK_LS_LOGICAL_TABLE_REPLACED = 19,
+  LOCK_LS_MEMBER_LIST_IN_DOING = 20,
+  CHECK_LS_LOGICAL_TABLE_REPLACED_LATER = 21,
+  REGISTER_TRANSFER_FINISH_IN = 22,
+  WAIT_TRANSFER_TABLET_STATUS_NORMAL = 23,
+  WAIT_ALL_LS_REPLICA_REPLAY_FINISH_SCN = 24,
+  REGISTER_TRANSFER_OUT = 25,
+  UNLOCK_TABLET_FOR_LOCK = 26,
+  UNLOCK_LS_MEMBER_LIST = 27,
+  FINISH_TRANS_COMMIT = 28,
+  TRANSFER_FINISH_END = 29,
+  // abort stage
+  TRANSFER_ABORT_BEGIN = 30,
+  UNLOCK_MEMBER_LIST_IN_ABORT = 31,
+  ABORT_TRANS_COMMIT = 32,
+  TRANSFER_ABORT_END = 33,
+  // backfill stage
+  TRANSFER_BACKFILLED_TABLE_BEGIN = 34,
+  TX_BACKFILL = 35,
+  TABLET_REPLACE = 36,
+  REPLACE_TIMESTAMP = 37,
+  TRANSFER_BACKFILLED_END = 38,
+  // start stage log replay
+  START_TRANSFER_OUT_FIRST_REPALY_LOG_TIMESTAMP = 39,
+  START_TRANSFER_OUT_LOG_SCN = 40,
+  START_TRANSFER_OUT_TX_ID = 41,
+  START_TRANSFER_OUT_REPLAY_COST = 42,
+  START_TRANSFER_IN_FIRST_REPALY_LOG_TIMESTAMP = 43,
+  START_TRANSFER_IN_LOG_SCN = 44,
+  START_TRANSFER_IN_TX_ID = 45,
+  START_TRANSFER_IN_REPLAY_COST = 46,
+  START_CHECK_LS_EXIST_TS = 47,
+  START_CHECK_LS_REPLAY_POINT_TS = 48,
+  // doing stage log replay
+  FINISH_TRANSFER_OUT_FIRST_REPALY_LOG_TIMESTAMP = 49,
+  FINISH_TRANSFER_OUT_LOG_SCN = 50,
+  FINISH_TRANSFER_OUT_TX_ID = 51,
+  FINISH_TRANSFER_OUT_REPLAY_COST = 52,
+  FINISH_TRANSFER_IN_FIRST_REPALY_LOG_TIMESTAMP = 53,
+  FINISH_TRANSFER_IN_LOG_SCN = 54,
+  FINISH_TRANSFER_IN_TX_ID = 55,
+  FINISH_TRANSFER_IN_REPLAY_COST = 56,
+  // start stage supplement
+  UNLOCK_MEMBER_LIST_IN_START = 57,
+  // reply stage supplement
+  ON_REGISTER_SUCCESS = 58,
+  ON_REPLAY_SUCCESS = 59,
+  // backfilled stage supplement
+  PREPARE_MERGE_CTX = 60,
+  PREPARE_INDEX_TREE = 61,
+  DO_BACKFILL_TX = 62,
+  TRANSFER_REPLACE_BEGIN = 63,
+  TRANSFER_REPLACE_END = 64,
+  TRANSFER_BACKFILL_START = 65,
+  // start
+  STOP_LS_SCHEDULE_MEMDIUM = 66,
+  MAX_NAME
+};
+
+enum class ObStorageHACostItemType
+{
+  ACCUM_COST_TYPE = 0,
+  FLUENT_TIMESTAMP_TYPE = 1,
+  CRUCIAL_TIMESTAMP = 2,
+  MAX_TYPE
+};
+
+struct ObStorageHAPerfDiagParams final
+{
+public:
+  ObStorageHAPerfDiagParams();
+  ~ObStorageHAPerfDiagParams();
+  void reset();
+  bool is_valid() const;
+  int assign(const ObStorageHAPerfDiagParams &params);
+
+  TO_STRING_KV(K_(dest_ls_id), K_(task_id), K_(task_type),
+      K_(item_type), K_(name), K_(tablet_id), K_(tablet_count));
+
+  ObLSID dest_ls_id_;
+  share::ObTransferTaskID task_id_;
+  ObStorageHADiagTaskType task_type_;
+  ObStorageHACostItemType item_type_;
+  ObStorageHACostItemName name_;
+  common::ObTabletID tablet_id_;
+  int64_t tablet_count_;
+  DISALLOW_COPY_AND_ASSIGN(ObStorageHAPerfDiagParams);
+};
+
+struct ObStorageHADiagTaskKey final
+{
+public:
+  ObStorageHADiagTaskKey();
+  ~ObStorageHADiagTaskKey() { reset(); }
+  void reset();
+  bool is_valid() const;
+  uint64_t hash() const;
+  int hash(uint64_t &hash_val) const;
+  bool operator==(const ObStorageHADiagTaskKey &key) const;
+  bool operator<(const ObStorageHADiagTaskKey &key) const;
+  int assign(const ObStorageHADiagTaskKey &key);
+
+  TO_STRING_KV(
+    K_(tenant_id),
+    K_(task_id),
+    K_(module),
+    K_(type),
+    K_(retry_id),
+    K_(diag_type),
+    K_(tablet_id));
+
+  uint64_t tenant_id_;
+  int64_t task_id_;
+  ObStorageHADiagModule module_;
+  ObStorageHADiagTaskType type_;
+  int64_t retry_id_;
+  ObStorageHADiagType diag_type_;
+  common::ObTabletID tablet_id_;
+};
+
+struct ObStorageHADiagTask;
+struct ObStorageHADiagInfo
+{
+public:
+  ObStorageHADiagInfo();
+  virtual ~ObStorageHADiagInfo() { reset(); }
+  virtual int get_info(char *info, const int64_t size) const = 0;
+  virtual int update(const ObStorageHADiagInfo &info) = 0;
+  virtual int assign(const ObStorageHADiagInfo &other);
+  virtual int deep_copy(ObIAllocator &allocator, ObStorageHADiagInfo *&out_info) const = 0;
+  virtual int get_task_id(char *info, const int64_t size) const = 0;
+  const char * get_module_str() const;
+  const char * get_type_str() const;
+  const char * get_transfer_error_diagnose_msg() const;
+  void reset();
+  bool is_valid() const;
+
+  const static char *ObStorageDiagModuleStr[static_cast<int64_t>(ObStorageHADiagModule::MAX_MODULE)];
+  const static char *ObStorageDiagTaskTypeStr[static_cast<int64_t>(ObStorageHADiagTaskType::MAX_TYPE)];
+  const static char *ObTransferErrorDiagMsg[static_cast<int>(share::ObStorageHACostItemName::MAX_NAME)];
+
+  TO_STRING_KV(
+    K_(task_id),
+    K_(ls_id),
+    K_(module),
+    K_(type),
+    K_(retry_id),
+    K_(result_code),
+    K_(timestamp),
+    K_(result_msg),
+    K_(tablet_id));
+
+  int64_t task_id_;
+  ObLSID ls_id_;
+  ObStorageHADiagModule module_;
+  ObStorageHADiagTaskType type_;
+  int64_t retry_id_;
+  int64_t result_code_;
+  int64_t timestamp_;
+  ObStorageHACostItemName result_msg_;
+  ObStorageHADiagTask *task_; // TODO(zhixing.yh) exist risk，it will be deleted after key placed in info
+  ObTabletID tablet_id_;
+
+  DISALLOW_COPY_AND_ASSIGN(ObStorageHADiagInfo);
+};
+
+struct ObStorageHADiagTask : public common::ObDLinkBase<ObStorageHADiagTask>
+{
+public:
+  ObStorageHADiagTask();
+  virtual ~ObStorageHADiagTask();
+  void reset();
+  bool is_valid() const;
+  TO_STRING_KV(K_(key), KPC_(val));
+
+  ObStorageHADiagTaskKey key_;
+  ObStorageHADiagInfo *val_;
+};
+
+struct ObTransferErrorDiagInfo : public ObStorageHADiagInfo
+{
+public:
+  ObTransferErrorDiagInfo();
+  virtual ~ObTransferErrorDiagInfo() { reset(); }
+  virtual int get_info(char *info, const int64_t size) const override;
+  virtual int get_task_id(char *info, const int64_t size) const override;
+  virtual int update(const ObStorageHADiagInfo &info) override;
+  virtual int deep_copy(ObIAllocator &allocator, ObStorageHADiagInfo *&out_info) const override;
+  virtual int assign(const ObStorageHADiagInfo &another) override;
+  bool is_valid() const;
+  void reset();
+
+  INHERIT_TO_STRING_KV(
+    "ObStorageHADiagInfo",
+    ObStorageHADiagInfo,
+    K_(trace_id),
+    K_(thread_id));
+
+  common::ObCurTraceId::TraceId trace_id_;
+  int64_t thread_id_;
+};
+
+struct ObIStorageHACostItem : public common::ObDLinkBase<ObIStorageHACostItem>
+{
+public:
+  ObIStorageHACostItem();
+  virtual ~ObIStorageHACostItem() { reset(); }
+  virtual int update(const ObIStorageHACostItem &item) = 0;
+  virtual int get_str(char *str, const int64_t size, int64_t &pos) const = 0;
+  virtual int assign(const ObIStorageHACostItem &item);
+  virtual int deep_copy(ObIAllocator &allocator, ObIStorageHACostItem *&item) const = 0;
+  virtual uint32_t get_size() const;
+  bool is_valid() const;
+  void reset();
+  const char * get_item_name() const;
+  const char * get_item_type() const;
+
+  const static char *ObStorageHACostItemNameStr[static_cast<int64_t>(ObStorageHACostItemName::MAX_NAME)];
+  const static char *ObStorageHACostItemTypeStr[static_cast<int64_t>(ObStorageHACostItemType::MAX_TYPE)];
+
+  TO_STRING_KV(
+    K_(name),
+    K_(type),
+    K_(retry_id));
+
+public:
+  ObStorageHACostItemName name_;
+  ObStorageHACostItemType type_;
+  int64_t retry_id_;
+};
+
+struct ObStorageHACostAccumItem : public ObIStorageHACostItem
+{
+public:
+  ObStorageHACostAccumItem();
+  virtual ~ObStorageHACostAccumItem() { reset(); }
+  virtual int update(const ObIStorageHACostItem &item) override;
+  virtual int get_str(char *str, const int64_t size, int64_t &pos) const override;
+  virtual int deep_copy(ObIAllocator &allocator, ObIStorageHACostItem *&item) const override;
+  virtual uint32_t get_size() const override;
+  bool is_valid() const;
+  void reset();
+  virtual int assign(const ObIStorageHACostItem &item) override;
+
+  INHERIT_TO_STRING_KV(
+    "ObIStorageHACostItem",
+    ObIStorageHACostItem,
+    K_(cost_time));
+public:
+  uint64_t cost_time_;
+};
+
+struct ObStorageHATimestampItem : public ObIStorageHACostItem
+{
+public:
+  ObStorageHATimestampItem();
+  virtual ~ObStorageHATimestampItem() { reset(); }
+  virtual int update(const ObIStorageHACostItem &item) override;
+  virtual int get_str(char *str, const int64_t size, int64_t &pos) const override;
+  virtual int assign(const ObIStorageHACostItem &item) override;
+  virtual int deep_copy(ObIAllocator &allocator, ObIStorageHACostItem *&item) const override;
+  virtual uint32_t get_size() const override;
+  bool is_valid() const;
+  void reset();
+
+  INHERIT_TO_STRING_KV(
+    "ObIStorageHACostItem",
+    ObIStorageHACostItem,
+    K_(timestamp));
+public:
+  uint64_t timestamp_;
+};
+
+struct ObTransferPerfDiagInfo : public ObStorageHADiagInfo
+{
+public:
+  ObTransferPerfDiagInfo();
+  virtual ~ObTransferPerfDiagInfo();
+  typedef ObDList<ObIStorageHACostItem> ItemList;
+  typedef hash::ObHashMap<int, ObIStorageHACostItem *> ItemMap;
+  int init(ObIAllocator *allocator, const int64_t tenant_id);
+  virtual int update(const ObStorageHADiagInfo &info) override;
+  virtual int deep_copy(ObIAllocator &allocator, ObStorageHADiagInfo *&out_info) const override;
+  virtual int get_task_id(char *info, const int64_t size) const override;
+  virtual int get_info(char *info, const int64_t size) const override;
+  virtual int assign(const ObStorageHADiagInfo &info) override;
+  int copy_item_list(const ObTransferPerfDiagInfo &info);
+  int add_item(const ObIStorageHACostItem &item);
+  bool is_inited() const;
+  bool is_valid() const;
+  void reuse();
+  void clear();
+  void destroy();
+
+  INHERIT_TO_STRING_KV(
+    "ObStorageHADiagInfo",
+    ObStorageHADiagInfo,
+    K_(is_inited),
+    K_(end_timestamp),
+    K_(tablet_count),
+    K_(items));
+
+private:
+  int find_item_(const ObStorageHACostItemName &name, bool &found, ObIStorageHACostItem *&item);
+  int update_item_list_(const ObTransferPerfDiagInfo &info);
+  static constexpr int64_t PERF_DIAG_BUCKET_NUM = 100;
+  static constexpr int64_t ITEM_MAX_SIZE = (1 << 12); // 4K
+
+public:
+  uint64_t end_timestamp_;
+  int64_t tablet_count_;
+  ItemList items_;
+  ItemMap item_map_;
+  uint64_t item_size_;
+
+private:
+  bool is_inited_;
+  ObIAllocator *allocator_;
+};
+
+}
+}
+#endif

src/share/parameter/ob_parameter_seed.ipp

@@ -1836,6 +1836,13 @@ DEF_BOOL(_enable_range_extraction_for_not_in, OB_TENANT_PARAMETER, "True",
 // DEF_BOOL(_enable_new_query_range_extraction, OB_TENANT_PARAMETER, "True",
 //     "decide whether use new algorithm to extract query range.",
 //     ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
+DEF_TIME(_ha_diagnose_history_recycle_interval, OB_CLUSTER_PARAMETER, "7d", "[2m, 180d]",
+         "The recycle interval time of diagnostic history data. Range: [2m, 180d]",
+         ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
+ERRSIM_DEF_TIME(errsim_transfer_diagnose_server_wait_time, OB_CLUSTER_PARAMETER, "5m", "[1s,1h]",
+        "the wakeup wait time of the transfer diagnose server. "
+        "Range: [0s, 1h]",
+        ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
 ERRSIM_DEF_STR(palf_inject_receive_log_error_zone, OB_CLUSTER_PARAMETER, "", "specifies the zone name that palf module want to inject error when receive log",
     ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
 ERRSIM_DEF_STR(migrate_check_member_list_error_zone, OB_CLUSTER_PARAMETER, "", "specifies the zone name that migrate want to inject error when change member list",

src/share/rc/ob_tenant_base.h

@@ -200,6 +200,7 @@ namespace observer
 }
 namespace storage {
   class MockTenantModuleEnv;
+  class ObStorageHADiagMgr;
 }
 
 namespace share
@@ -357,6 +358,7 @@ using ObTableScanIteratorObjPool = common::ObServerObjectPool<oceanbase::storage
       storage::ObTabletMemtableMgrPool*,            \
       rootserver::ObMViewMaintenanceService*,       \
       storage::checkpoint::ObCheckpointDiagnoseMgr*, \
+      storage::ObStorageHADiagMgr*,                  \
       common::sqlclient::ObTenantDblinkKeeper*,      \
       storage::ObGlobalIteratorPool*                 \
   )

src/storage/CMakeLists.txt

@@ -203,6 +203,8 @@ ob_set_subtarget(ob_storage high_availability
   high_availability/ob_rebuild_service.cpp
   high_availability/ob_tablet_transfer_info.cpp
   high_availability/ob_ls_block_tx_service.cpp
+  high_availability/ob_storage_ha_diagnose_mgr.cpp
+  high_availability/ob_storage_ha_diagnose_service.cpp
 )
 
 ob_set_subtarget(ob_storage restore

src/storage/high_availability/ob_finish_transfer.cpp

@@ -23,6 +23,8 @@
 #include "storage/high_availability/ob_transfer_lock_utils.h"
 #include "storage/tablet/ob_tablet.h"
 #include "observer/ob_server_event_history_table_operator.h"
+#include "share/ob_storage_ha_diagnose_struct.h"
+#include "storage/high_availability/ob_storage_ha_diagnose_mgr.h"
 
 using namespace oceanbase::common;
 using namespace oceanbase::share;
@@ -50,7 +52,9 @@ ObTxFinishTransfer::ObTxFinishTransfer()
       dest_ls_id_(),
       mutex_(),
       cond_(),
-      sql_proxy_(NULL)
+      sql_proxy_(NULL),
+      round_(0),
+      diagnose_result_msg_(share::ObStorageHACostItemName::MAX_NAME)
 {}
 
 ObTxFinishTransfer::~ObTxFinishTransfer()
@@ -120,7 +124,11 @@ int ObTxFinishTransfer::do_tx_transfer_doing_(const ObTransferTaskID &task_id, c
   ObDisplayTabletList table_lock_tablet_list;
   transaction::tablelock::ObTableLockOwnerID lock_owner_id;
   ObTimeoutCtx timeout_ctx;
-  const int64_t tmp_round = round;
+  round_ = round;
+  const int64_t start_ts = ObTimeUtil::current_time();
+  process_perf_diagnose_info_(ObStorageHACostItemName::TRANSFER_FINISH_BEGIN,
+      start_ts, tablet_list.count(), round_, false/*is_report*/);
+  diagnose_result_msg_ = share::ObStorageHACostItemName::MAX_NAME;
   if (!task_id.is_valid() || OB_INVALID_ID == tenant_id || !src_ls_id.is_valid() || !dest_ls_id.is_valid()) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("get invalid args", K(ret), K(task_id), K(tenant_id), K(src_ls_id), K(dest_ls_id));
@@ -203,6 +211,8 @@ int ObTxFinishTransfer::do_tx_transfer_doing_(const ObTransferTaskID &task_id, c
         LOG_INFO("transfer in tablet not ready", K(ret), K(tenant_id), K(dest_ls_id));
         transfer_service->wakeup();
       } else {
+        process_perf_diagnose_info_(ObStorageHACostItemName::CHECK_LS_LOGICAL_TABLE_REPLACED_LATER,
+            0/*start_ts*/, tablet_list.count(), round_, false/*is_report*/);
         // 4. The leader node of dest_ls registers the multi-source transaction
         // ObInnerSQLConnection->register_multi_source_data, and the type is TX_FINISH_TRANSFER_IN (two-way barrier)
         //    The content of the log is src_ls_id, dest_ls_id, tablet_list. This step requires forcibly flushing the redo
@@ -280,13 +290,22 @@ int ObTxFinishTransfer::do_tx_transfer_doing_(const ObTransferTaskID &task_id, c
         // 8. unlock table lock on src ls for tablet (must be successful)
         else if (OB_FAIL(ObTransferLockUtil::unlock_tablet_on_src_ls_for_table_lock(
                       trans, tenant_id, src_ls_id, lock_owner_id, table_lock_tablet_list))) {
+          diagnose_result_msg_ = share::ObStorageHACostItemName::UNLOCK_TABLET_FOR_LOCK;
           LOG_WARN("failed to unlock tablet on src ls for table lock", KR(ret),
               K(tenant_id), K(src_ls_id), K(lock_owner_id), K(table_lock_tablet_list));
         }
         // 9. unlock member list
-        else if (OB_FAIL(
-                    unlock_ls_member_list_(tenant_id, dest_ls_id, member_list, lock_status, CONFIG_CHANGE_TIMEOUT))) {
-          LOG_WARN("failed to unlock ls member list", K(ret), K(tenant_id), K(dest_ls_id), K(member_list));
+        else {
+          process_perf_diagnose_info_(ObStorageHACostItemName::UNLOCK_TABLET_FOR_LOCK,
+              0/*start_ts*/, tablet_list.count(), round_, false/*is_report*/);
+          if (OB_FAIL(
+                      unlock_ls_member_list_(tenant_id, dest_ls_id, member_list, lock_status, CONFIG_CHANGE_TIMEOUT))) {
+            diagnose_result_msg_ = share::ObStorageHACostItemName::UNLOCK_LS_MEMBER_LIST;
+            LOG_WARN("failed to unlock ls member list", K(ret), K(tenant_id), K(dest_ls_id), K(member_list));
+          } else {
+            process_perf_diagnose_info_(ObStorageHACostItemName::UNLOCK_LS_MEMBER_LIST,
+                0/*start_ts*/, tablet_list.count(), round_, false/*is_report*/);
+          }
         }
 
 #ifdef ERRSIM
@@ -303,9 +322,15 @@ int ObTxFinishTransfer::do_tx_transfer_doing_(const ObTransferTaskID &task_id, c
         if (OB_TMP_FAIL(commit_trans_(is_commit, trans))) {
           if (OB_SUCCESS == ret) {
             ret = tmp_ret;
+            diagnose_result_msg_ = share::ObStorageHACostItemName::FINISH_TRANS_COMMIT;
           }
         } else if (is_commit) {
           round = 0;
+          ObTransferUtils::reset_related_info(dest_ls_id);
+        }
+        if (OB_SUCC(ret)) {
+          process_perf_diagnose_info_(ObStorageHACostItemName::FINISH_TRANS_COMMIT,
+              0/*start_ts*/, tablet_list.count(), round_, false/*is_report*/);
         }
         // 11. After the dest_ls leader succeeds,
         // it will report the corresponding results to RS.
@@ -316,9 +341,28 @@ int ObTxFinishTransfer::do_tx_transfer_doing_(const ObTransferTaskID &task_id, c
       }
     }
   }
-  if (OB_TMP_FAIL(record_server_event_(ret, is_ready, tmp_round, start_scn))) {
+  if (OB_TMP_FAIL(record_server_event_(ret, is_ready, round_, start_scn))) {
     LOG_WARN("failed to record server event", K(tmp_ret), K(ret), K(is_ready));
   }
+#ifdef ERRSIM
+  if (OB_SUCC(ret)) {
+    ret = EN_TRANSFER_DIAGNOSE_DOING_FAILED ? : OB_SUCCESS;
+    if (OB_FAIL(ret)) {
+      STORAGE_LOG(WARN, "fake EN_TRANSFER_DIAGNOSE_DOING_FAILED", K(ret));
+    }
+  }
+#endif
+  if (OB_FAIL(ret)) {
+    if (OB_TMP_FAIL(ObStorageHADiagMgr::add_transfer_error_diagnose_info(task_id, dest_ls_id,
+        ObStorageHADiagTaskType::TRANSFER_DOING, round_, ret, diagnose_result_msg_))) {
+      LOG_WARN("failed to add error diagnose info", K(tmp_ret), K(ret), K(task_id), K(dest_ls_id), K(round_));
+    }
+  }
+
+  if (OB_SUCC(ret) && is_ready) {
+    process_perf_diagnose_info_(ObStorageHACostItemName::TRANSFER_FINISH_END,
+        0/*start_ts*/, tablet_list.count(), round_, true/*is_report*/);
+  }
   return ret;
 }
 
@@ -353,6 +397,11 @@ int ObTxFinishTransfer::unlock_src_and_dest_ls_member_list_(const uint64_t tenan
 
   }
   UNUSEDx(member_list);
+  if (OB_SUCC(ret)) {
+    process_perf_diagnose_info_(ObStorageHACostItemName::UNLOCK_SRC_AND_DEST_LS_MEMBER_LIST, round_, false/*is_report*/);
+  } else {
+    diagnose_result_msg_ = share::ObStorageHACostItemName::UNLOCK_SRC_AND_DEST_LS_MEMBER_LIST;
+  }
   return ret;
 }
 
@@ -417,6 +466,11 @@ int ObTxFinishTransfer::wait_transfer_tablet_status_normal_(
       }
     }
   }
+  if (OB_SUCC(ret)) {
+    process_perf_diagnose_info_(ObStorageHACostItemName::WAIT_TRANSFER_TABLET_STATUS_NORMAL, 0/*start_ts*/, tablet_list.count(), round_, false/*is_report*/);
+  } else {
+    diagnose_result_msg_ = share::ObStorageHACostItemName::WAIT_TRANSFER_TABLET_STATUS_NORMAL;
+  }
   return ret;
 }
 
@@ -503,6 +557,11 @@ int ObTxFinishTransfer::check_ls_logical_table_replaced(const uint64_t tenant_id
   } else {
     LOG_INFO("check ls logical table replace", K(tenant_id), K(dest_ls_id), K(addr_array), K(tablet_list), K(quorum), K(all_backfilled));
   }
+  if (OB_SUCC(ret)) {
+    process_perf_diagnose_info_(ObStorageHACostItemName::CHECK_LS_LOGICAL_TABLE_REPLACED, 0/*start_ts*/, tablet_list.count(), round_, false/*is_report*/);
+  } else {
+    diagnose_result_msg_ = share::ObStorageHACostItemName::CHECK_LS_LOGICAL_TABLE_REPLACED;
+  }
   return ret;
 }
 
@@ -592,7 +651,7 @@ int ObTxFinishTransfer::do_tx_finish_transfer_in_(const ObTransferTaskID &task_i
     ret = OB_ERR_UNEXPECTED;
     LOG_WARN("conn should not be null", K(ret));
   } else if (OB_FAIL(build_tx_finish_transfer_in_info_(
-                 src_ls_id, dest_ls_id, start_scn, tablet_list, finish_transfer_in_info))) {
+                 task_id, src_ls_id, dest_ls_id, start_scn, tablet_list, finish_transfer_in_info))) {
     LOG_WARN("failed to build tx finish transfer in info",
         K(ret),
         K(src_ls_id),
@@ -618,6 +677,11 @@ int ObTxFinishTransfer::do_tx_finish_transfer_in_(const ObTransferTaskID &task_i
     }
 #endif
   }
+  if (OB_SUCC(ret)) {
+    process_perf_diagnose_info_(ObStorageHACostItemName::REGISTER_TRANSFER_FINISH_IN, 0/*start_ts*/, tablet_list.count(), round_, false/*is_report*/);
+  } else {
+    diagnose_result_msg_ = share::ObStorageHACostItemName::REGISTER_TRANSFER_FINISH_IN;
+  }
   return ret;
 }
 
@@ -641,7 +705,7 @@ int ObTxFinishTransfer::do_tx_finish_transfer_out_(const ObTransferTaskID &task_
     ret = OB_ERR_UNEXPECTED;
     LOG_WARN("conn should not be null", K(ret));
   } else if (OB_FAIL(build_tx_finish_transfer_out_info_(
-                 src_ls_id, dest_ls_id, finish_scn, tablet_list, finish_transfer_out_info))) {
+                 task_id, src_ls_id, dest_ls_id, finish_scn, tablet_list, finish_transfer_out_info))) {
     LOG_WARN("failed to build tx finish transfer out info",
         K(ret),
         K(src_ls_id),
@@ -666,9 +730,13 @@ int ObTxFinishTransfer::do_tx_finish_transfer_out_(const ObTransferTaskID &task_
       }
     }
 #endif
-
     DEBUG_SYNC(AFTER_FINISH_TRANSFER_OUT);
   }
+  if (OB_SUCC(ret)) {
+    process_perf_diagnose_info_(ObStorageHACostItemName::REGISTER_TRANSFER_OUT, 0/*start_ts*/, tablet_list.count(), round_, false/*is_report*/);
+  } else {
+    diagnose_result_msg_ = share::ObStorageHACostItemName::REGISTER_TRANSFER_OUT;
+  }
   return ret;
 }
 
@@ -711,7 +779,11 @@ int ObTxFinishTransfer::wait_all_ls_replica_replay_scn_(const ObTransferTaskID &
 #endif
 
   DEBUG_SYNC(AFTER_DOING_TRANSFER_WAIT_REPLAY_SCN);
-
+  if (OB_SUCC(ret)) {
+    process_perf_diagnose_info_(ObStorageHACostItemName::WAIT_ALL_LS_REPLICA_REPLAY_FINISH_SCN, round_, false/*is_report*/);
+  } else {
+    diagnose_result_msg_ = share::ObStorageHACostItemName::WAIT_ALL_LS_REPLICA_REPLAY_FINISH_SCN;
+  }
   return ret;
 }
 
@@ -822,10 +894,16 @@ int ObTxFinishTransfer::lock_ls_member_list_(const uint64_t tenant_id, const sha
 
     DEBUG_SYNC(AFTER_DOING_TRANSFER_LOCK_MEMBER_LIST);
   }
+  if (OB_SUCC(ret)) {
+    process_perf_diagnose_info_(ObStorageHACostItemName::LOCK_LS_MEMBER_LIST_IN_DOING, round_, false/*is_report*/);
+  } else {
+    diagnose_result_msg_ = share::ObStorageHACostItemName::LOCK_LS_MEMBER_LIST_IN_DOING;
+  }
   return ret;
 }
 
-int ObTxFinishTransfer::build_tx_finish_transfer_in_info_(const share::ObLSID &src_ls_id,
+int ObTxFinishTransfer::build_tx_finish_transfer_in_info_(
+    const share::ObTransferTaskID &task_id, const share::ObLSID &src_ls_id,
     const share::ObLSID &dest_ls_id, const share::SCN &start_scn,
     const common::ObArray<ObTransferTabletInfo> &tablet_list, ObTXFinishTransferInInfo &transfer_in_info)
 {
@@ -838,6 +916,7 @@ int ObTxFinishTransfer::build_tx_finish_transfer_in_info_(const share::ObLSID &s
     transfer_in_info.src_ls_id_ = src_ls_id;
     transfer_in_info.dest_ls_id_ = dest_ls_id;
     transfer_in_info.start_scn_ = start_scn;
+    transfer_in_info.task_id_ = task_id;
     transfer_in_info.data_version_ = DEFAULT_MIN_DATA_VERSION;
     if (OB_FAIL(transfer_in_info.tablet_list_.assign(tablet_list))) {
       LOG_WARN("failed to assign tablet list", K(ret), K(tablet_list));
@@ -846,7 +925,8 @@ int ObTxFinishTransfer::build_tx_finish_transfer_in_info_(const share::ObLSID &s
   return ret;
 }
 
-int ObTxFinishTransfer::build_tx_finish_transfer_out_info_(const share::ObLSID &src_ls_id,
+int ObTxFinishTransfer::build_tx_finish_transfer_out_info_(
+    const share::ObTransferTaskID &task_id, const share::ObLSID &src_ls_id,
     const share::ObLSID &dest_ls_id, const share::SCN &finish_scn,
     const common::ObArray<ObTransferTabletInfo> &tablet_list, ObTXFinishTransferOutInfo &transfer_out_info)
 {
@@ -859,6 +939,7 @@ int ObTxFinishTransfer::build_tx_finish_transfer_out_info_(const share::ObLSID &
     transfer_out_info.src_ls_id_ = src_ls_id;
     transfer_out_info.dest_ls_id_ = dest_ls_id;
     transfer_out_info.finish_scn_ = finish_scn;
+    transfer_out_info.task_id_ = task_id;
     transfer_out_info.data_version_ = DEFAULT_MIN_DATA_VERSION;
     if (OB_FAIL(transfer_out_info.tablet_list_.assign(tablet_list))) {
       LOG_WARN("failed to assign tablet list", K(ret), K(tablet_list));
@@ -1111,5 +1192,47 @@ int ObTxFinishTransfer::write_server_event_(const int32_t result, const ObSqlStr
   return ret;
 }
 
+void ObTxFinishTransfer::process_perf_diagnose_info_(
+    const ObStorageHACostItemName name,
+    const int64_t start_ts,
+    const int64_t tablet_count,
+    const int64_t round, const bool is_report) const
+{
+  int ret = OB_SUCCESS;
+  ObArenaAllocator alloc;
+  ObTransferPerfDiagInfo info;
+  ObStorageHATimestampItem item;
+  item.name_ = name;
+  item.type_ = ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE;
+  item.retry_id_ = round;
+  item.timestamp_ = ObTimeUtil::current_time();
+  common::ObTabletID tablet_id;
+  share::ObStorageHADiagTaskKey key;
+  if (OB_FAIL(info.init(&alloc, MTL_ID()))) {
+    LOG_WARN("fail to init info", K(ret));
+  } else if (OB_FAIL(info.add_item(item))) {
+    LOG_WARN("fail to add item", K(ret), K(item));
+  } else if (OB_FAIL(ObStorageHADiagMgr::construct_diagnose_info_key(task_id_, ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE,
+      ObStorageHADiagTaskType::TRANSFER_DOING, ObStorageHADiagType::PERF_DIAGNOSE, round, tablet_id, key))) {
+    LOG_WARN("failed to construct error diagnose info key", K(ret), K(task_id_), K(round), K(tablet_id));
+  } else if (OB_FAIL(ObStorageHADiagMgr::construct_diagnose_info(task_id_, dest_ls_id_,
+      ObStorageHADiagTaskType::TRANSFER_DOING, round, OB_SUCCESS, ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE, info))) {
+    LOG_WARN("failed to construct diagnose info", K(ret), K(task_id_), K(round), K(dest_ls_id_));
+  } else if (OB_FAIL(ObStorageHADiagMgr::add_transfer_perf_diagnose_info(key, start_ts, tablet_count, is_report, info))) {
+    LOG_WARN("failed to add perf diagnose info", K(ret), K(key), K(tablet_count), K(round), K(info), K(start_ts), K(is_report));
+  }
+}
+
+void ObTxFinishTransfer::process_perf_diagnose_info_(
+    const ObStorageHACostItemName name,
+    const int64_t round,
+    const bool is_report) const
+{
+  // start_ts and tablet_count only need to be effective on the first call
+  // needn't to update in the later
+  process_perf_diagnose_info_(name,
+      0/*start_ts*/, 0/*tablet_count*/, round_, false/*is_report*/);
+}
+
 }  // namespace storage
 }  // namespace oceanbase

src/storage/high_availability/ob_finish_transfer.h

@@ -162,7 +162,8 @@ private:
   // @param[in]: start_scn
   // @param[in]: tablet_list
   // @param[out]: transfer_out_info
-  int build_tx_finish_transfer_in_info_(const share::ObLSID &src_ls_id, const share::ObLSID &dest_ls_id,
+  int build_tx_finish_transfer_in_info_(const share::ObTransferTaskID &task_id, const share::ObLSID &src_ls_id,
+      const share::ObLSID &dest_ls_id,
       const share::SCN &start_scn, const common::ObArray<share::ObTransferTabletInfo> &tablet_list,
       ObTXFinishTransferInInfo &transfer_out_info);
 
@@ -172,7 +173,8 @@ private:
   // @param[in]: finish_scn
   // @param[in]: tablet_list
   // @param[out]: transfer_out_info
-  int build_tx_finish_transfer_out_info_(const share::ObLSID &src_ls_id, const share::ObLSID &dest_ls_id,
+  int build_tx_finish_transfer_out_info_(const share::ObTransferTaskID &task_id, const share::ObLSID &src_ls_id,
+      const share::ObLSID &dest_ls_id,
       const share::SCN &finish_scn, const common::ObArray<share::ObTransferTabletInfo> &tablet_list,
       ObTXFinishTransferOutInfo &transfer_out_info);
 
@@ -268,6 +270,17 @@ private:
       const ObSqlString &extra_info,
       const share::ObTransferStatus &status) const;
 
+  void process_perf_diagnose_info_(
+      const ObStorageHACostItemName name,
+      const int64_t start_ts,
+      const int64_t tablet_count,
+      const int64_t round, const bool is_report) const;
+
+  void process_perf_diagnose_info_(
+      const ObStorageHACostItemName name,
+      const int64_t round,
+      const bool is_report) const;
+
 private:
   static const int64_t DEFAULT_WAIT_INTERVAL_US = 10_ms;
   static const int64_t TASK_EXECUTE_LONG_WARNING_THRESHOLD = 60_min;
@@ -281,6 +294,8 @@ private:
   mutable lib::ObMutex mutex_;
   common::ObThreadCond cond_;
   common::ObMySQLProxy *sql_proxy_;
+  int64_t round_;
+  share::ObStorageHACostItemName diagnose_result_msg_;
   DISALLOW_COPY_AND_ASSIGN(ObTxFinishTransfer);
 };
 

src/storage/high_availability/ob_storage_ha_diagnose_mgr.cpp

@@ -0,0 +1,747 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX STORAGE
+#include "ob_storage_ha_diagnose_mgr.h"
+#include "storage/meta_mem/ob_tenant_meta_mem_mgr.h"
+#include "observer/omt/ob_tenant_config_mgr.h"
+
+namespace oceanbase
+{
+using namespace share;
+
+namespace storage
+{
+/**
+ * ------------------------------ObStorageHADiagIterator---------------------
+ */
+ObStorageHADiagIterator::ObStorageHADiagIterator()
+  : keys_(),
+    cnt_(0),
+    cur_idx_(0),
+    is_opened_(false)
+{
+}
+
+int ObStorageHADiagIterator::open(const ObStorageHADiagType &type)
+{
+  int ret = OB_SUCCESS;
+  if (is_opened_) {
+    ret = OB_INIT_TWICE;
+    LOG_WARN("The ObStorageHADiagIterator has been opened", K(ret));
+  } else if (type >= ObStorageHADiagType::MAX_TYPE
+    || type < ObStorageHADiagType::ERROR_DIAGNOSE) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument!", K(ret), K(type));
+  } else {
+    ObStorageHADiagMgr *mgr = nullptr;
+    if (OB_ISNULL(mgr = MTL(ObStorageHADiagMgr *))) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_WARN("failed to get ObStorageHADiagMgr from MTL", K(ret));
+    } else if (OB_FAIL(mgr->get_iter_array(type, keys_))) {
+      LOG_WARN("failed to get iter info", K(ret), K(type));
+    } else {
+      cur_idx_ = 0;
+      cnt_ = keys_.count();
+      is_opened_ = true;
+    }
+  }
+  return ret;
+}
+
+int ObStorageHADiagIterator::get_next_info(ObStorageHADiagInfo &info)
+{
+  int ret = OB_SUCCESS;
+  if (!is_opened_) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not init", K(ret));
+  } else if (cur_idx_ >= cnt_) {
+    ret = OB_ITER_END;
+  } else if (cnt_ < 0) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("unexpected array count", K(ret), K(cnt_));
+  } else {
+    ObStorageHADiagMgr *mgr = nullptr;
+    if (OB_ISNULL(mgr = MTL(ObStorageHADiagMgr *))) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_WARN("failed to get ObStorageHADiagMgr from MTL", K(ret));
+    } else {
+      while (OB_SUCC(ret) && cur_idx_ <= cnt_) {
+        if (cur_idx_ == cnt_) {
+          ret = OB_ITER_END;
+          break;
+        } else if (OB_FAIL(mgr->get(keys_.at(cur_idx_++), info))) {
+          //overwrite ret
+          if (OB_ENTRY_NOT_EXIST == ret) {
+            ret = OB_SUCCESS;
+            LOG_INFO("info has already been cleaned", K(ret), "key", keys_.at(cur_idx_ - 1));
+          } else {
+            LOG_WARN("fail to get info", K(ret), "key", keys_.at(cur_idx_ - 1));
+          }
+        } else {
+          break;
+        }
+      }
+    }
+  }
+  return ret;
+}
+
+bool ObStorageHADiagIterator::is_valid() const
+{
+  return cur_idx_ >= 0 && cnt_ >= 0 && !keys_.empty();
+}
+
+void ObStorageHADiagIterator::reset()
+{
+  is_opened_= false;
+  keys_.reset();
+  cnt_ = 0;
+  cur_idx_ = 0;
+}
+
+int ObStorageHADiagIterator::get_cur_key(ObStorageHADiagTaskKey &key) const
+{
+  int ret = OB_SUCCESS;
+  key.reset();
+  if (!is_opened_) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not init", K(ret));
+  } else {
+    if (0 == keys_.count() || cur_idx_ >= cnt_) {
+      ret = OB_ITER_END;
+    } else {
+      key = keys_.at(cur_idx_);
+      if (!key.is_valid()) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_WARN("key is invalid", K(ret), K(key));
+      }
+    }
+  }
+  return ret;
+}
+
+bool ObStorageHADiagIterator::is_opened() const
+{
+  return is_opened_;
+}
+/**
+ * ------------------------------ObStorageHADiagMgr---------------------
+ */
+ObStorageHADiagMgr::ObStorageHADiagMgr()
+  : is_inited_(false),
+    allocator_(),
+    task_list_(),
+    service_(nullptr),
+    lock_()
+{
+}
+
+int ObStorageHADiagMgr::mtl_init(ObStorageHADiagMgr *&storage_ha_diag_mgr)
+{
+  return storage_ha_diag_mgr->init(MTL_ID(), INFO_PAGE_SIZE, INFO_MAX_SIZE);
+}
+
+int ObStorageHADiagMgr::init(
+        const uint64_t tenant_id,
+        const int64_t page_size,
+        const int64_t max_size)
+{
+  int ret = OB_SUCCESS;
+  if (IS_INIT) {
+    ret = OB_INIT_TWICE;
+    LOG_WARN("ObStorageHADiagMgr has already been inited", K(ret));
+  } else if (OB_INVALID_TENANT_ID == tenant_id
+    || page_size <= 0
+    || max_size <= 0
+    || max_size < page_size) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(tenant_id), K(page_size), K(max_size));
+  } else {
+    int64_t upper_align_size = upper_align(max_size, page_size);
+    if (OB_FAIL(allocator_.init(ObMallocAllocator::get_instance(),
+                                    page_size,
+                                    lib::ObMemAttr(tenant_id, "SHADiagMgr"),
+                                    0/*init_size*/,
+                                    upper_align_size,
+                                    upper_align_size))) {
+      LOG_WARN("failed to init allocator", K(ret), K(page_size), K(tenant_id), K(upper_align_size));
+    }
+  }
+  if (OB_SUCC(ret)) {
+    service_ = &(ObStorageHADiagService::instance());
+    is_inited_ = true;
+  } else {
+    clear();
+  }
+  return ret;
+}
+
+void ObStorageHADiagMgr::clear()
+{
+  is_inited_ = false;
+  allocator_.reset();
+  task_list_.reset();
+  service_ = nullptr;
+}
+
+void ObStorageHADiagMgr::destroy()
+{
+  is_inited_ = false;
+  common::SpinWLockGuard guard(lock_);
+  clear_with_no_lock_();
+  allocator_.reset();
+  service_ = nullptr;
+}
+
+void ObStorageHADiagMgr::clear_with_no_lock_()
+{
+  DLIST_FOREACH_REMOVESAFE_NORET(iter, task_list_) {
+    ObStorageHADiagTask *tmp_task = task_list_.remove(iter);
+    free_(tmp_task);
+  }
+  task_list_.clear();
+}
+
+int ObStorageHADiagMgr::add_or_update(
+    const ObStorageHADiagTaskKey &task_key,
+    const ObStorageHADiagInfo &input_info,
+    const bool is_report)
+{
+  int ret = OB_SUCCESS;
+  ObStorageHADiagInfo *info = nullptr;
+  ObStorageHADiagTask *task = nullptr;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not init", K(ret));
+  } else if (!task_key.is_valid() || !input_info.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(task_key), K(input_info));
+  } else {
+    common::SpinWLockGuard guard(lock_);
+    if (OB_FAIL(get_task_(task_key, info))) {
+      //overwrite ret
+      if (OB_ENTRY_NOT_EXIST == ret) {
+        if (OB_FAIL(add_task_(task_key, input_info, task))) {
+          LOG_WARN("failed to add task", K(ret), K(task_key), K(input_info));
+        }
+      } else {
+        LOG_WARN("failed to check task exist", K(ret), K(task_key));
+      }
+    } else {
+      if (OB_FAIL(info->update(input_info))) {
+        LOG_WARN("failed to update info", K(ret), K(input_info));
+      }
+    }
+  }
+  if (OB_FAIL(ret)) {
+    //do nothing
+  } else if (is_report) {
+    if (OB_FAIL(service_->add_task(task_key))) {
+      LOG_WARN("failed to add task to service", K(ret), K(task_key));
+    } else if (REACH_TENANT_TIME_INTERVAL(1 * 100 * 1000L)) {
+      service_->wakeup();
+    }
+  }
+  return ret;
+}
+
+int ObStorageHADiagMgr::add_task_(
+    const ObStorageHADiagTaskKey &task_key,
+    const ObStorageHADiagInfo &input_info,
+    ObStorageHADiagTask *&out_task)
+{
+  int ret = OB_SUCCESS;
+  out_task = nullptr;
+  if (!task_key.is_valid() || !input_info.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(task_key), K(input_info));
+  } else {
+    while (OB_SUCC(ret)) {
+      if (OB_FAIL(create_task_(task_key, input_info, out_task))) {
+        if (OB_ALLOCATE_MEMORY_FAILED == ret) {
+          // overwrite ret
+          if (!task_list_.is_empty() && OB_FAIL(purge_task_(task_list_.get_first()))) {
+            LOG_WARN("failed to purge task", K(ret), K(task_list_));
+          }
+        } else {
+          LOG_WARN("failed to create task", K(ret), K(task_key), K(input_info));
+        }
+      } else {
+        break;
+      }
+    }
+    if (OB_FAIL(ret)) {
+      //do nothing
+    } else if (!task_list_.add_last(out_task)) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_WARN("failed to add list", K(ret), KPC(out_task));
+    }
+  }
+  return ret;
+}
+//TODO(zhixing.yh) modify task key relationship with info, info include task key
+int ObStorageHADiagMgr::create_task_(
+    const ObStorageHADiagTaskKey &task_key,
+    const ObStorageHADiagInfo &input_info,
+    ObStorageHADiagTask *&task)
+{
+  int ret = OB_SUCCESS;
+  task = nullptr;
+  void *buf = nullptr;
+  ObStorageHADiagInfo *info = nullptr;
+  ObStorageHADiagTask *tmp_task = nullptr;
+  if (!task_key.is_valid() || !input_info.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(task_key), K(input_info));
+  } else if (OB_FAIL(input_info.deep_copy(allocator_, info))) {
+    LOG_WARN("failed to alloc memory for diagnose info", K(ret), K(input_info));
+  } else if (OB_ISNULL(buf = allocator_.alloc(sizeof(ObStorageHADiagTask)))) {
+    ret = OB_ALLOCATE_MEMORY_FAILED;
+    LOG_WARN("alloc memory for ObStorageHADiagTask failed", K(ret));
+  } else if (OB_ISNULL(tmp_task = new (buf) ObStorageHADiagTask())) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("new storage ha monitor info is nullptr", K(ret));
+  } else {
+    buf = nullptr;
+    if (OB_FAIL(tmp_task->key_.assign(task_key))) {
+      LOG_WARN("failed to assign task key", K(ret), K(task_key));
+    } else {
+      tmp_task->val_ = info;
+      info->task_ = tmp_task;
+      info = nullptr;
+    }
+    if (OB_NOT_NULL(info)) {
+      tmp_task->~ObStorageHADiagTask();
+      allocator_.free(tmp_task);
+      tmp_task = nullptr;
+    }
+  }
+
+  if (OB_FAIL(ret)) {
+    if (OB_NOT_NULL(info)) {
+      free_info_(info);
+    }
+    if (OB_NOT_NULL(buf)) {
+      allocator_.free(buf);
+      buf = nullptr;
+    }
+  } else {
+    task = tmp_task;
+  }
+
+  return ret;
+}
+
+int ObStorageHADiagMgr::purge_task_(ObStorageHADiagTask *task)
+{
+  int ret = OB_SUCCESS;
+  if (OB_ISNULL(task)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("task is nullptr", K(ret));
+  } else {
+    ObStorageHADiagTask *tmp_task = task_list_.remove(task);
+    free_(tmp_task);
+  }
+  return ret;
+}
+
+int ObStorageHADiagMgr::del(const ObStorageHADiagTaskKey &task_key)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not init", K(ret));
+  } else if (!task_key.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("task key is invalid", K(ret), K(task_key));
+  } else {
+    common::SpinWLockGuard guard(lock_);
+    DLIST_FOREACH_X(iter, task_list_, OB_SUCC(ret)) {
+      if (iter->key_ == task_key) {
+        if (OB_FAIL(purge_task_(iter))) {
+          LOG_WARN("failed to purge task", K(ret), K(task_key));
+        }
+        break;
+      }
+    }
+  }
+
+  return ret;
+}
+
+int ObStorageHADiagMgr::get_iter_array(
+    const ObStorageHADiagType &type,
+    ObIArray<ObStorageHADiagTaskKey> &array) const
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not init", K(ret));
+  } else if (type < ObStorageHADiagType::ERROR_DIAGNOSE
+             || type >= ObStorageHADiagType::MAX_TYPE) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(type));
+  } else {
+    common::SpinRLockGuard guard(lock_);
+    DLIST_FOREACH_X(it, task_list_, OB_SUCC(ret)) {
+      if (type == it->key_.diag_type_) {
+        if (OB_FAIL(array.push_back(it->key_))) {
+          LOG_WARN("failed to push back array.", K(ret), K(it->key_));
+        }
+      }
+    }
+  }
+  return ret;
+}
+
+int ObStorageHADiagMgr::get(const ObStorageHADiagTaskKey &key, ObStorageHADiagInfo &info) const
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not init", K(ret));
+  } else if (!key.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(key));
+  } else {
+    common::SpinRLockGuard guard(lock_);
+    if (OB_FAIL(copy_with_nolock_(key, info))) {
+      LOG_WARN("failed to copy info", K(ret), K(key));
+    }
+  }
+  return ret;
+}
+
+int ObStorageHADiagMgr::get_task_(
+    const ObStorageHADiagTaskKey &key,
+    ObStorageHADiagInfo *&info) const
+{
+  int ret = OB_SUCCESS;
+  info = nullptr;
+  bool found_info = false;
+  DLIST_FOREACH_X(iter, task_list_, OB_SUCC(ret)) {
+    if (iter->key_ == key) {
+      info = iter->val_;
+      found_info = true;
+      break;
+    }
+  }
+  if (OB_SUCC(ret)) {
+    if (!found_info) {
+      ret = OB_ENTRY_NOT_EXIST;
+    }
+  }
+  return ret;
+}
+
+int ObStorageHADiagMgr::copy_with_nolock_(const ObStorageHADiagTaskKey &key, ObStorageHADiagInfo &info) const
+{
+  int ret = OB_SUCCESS;
+  ObStorageHADiagInfo *task_info = nullptr;
+  if (OB_FAIL(get_task_(key, task_info))) {
+    LOG_WARN("fail to check task exist", K(ret), K(key));
+  } else if (OB_FAIL(info.assign(*task_info))) {
+    LOG_WARN("fail to copy info", K(ret), KPC(task_info));
+  } else if (ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE == key.module_
+      && OB_FAIL(static_cast<ObTransferPerfDiagInfo &>(info).copy_item_list(
+      static_cast<ObTransferPerfDiagInfo &>(*task_info)))) {
+    LOG_WARN("fail to copy item list to info", K(ret), KPC(task_info));
+  }
+  return ret;
+}
+
+void ObStorageHADiagMgr::free_info_(ObStorageHADiagInfo *&info)
+{
+  if (OB_NOT_NULL(info)) {
+    info->~ObStorageHADiagInfo();
+    allocator_.free(info);
+    info = nullptr;
+  }
+}
+
+void ObStorageHADiagMgr::free_task_(ObStorageHADiagTask *&task)
+{
+  if (OB_NOT_NULL(task)) {
+    task->~ObStorageHADiagTask();
+    allocator_.free(task);
+    task = nullptr;
+  }
+}
+
+void ObStorageHADiagMgr::free_(ObStorageHADiagTask *&task)
+{
+  if (OB_NOT_NULL(task)) {
+    free_info_(task->val_);
+    free_task_(task);
+  }
+}
+//TODO(zhixing.yh) modify task key relationship with info, info include task key
+//can not reset info, perf diagnose info perhaps already init
+int ObStorageHADiagMgr::construct_diagnose_info(
+    const share::ObTransferTaskID task_id, const share::ObLSID &ls_id,
+    const share::ObStorageHADiagTaskType type, const int64_t retry_id,
+    const int result_code, const share::ObStorageHADiagModule module,
+    share::ObStorageHADiagInfo &info)
+{
+  int ret = OB_SUCCESS;
+  if (!task_id.is_valid()
+      || !ls_id.is_valid()
+      || type >= ObStorageHADiagTaskType::MAX_TYPE
+      || type < ObStorageHADiagTaskType::TRANSFER_START
+      || module >= ObStorageHADiagModule::MAX_MODULE
+      || module < ObStorageHADiagModule::TRANSFER_ERROR_DIAGNOSE
+      || retry_id < 0) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(task_id), K(ls_id), K(type), K(module), K(retry_id), K(result_code));
+  } else {
+    info.task_id_ = task_id.id();
+    info.ls_id_ = ls_id;
+    info.module_ = module;
+    info.type_ = type;
+    info.retry_id_ = retry_id;
+    info.result_code_ = result_code;
+  }
+  return ret;
+}
+
+int ObStorageHADiagMgr::construct_error_diagnose_info(share::ObTransferErrorDiagInfo &info)
+{
+  int ret = OB_SUCCESS;
+  info.timestamp_ = ObTimeUtility::current_time();
+  info.trace_id_ = *ObCurTraceId::get_trace_id();
+  info.thread_id_ = GETTID();
+  return ret;
+}
+
+int ObStorageHADiagMgr::construct_diagnose_info_key(
+    const share::ObTransferTaskID task_id,
+    const share::ObStorageHADiagModule module,
+    const share::ObStorageHADiagTaskType type,
+    const share::ObStorageHADiagType diag_type,
+    const int64_t retry_id,
+    const common::ObTabletID &tablet_id,
+    share::ObStorageHADiagTaskKey &key)
+{
+  int ret = OB_SUCCESS;
+  key.reset();
+  // tablet_id do not check, it is only effective in backfill
+  if (!task_id.is_valid()
+      || module >= ObStorageHADiagModule::MAX_MODULE
+      || module < ObStorageHADiagModule::TRANSFER_ERROR_DIAGNOSE
+      || type >= ObStorageHADiagTaskType::MAX_TYPE
+      || type < ObStorageHADiagTaskType::TRANSFER_START
+      || diag_type >= ObStorageHADiagType::MAX_TYPE
+      || diag_type < ObStorageHADiagType::ERROR_DIAGNOSE
+      || retry_id < 0) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(task_id), K(module), K(type),
+        K(diag_type), K(retry_id));
+  } else {
+    key.tenant_id_ = MTL_ID();
+    key.task_id_ = task_id.id();
+    key.module_ = module;
+    key.type_ = type;
+    key.retry_id_ = retry_id;
+    key.diag_type_ = diag_type;
+    key.tablet_id_ = tablet_id;
+  }
+  return ret;
+}
+
+//TODO(zhixing.yh) add result_msg
+int ObStorageHADiagMgr::add_info_to_mgr(
+    const ObStorageHADiagInfo &info,
+    const ObStorageHADiagTaskKey &key,
+    const bool is_report)
+{
+  int ret = OB_SUCCESS;
+  ObStorageHADiagMgr *mgr = nullptr;
+  if (!info.is_valid() || !key.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(info), K(key));
+  } else if (OB_ISNULL(mgr = MTL(ObStorageHADiagMgr *))) {
+    ret = OB_ERR_UNEXPECTED;
+    STORAGE_LOG(WARN, "failed to get ObStorageHADiagMgr from mtl", K(ret));
+  } else if (OB_FAIL(mgr->add_or_update(key, info, is_report))) {
+    STORAGE_LOG(WARN, "failed to add transfer diag info", K(ret), K(key), K(info));
+  } else {
+    STORAGE_LOG(DEBUG, "success to add transfer diag info", K(ret), K(key), K(info));
+  }
+  return ret;
+}
+
+int ObStorageHADiagMgr::append_perf_diagnose_info(
+    const common::ObTabletID &tablet_id,
+    share::ObStorageHADiagInfo &info)
+{
+  int ret = OB_SUCCESS;
+  //do not check tablet_id invalid
+  info.tablet_id_ = tablet_id;
+  return ret;
+}
+
+int ObStorageHADiagMgr::append_error_diagnose_info(
+    const common::ObTabletID &tablet_id,
+    const share::ObStorageHACostItemName result_msg,
+    share::ObStorageHADiagInfo &info)
+{
+  int ret = OB_SUCCESS;
+  // MAX_NAME also is valid
+  if (result_msg < share::ObStorageHACostItemName::TRANSFER_START_BEGIN
+      || result_msg > share::ObStorageHACostItemName::MAX_NAME) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(result_msg));
+  } else {
+    //do not check tablet_id invalid
+    info.result_msg_ = result_msg;
+    info.tablet_id_ = tablet_id;
+  }
+  return ret;
+}
+
+int ObStorageHADiagMgr::add_transfer_error_diagnose_info(
+    const share::ObTransferTaskID task_id, const share::ObLSID &ls_id,
+    const share::ObStorageHADiagTaskType type, const int64_t retry_id,
+    const int result_code, const common::ObTabletID &tablet_id, const share::ObStorageHACostItemName result_msg)
+{
+  int ret = OB_SUCCESS;
+  share::ObTransferErrorDiagInfo info;
+  share::ObStorageHADiagTaskKey key;
+  if (!task_id.is_valid()
+      || !ls_id.is_valid()
+      || type >= ObStorageHADiagTaskType::MAX_TYPE
+      || type < ObStorageHADiagTaskType::TRANSFER_START
+      || retry_id < 0
+      || OB_SUCCESS == result_code) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(task_id), K(ls_id), K(type), K(retry_id), K(result_code));
+  } else if (OB_FAIL(construct_diagnose_info(task_id, ls_id, type, retry_id, result_code,
+      ObStorageHADiagModule::TRANSFER_ERROR_DIAGNOSE, info))) {
+    LOG_WARN("failed to construct error diagnose info", K(ret), K(task_id), K(ls_id), K(type), K(retry_id), K(result_code));
+  } else if (OB_FAIL(append_error_diagnose_info(tablet_id, result_msg, info))) {
+    LOG_WARN("fail to append diagnose info", K(ret), K(tablet_id), K(result_msg));
+  } else if (OB_FAIL(construct_error_diagnose_info(info))) {
+    LOG_WARN("failed to construct error diagnose info", K(ret));
+  } else if (OB_FAIL(construct_diagnose_info_key(task_id, ObStorageHADiagModule::TRANSFER_ERROR_DIAGNOSE,
+      type, ObStorageHADiagType::ERROR_DIAGNOSE, retry_id, tablet_id, key))) {
+    LOG_WARN("failed to construct error diagnose info key", K(ret), K(task_id), K(type), K(retry_id), K(tablet_id));
+  } else if (OB_FAIL(add_info_to_mgr(info, key, true/*is_report*/))) {
+    LOG_WARN("failed to add error diagnose info", K(ret), K(info), K(key));
+  }
+  return ret;
+}
+
+int ObStorageHADiagMgr::add_transfer_error_diagnose_info(
+    const share::ObTransferTaskID task_id, const share::ObLSID &ls_id,
+    const share::ObStorageHADiagTaskType type, const int64_t retry_id,
+    const int result_code, const share::ObStorageHACostItemName result_msg)
+{
+  int ret = OB_SUCCESS;
+  common::ObTabletID unused_tablet_id;
+  if (OB_FAIL(add_transfer_error_diagnose_info(task_id, ls_id, type, retry_id,
+      result_code, unused_tablet_id, result_msg))) {
+    LOG_WARN("fail to add transfer error diagnose info", K(ret), K(task_id), K(ls_id), K(type),
+        K(retry_id), K(result_code), K(unused_tablet_id), K(result_msg));
+  }
+  return ret;
+}
+
+int ObStorageHADiagMgr::construct_perf_diagnose_info(
+    const int64_t tablet_count,
+    const int64_t start_timestamp,
+    share::ObTransferPerfDiagInfo &info)
+{
+  int ret = OB_SUCCESS;
+  if (start_timestamp < 0 || tablet_count < 0) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(start_timestamp));
+  } else {
+    info.timestamp_ = start_timestamp;
+    info.end_timestamp_ = ObTimeUtility::current_time();
+    info.tablet_count_ = tablet_count;
+  }
+  return ret;
+}
+
+int ObStorageHADiagMgr::add_transfer_perf_diagnose_info(
+    const share::ObStorageHADiagTaskKey &key,
+    const int64_t start_timestamp, const int64_t tablet_count,
+    const bool is_report, share::ObTransferPerfDiagInfo &info)
+{
+  int ret = OB_SUCCESS;
+  if (!key.is_valid() || start_timestamp < 0 || tablet_count < 0) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(key), K(start_timestamp), K(tablet_count));
+  } else if (OB_FAIL(construct_perf_diagnose_info(tablet_count, start_timestamp, info))) {
+    LOG_WARN("failed to construct perf diagnose info", K(ret), K(tablet_count), K(start_timestamp));
+  } else if (OB_FAIL(add_info_to_mgr(info, key, is_report))) {
+    LOG_WARN("failed to add perf diagnose info", K(ret), K(info), K(key), K(is_report));
+  }
+  return ret;
+}
+
+int ObStorageHADiagMgr::add_key_to_service_(const ObIArray<ObStorageHADiagTaskKey> &task_keys)
+{
+  int ret = OB_SUCCESS;
+  for (int64_t i = 0; OB_SUCC(ret) && i < task_keys.count(); i++) {
+    const ObStorageHADiagTaskKey &key = task_keys.at(i);
+    if (OB_FAIL(service_->add_task(key))) {
+      LOG_WARN("failed to add task to service", K(ret), K(key));
+    }
+  }
+  return ret;
+}
+
+int ObStorageHADiagMgr::add_task_key_to_report_array_(
+    const share::ObTransferTaskID &task_id,
+    ObIArray<ObStorageHADiagTaskKey> &task_keys) const
+{
+  int ret = OB_SUCCESS;
+  task_keys.reset();
+  if (!task_id.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(task_id));
+  } else {
+    common::SpinRLockGuard guard(lock_);
+    DLIST_FOREACH_X(iter, task_list_, OB_SUCC(ret)) {
+      if ((ObStorageHADiagType::PERF_DIAGNOSE == iter->key_.diag_type_
+          && iter->key_.task_id_ == task_id.id()
+          && (ObStorageHADiagTaskType::TRANSFER_START == iter->key_.type_
+          || ObStorageHADiagTaskType::TRANSFER_DOING == iter->key_.type_
+          || ObStorageHADiagTaskType::TRANSFER_ABORT == iter->key_.type_))) {
+        if (OB_FAIL(task_keys.push_back(iter->key_))) {
+          LOG_WARN("failed to add task", K(ret), K(iter->key_));
+        }
+      }
+    }
+  }
+  return ret;
+}
+
+int ObStorageHADiagMgr::report_task(const share::ObTransferTaskID &task_id)
+{
+  int ret = OB_SUCCESS;
+  ObArray<ObStorageHADiagTaskKey> task_keys;
+  if (!task_id.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(task_id));
+  } else if (OB_FAIL(add_task_key_to_report_array_(task_id, task_keys))) {
+    LOG_WARN("failed to add task to report array", K(ret), K(task_id));
+  } else if (OB_FAIL(add_key_to_service_(task_keys))) {
+    LOG_WARN("failed to add task to service", K(ret), K(task_keys));
+  }
+  return ret;
+}
+
+}
+}

src/storage/high_availability/ob_storage_ha_diagnose_mgr.h

@@ -0,0 +1,158 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEABASE_STORAGE_HA_DIAGNOSE_
+#define OCEABASE_STORAGE_HA_DIAGNOSE_
+
+#include "share/ob_storage_ha_diagnose_struct.h"
+#include "ob_storage_ha_diagnose_service.h"
+#include "storage/ob_i_store.h"
+#include "lib/list/ob_dlist.h"
+
+namespace oceanbase
+{
+using namespace share;
+namespace storage
+{
+class ObStorageHADiagIterator final
+{
+public:
+  ObStorageHADiagIterator();
+  ~ObStorageHADiagIterator() { reset(); }
+  int open(const ObStorageHADiagType &type);
+  int get_next_info(ObStorageHADiagInfo &info);
+  bool is_valid() const;
+  int get_cur_key(ObStorageHADiagTaskKey &key) const;
+  void reset();
+  bool is_opened() const;
+
+  TO_STRING_KV(
+    K_(cnt),
+    K_(cur_idx),
+    K_(is_opened));
+
+private:
+  ObArray<ObStorageHADiagTaskKey> keys_;
+  int64_t cnt_;
+  int64_t cur_idx_;
+  bool is_opened_;
+
+  DISALLOW_COPY_AND_ASSIGN(ObStorageHADiagIterator);
+};
+
+class ObStorageHADiagMgr final
+{
+public:
+  ObStorageHADiagMgr();
+  ~ObStorageHADiagMgr() { destroy(); }
+  static int mtl_init(ObStorageHADiagMgr *&storage_ha_diag_mgr);
+  int init(
+      const uint64_t tenant_id,
+      const int64_t page_size = INFO_PAGE_SIZE,
+      const int64_t max_size = INFO_MAX_SIZE);
+
+  int add_or_update(
+      const ObStorageHADiagTaskKey &task_key,
+      const ObStorageHADiagInfo &input_info,
+      const bool is_report);
+  int del(const ObStorageHADiagTaskKey &task_keys);
+  int get_iter_array(const ObStorageHADiagType &type, ObIArray<ObStorageHADiagTaskKey> &array) const;
+  int get(const ObStorageHADiagTaskKey &key, ObStorageHADiagInfo &info) const;
+  void destroy();
+  void clear();
+
+  static int construct_diagnose_info(
+      const share::ObTransferTaskID task_id, const share::ObLSID &ls_id,
+      const share::ObStorageHADiagTaskType type, const int64_t retry_id,
+      const int result_code, const share::ObStorageHADiagModule module,
+      share::ObStorageHADiagInfo &info);
+  static int construct_diagnose_info_key(
+      const share::ObTransferTaskID task_id,
+      const share::ObStorageHADiagModule module,
+      const share::ObStorageHADiagTaskType type,
+      const share::ObStorageHADiagType diag_type,
+      const int64_t retry_id,
+      const common::ObTabletID &tablet_id,
+      share::ObStorageHADiagTaskKey &key);
+  static int construct_error_diagnose_info(share::ObTransferErrorDiagInfo &info);
+
+  static int add_info_to_mgr(
+      const ObStorageHADiagInfo &info,
+      const ObStorageHADiagTaskKey &key,
+      const bool is_report);
+
+  static int append_perf_diagnose_info(
+    const common::ObTabletID &tablet_id,
+    share::ObStorageHADiagInfo &info);
+
+  static int append_error_diagnose_info(
+    const common::ObTabletID &tablet_id,
+    const share::ObStorageHACostItemName result_msg,
+    share::ObStorageHADiagInfo &info);
+
+  static int add_transfer_error_diagnose_info(
+      const share::ObTransferTaskID task_id, const share::ObLSID &ls_id,
+      const share::ObStorageHADiagTaskType type, const int64_t retry_id,
+      const int result_code, const share::ObStorageHACostItemName result_msg);
+  static int add_transfer_error_diagnose_info(
+      const share::ObTransferTaskID task_id, const share::ObLSID &ls_id,
+      const share::ObStorageHADiagTaskType type, const int64_t retry_id,
+      const int result_code, const common::ObTabletID &tablet_id, const share::ObStorageHACostItemName result_msg);
+  static int construct_perf_diagnose_info(
+      const int64_t tablet_count,
+      const int64_t start_timestamp,
+      share::ObTransferPerfDiagInfo &info);
+  static int add_transfer_perf_diagnose_info(
+    const share::ObStorageHADiagTaskKey &key, const int64_t start_timestamp,
+    const int64_t tablet_count, const bool is_report, share::ObTransferPerfDiagInfo &info);
+  int report_task(const share::ObTransferTaskID &task_id);
+  int add_task_key_to_report_array_(
+    const share::ObTransferTaskID &task_id,
+    ObIArray<ObStorageHADiagTaskKey> &task_keys) const;
+  int add_key_to_service_(const ObIArray<ObStorageHADiagTaskKey> &task_keys);
+
+public:
+  static const int64_t INFO_PAGE_SIZE = (1 << 16); // 64KB
+  static const int64_t INFO_PAGE_SIZE_LIMIT = (1 << 11); // 2KB
+  static const int64_t INFO_MAX_SIZE = 16LL * 1024LL * 1024LL; // 16MB // lowest
+
+private:
+  int create_task_(
+      const ObStorageHADiagTaskKey &task_key,
+      const ObStorageHADiagInfo &input_info,
+      ObStorageHADiagTask *&task);
+  int purge_task_(ObStorageHADiagTask *task);
+
+  int add_task_(
+      const ObStorageHADiagTaskKey &task_key,
+      const ObStorageHADiagInfo &input_info,
+      ObStorageHADiagTask *&out_task);
+  void clear_with_no_lock_();
+  int get_task_(const ObStorageHADiagTaskKey &key, ObStorageHADiagInfo *&info) const;
+  int copy_with_nolock_(const ObStorageHADiagTaskKey &key, ObStorageHADiagInfo &info) const;
+  void free_task_(ObStorageHADiagTask *&task);
+  void free_info_(ObStorageHADiagInfo *&val);
+  void free_(ObStorageHADiagTask *&task);
+
+private:
+  bool is_inited_;
+  common::ObFIFOAllocator allocator_;
+  ObDList<ObStorageHADiagTask> task_list_;
+  ObStorageHADiagService *service_;
+  SpinRWLock lock_;
+
+  DISALLOW_COPY_AND_ASSIGN(ObStorageHADiagMgr);
+};
+
+}
+}
+#endif

src/storage/high_availability/ob_storage_ha_diagnose_service.cpp

@@ -0,0 +1,690 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX STORAGE
+
+#include "ob_storage_ha_diagnose_service.h"
+#include "ob_storage_ha_diagnose_mgr.h"
+#include "observer/ob_server_struct.h"
+#include "storage/tx_storage/ob_ls_handle.h"
+#include "storage/tx_storage/ob_ls_service.h"
+#include "observer/omt/ob_multi_tenant.h"
+
+namespace oceanbase
+{
+using namespace share;
+namespace storage
+{
+
+ObStorageHADiagService::ObStorageHADiagService()
+  : is_inited_(false),
+    thread_cond_(),
+    wakeup_cnt_(0),
+    op_(),
+    sql_proxy_(nullptr),
+    task_keys_(),
+    lock_(),
+    err_diag_end_timestamp_(0),
+    perf_diag_end_timestamp_(0)
+{
+}
+
+int ObStorageHADiagService::init(common::ObMySQLProxy *sql_proxy)
+{
+  int ret = OB_SUCCESS;
+  if (is_inited_) {
+    ret = OB_INIT_TWICE;
+    LOG_WARN("storage ha diagnose service is aleady init", K(ret));
+  } else if (OB_FAIL(op_.init())) {
+    LOG_WARN("failed to init op", K(ret));
+  } else if (OB_FAIL(thread_cond_.init(ObWaitEventIds::HA_SERVICE_COND_WAIT))) {
+    LOG_WARN("failed to init ha service thread cond", K(ret));
+  } else {
+    lib::ObMemAttr attr(OB_SERVER_TENANT_ID, "StorageHADiag");
+    common::SpinWLockGuard guard(lock_);
+    if (OB_FAIL(task_keys_.create(REPORT_KEY_MAX_NUM, attr))) {
+      LOG_WARN("failed to init hash map", K(ret));
+    } else {
+      sql_proxy_ = sql_proxy;
+      is_inited_ = true;
+    }
+  }
+
+  return ret;
+}
+
+void ObStorageHADiagService::wakeup()
+{
+  ObThreadCondGuard cond_guard(thread_cond_);
+  wakeup_cnt_++;
+  thread_cond_.signal();
+}
+
+void ObStorageHADiagService::destroy()
+{
+  COMMON_LOG(INFO, "ObStorageHADiagService starts to destroy");
+  is_inited_ = false;
+  thread_cond_.destroy();
+  wakeup_cnt_ = 0;
+  sql_proxy_ = nullptr;
+  common::SpinWLockGuard guard(lock_);
+  task_keys_.destroy();
+  err_diag_end_timestamp_ = 0;
+  perf_diag_end_timestamp_ = 0;
+  COMMON_LOG(INFO, "ObStorageHADiagService destroyed");
+}
+
+void ObStorageHADiagService::stop()
+{
+  if (is_inited_) {
+    COMMON_LOG(INFO, "ObStorageHADiagService starts to stop");
+    lib::ThreadPool::stop();
+    wakeup();
+    COMMON_LOG(INFO, "ObStorageHADiagService stopped");
+  }
+}
+
+void ObStorageHADiagService::wait()
+{
+  if (is_inited_) {
+    COMMON_LOG(INFO, "ObStorageHADiagService starts to wait");
+    lib::ThreadPool::wait();
+    COMMON_LOG(INFO, "ObStorageHADiagService finish to wait");
+  }
+}
+
+int ObStorageHADiagService::start()
+{
+  int ret = OB_SUCCESS;
+  if (!is_inited_) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("storage ha monitor service do not init", K(ret));
+  } else {
+    if (OB_FAIL(lib::ThreadPool::start())) {
+      COMMON_LOG(WARN, "ObStorageHADiagService start thread failed", K(ret));
+    } else {
+      COMMON_LOG(INFO, "ObStorageHADiagService start");
+    }
+  }
+  return ret;
+}
+
+void ObStorageHADiagService::run1()
+{
+  int ret = OB_SUCCESS;
+  lib::set_thread_name("ObStorageHADiagService");
+  uint64_t data_version = 0;
+  while (!has_set_stop()) {
+    if (IS_NOT_INIT) {
+      ret = OB_NOT_INIT;
+      LOG_WARN("not init", K(ret));
+    } else if (observer::ObServiceStatus::SS_SERVING != GCTX.status_) {
+      ret = OB_SERVER_IS_INIT;
+      LOG_WARN("server is not serving", K(ret), K(GCTX.status_));
+    } else if (OB_FAIL(GET_MIN_DATA_VERSION(OB_SYS_TENANT_ID, data_version))) {
+      LOG_WARN("failed to get data version", K(ret));
+    } else if (!((data_version >= DATA_VERSION_4_2_2_0 && data_version < DATA_VERSION_4_3_0_0)
+        || data_version >= DATA_VERSION_4_3_1_0)) { /* [4.2.2, 4.3.0) && [4.3.1, ~) */
+      //do nothing
+    } else if (OB_FAIL(do_clean_history_(ObStorageHADiagModule::TRANSFER_ERROR_DIAGNOSE,
+        err_diag_end_timestamp_))) {
+      LOG_WARN("failed to do clean error diagnose info", K(ret), K(err_diag_end_timestamp_));
+    } else if (OB_FAIL(do_clean_history_(ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE,
+        perf_diag_end_timestamp_))) {
+      LOG_WARN("failed to do clean perf diagnose info", K(ret), K(perf_diag_end_timestamp_));
+    } else if (OB_FAIL(do_report_())) {
+      LOG_WARN("failed to do report", K(ret));
+    } else if (OB_FAIL(do_clean_transfer_related_info_())) {
+      LOG_WARN("failed to do clean transfer related info", K(ret));
+    }
+
+    ObThreadCondGuard guard(thread_cond_);
+    if (has_set_stop() || wakeup_cnt_ > 0) {
+      wakeup_cnt_ = 0;
+    } else {
+      int64_t wait_time_ms = 5 * 60 * 1000L;
+#ifdef ERRSIM
+      wait_time_ms = GCONF.errsim_transfer_diagnose_server_wait_time / 1000;
+#endif
+      thread_cond_.wait(wait_time_ms);
+    }
+  }
+}
+
+ObStorageHADiagService &ObStorageHADiagService::instance()
+{
+  static ObStorageHADiagService storage_ha_diag_service;
+  return storage_ha_diag_service;
+}
+
+int ObStorageHADiagService::get_info_from_type_(const ObStorageHADiagTaskKey &key, ObStorageHADiagInfo *&info,
+                                                ObTransferErrorDiagInfo &transfer_err_diag,
+                                                ObTransferPerfDiagInfo &transfer_perf_diag,
+                                                ObIAllocator &alloc)
+{
+  int ret = OB_SUCCESS;
+  info = nullptr;
+  if (key.is_valid()) {
+    switch(key.module_) {
+      case ObStorageHADiagModule::TRANSFER_ERROR_DIAGNOSE: {
+        info = &transfer_err_diag;
+        break;
+      }
+      case ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE: {
+        if (!transfer_perf_diag.is_inited() && OB_FAIL(transfer_perf_diag.init(&alloc, OB_SERVER_TENANT_ID))) {
+          LOG_WARN("fail to init info", K(ret));
+        } else {
+          info = &transfer_perf_diag;
+        }
+        break;
+      }
+      default: {
+        ret = OB_INVALID_ARGUMENT;
+        LOG_WARN("invalid module", K(ret), K(key.module_));
+        break;
+      }
+    }
+  } else {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("unexpected error", K(ret), K(key));
+  }
+
+  return ret;
+}
+
+int ObStorageHADiagService::add_task(const ObStorageHADiagTaskKey &key)
+{
+  int ret =OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not init", K(ret));
+  } else if (!key.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(key));
+  } else {
+    common::SpinWLockGuard guard(lock_);
+    int64_t now = ObTimeUtility::current_time();
+    if (REPORT_KEY_MAX_NUM == task_keys_.size()) {
+      if (OB_FAIL(remove_oldest_without_lock_())) {
+        LOG_WARN("failed to remove first", K(ret), K(task_keys_.size()));
+      }
+    }
+    if (OB_FAIL(ret)) {
+      // do nothing
+    } else if (OB_FAIL(task_keys_.set_refactored(key, now))) {
+      LOG_WARN("failed to add key to hash map", K(ret), K(key), K(now));
+    }
+  }
+  return ret;
+}
+
+int ObStorageHADiagService::get_oldest_key_without_lock_(ObStorageHADiagTaskKey &key) const
+{
+  int ret = OB_SUCCESS;
+  key.reset();
+  int64_t old_timestamp = INT64_MAX;
+  TaskKeyMap::const_iterator it = task_keys_.begin();
+  for (; OB_SUCC(ret) && it != task_keys_.end(); ++it) {
+    int64_t timestamp = it->second;
+    if (timestamp < old_timestamp) {
+      old_timestamp = timestamp;
+      key = it->first;
+    }
+  }
+  if (!key.is_valid()) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("key is invalid", K(ret), K(key));
+  }
+  return ret;
+}
+
+int ObStorageHADiagService::del_task_in_mgr_(const ObStorageHADiagTaskKey &task_key) const
+{
+  int ret = OB_SUCCESS;
+  ObStorageHADiagMgr *mgr = nullptr;
+  if (!task_key.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(task_key));
+  } else {
+    MAKE_TENANT_SWITCH_SCOPE_GUARD(guard);
+    if (OB_FAIL(guard.switch_to(task_key.tenant_id_, false))) {
+      if (OB_TENANT_NOT_IN_SERVER == ret) {
+        // overwrite ret
+        ret = OB_SUCCESS;
+      } else {
+        LOG_WARN("fail to switch tenant", K(ret), K(task_key.tenant_id_));
+      }
+    } else {
+      if (OB_ISNULL(mgr = MTL(ObStorageHADiagMgr *))) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_WARN("fail to get ObStorageHADiagMgr", K(ret), K(task_key.tenant_id_));
+      } else if (OB_FAIL(mgr->del(task_key))) {
+        LOG_WARN("fail to del task key", K(ret), K(task_key));
+      }
+    }
+  }
+  return ret;
+}
+
+int ObStorageHADiagService::remove_oldest_without_lock_()
+{
+  int ret = OB_SUCCESS;
+  ObStorageHADiagTaskKey key;
+  if (OB_FAIL(get_oldest_key_without_lock_(key))) {
+    LOG_WARN("failed to get oldest key", K(ret), K(key));
+  } else if (OB_FAIL(clean_task_key_without_lock_(key))) {
+    LOG_WARN("failed to clean task", K(ret), K(key));
+  } else if (OB_FAIL(del_task_in_mgr_(key))) {
+    LOG_WARN("failed to del task in mgr", K(ret), K(key));
+  } else {
+    LOG_INFO("succeed to remove task key", K(ret), K(key));
+  }
+  return ret;
+}
+
+int ObStorageHADiagService::do_clean_history_(const ObStorageHADiagModule module, int64_t &end_timestamp)
+{
+  int ret = OB_SUCCESS;
+  int tmp_ret = OB_SUCCESS;
+  ObArray<int64_t> timestamp_array;
+  int64_t delete_index = 0;
+  while (OB_SUCC(ret)) {
+    const int64_t now = ObTimeUtility::current_time();
+    const int64_t delete_timestamp = now - GCONF._ha_diagnose_history_recycle_interval;
+    if (OB_FAIL(op_.get_batch_row_keys(*sql_proxy_, OB_SYS_TENANT_ID,
+        module, end_timestamp, timestamp_array))) {
+      LOG_WARN("failed to get row keys", K(ret), K(end_timestamp), K(timestamp_array));
+    } else if (0 == timestamp_array.count()) {
+      break;
+      LOG_INFO("have not item need to clean ", K(ret), K(timestamp_array.count()));
+    } else if (timestamp_array.at(0) > delete_timestamp) {
+      break;
+      LOG_INFO("have not item need to clean ", K(ret), K(delete_timestamp), K(timestamp_array.at(0)));
+    } else if (OB_FAIL(op_.do_batch_delete(*sql_proxy_, OB_SYS_TENANT_ID,
+        module, timestamp_array, delete_timestamp, delete_index))) {
+      LOG_WARN("failed to delete batch row", K(ret), K(timestamp_array), K(delete_timestamp));
+    } else {
+      end_timestamp = timestamp_array.at(delete_index);
+    }
+  }
+  end_timestamp = 0;
+  return ret;
+}
+
+int ObStorageHADiagService::insert_inner_table_(
+    ObMySQLTransaction &trans,
+    ObStorageHADiagMgr *mgr,
+    const ObStorageHADiagTaskKey &task_key,
+    ObStorageHADiagInfo &info)
+{
+  int ret = OB_SUCCESS;
+  if (OB_ISNULL(mgr) || !task_key.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(task_key), KP(mgr));
+  } else if (OB_FAIL(mgr->get(task_key, info))) {
+    //overwrite ret
+    if (OB_ENTRY_NOT_EXIST == ret) {
+      ret = OB_SUCCESS;
+    }
+    LOG_INFO("info has already been cleaned", K(ret), K(task_key));
+  } else if (OB_FAIL(op_.insert_row(trans, OB_SYS_TENANT_ID, task_key.tenant_id_, info, info.module_))) {
+    LOG_WARN("failed to insert row", K(ret), K(info));
+  }
+  return ret;
+}
+
+int ObStorageHADiagService::deep_copy_keys_(TaskKeyArray &do_report_keys) const
+{
+  int ret =OB_SUCCESS;
+  do_report_keys.reset();
+  common::SpinRLockGuard guard(lock_);
+  TaskKeyMap::const_iterator it = task_keys_.begin();
+  for (int64_t i = 0; OB_SUCC(ret) && it != task_keys_.end() && i < ONCE_REPORT_KEY_MAX_NUM; ++it, ++i) {
+    const ObStorageHADiagTaskKey &key = it->first;
+    if (OB_FAIL(do_report_keys.push_back(key))) {
+      LOG_WARN("failed to add key to report keys", K(ret), K(key), K(do_report_keys));
+    }
+  }
+  return ret;
+}
+
+int ObStorageHADiagService::clean_task_key_without_lock_(const ObStorageHADiagTaskKey &task_key)
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(task_keys_.erase_refactored(task_key))) {
+    if (OB_HASH_NOT_EXIST == ret) {
+      ret = OB_SUCCESS;
+    } else {
+      LOG_WARN("fail to erase_refactored", KR(ret), K(task_key));
+    }
+  }
+  return ret;
+}
+
+int ObStorageHADiagService::report_to_inner_table_(
+    ObMySQLTransaction &trans,
+    ObStorageHADiagMgr *mgr,
+    const ObStorageHADiagTaskKey &task_key)
+{
+  int ret = OB_SUCCESS;
+  ObStorageHADiagInfo *info = nullptr;
+  ObArenaAllocator alloc;
+  ObTransferErrorDiagInfo transfer_err_diag;
+  ObTransferPerfDiagInfo transfer_perf_diag;
+  if (OB_ISNULL(mgr) || !task_key.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(task_key));
+  } else {
+    if (OB_FAIL(get_info_from_type_(task_key, info, transfer_err_diag, transfer_perf_diag, alloc))) {
+      LOG_WARN("failed to get info", K(ret), K(task_key), K(transfer_err_diag), K(transfer_perf_diag));
+    } else if (OB_FAIL(insert_inner_table_(trans, mgr, task_key, *info))) {
+      LOG_WARN("failed to insert inner table", K(ret), K(task_key));
+    }
+  }
+  return ret;
+}
+
+int ObStorageHADiagService::start_trans_(
+    ObTimeoutCtx &timeout_ctx,
+    ObMySQLTransaction &trans)
+{
+  int ret = OB_SUCCESS;
+  int64_t stmt_timeout = 10_s;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("ObStorageHADiagService do not init", K(ret));
+  } else if (OB_FAIL(timeout_ctx.set_trx_timeout_us(stmt_timeout))) {
+    LOG_WARN("fail to set trx timeout", K(ret), K(stmt_timeout));
+  } else if (OB_FAIL(timeout_ctx.set_timeout(stmt_timeout))) {
+    LOG_WARN("set timeout context failed", K(ret));
+  } else if (OB_FAIL(trans.start(sql_proxy_, OB_SYS_TENANT_ID))) {
+    LOG_WARN("failed to start trans", K(ret));
+  }
+  return ret;
+}
+
+int ObStorageHADiagService::commit_trans_(
+    const int32_t result,
+    ObMySQLTransaction &trans)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("ObStorageHADiagService do not init", K(ret));
+  } else if (OB_FAIL(trans.end(OB_SUCCESS == result))) {
+    LOG_WARN("end transaction failed", K(ret));
+  }
+
+  return ret;
+}
+
+int ObStorageHADiagService::report_batch_task_key_(ObMySQLTransaction &trans, const TaskKeyArray &task_keys)
+{
+  int ret = OB_SUCCESS;
+  uint64_t tenant_id = OB_INVALID_TENANT_ID;
+  ObStorageHADiagMgr *mgr = nullptr;
+  for (int64_t cur_index = 0; OB_SUCC(ret) && cur_index < task_keys.count(); cur_index++) {
+    const ObStorageHADiagTaskKey &task_key = task_keys.at(cur_index);
+    if (!task_key.is_valid()) {
+      ret = OB_INVALID_ARGUMENT;
+      LOG_WARN("invalid argument!", K(ret), K(task_key));
+    } else {
+      MAKE_TENANT_SWITCH_SCOPE_GUARD(guard);
+      if (task_key.tenant_id_ != tenant_id) {
+        if (OB_FAIL(guard.switch_to(task_key.tenant_id_, false))) {
+          if (OB_TENANT_NOT_IN_SERVER == ret) {
+            // overwrite ret
+            ret = OB_SUCCESS;
+            continue;
+          } else {
+            LOG_WARN("fail to switch tenant", K(ret), K(task_key.tenant_id_));
+          }
+        } else {
+          tenant_id = task_key.tenant_id_;
+          if (OB_ISNULL(mgr = MTL(ObStorageHADiagMgr *))) {
+            ret = OB_ERR_UNEXPECTED;
+            LOG_WARN("fail to get ObStorageHADiagMgr", K(ret), K(tenant_id));
+          }
+        }
+      }
+      if (OB_FAIL(ret)) {
+        //do nothing
+      } else if (OB_FAIL(report_to_inner_table_(trans, mgr, task_key))) {
+        LOG_WARN("fail to report to inner table", K(ret), K(task_key));
+      }
+    }
+  }
+  return ret;
+}
+
+int ObStorageHADiagService::batch_del_task_in_mgr_(const TaskKeyArray &task_keys)
+{
+  int ret = OB_SUCCESS;
+  uint64_t tenant_id = OB_INVALID_TENANT_ID;
+  ObStorageHADiagMgr *mgr = nullptr;
+  for (int64_t cur_index = 0; OB_SUCC(ret) && cur_index < task_keys.count(); cur_index++) {
+    const ObStorageHADiagTaskKey &task_key = task_keys.at(cur_index);
+    if (!task_key.is_valid()) {
+      ret = OB_INVALID_ARGUMENT;
+      LOG_WARN("invalid argument!", K(ret), K(task_key));
+    } else {
+      MAKE_TENANT_SWITCH_SCOPE_GUARD(guard);
+      if (task_key.tenant_id_ != tenant_id) {
+        if (OB_FAIL(guard.switch_to(task_key.tenant_id_, false))) {
+          if (OB_TENANT_NOT_IN_SERVER == ret) {
+            // overwrite ret
+            ret = OB_SUCCESS;
+            continue;
+          } else {
+            LOG_WARN("fail to switch tenant", K(ret), K(task_key.tenant_id_));
+          }
+        } else {
+          tenant_id = task_key.tenant_id_;
+          if (OB_ISNULL(mgr = MTL(ObStorageHADiagMgr *))) {
+            ret = OB_ERR_UNEXPECTED;
+            LOG_WARN("fail to get ObStorageHADiagMgr", K(ret), K(tenant_id));
+          }
+        }
+      }
+      if (OB_FAIL(ret)) {
+        //do nothing
+      } else if (OB_FAIL(mgr->del(task_key))) {
+        LOG_WARN("fail to del task key", K(ret), K(task_key));
+      } else {
+        common::SpinWLockGuard guard(lock_);
+        if (OB_FAIL(clean_task_key_without_lock_(task_key))) {
+          LOG_WARN("failed to clean task keys", K(ret), K(task_key));
+        }
+      }
+    }
+  }
+  return ret;
+}
+
+int ObStorageHADiagService::report_process_(const TaskKeyArray &task_keys)
+{
+  int ret = OB_SUCCESS;
+  int tmp_ret = OB_SUCCESS;
+  uint64_t tenant_id = OB_INVALID_TENANT_ID;
+  ObMySQLTransaction trans;
+  ObTimeoutCtx timeout_ctx;
+  ObStorageHADiagMgr *mgr = nullptr;
+  if (OB_FAIL(start_trans_(timeout_ctx, trans))) {
+    LOG_WARN("failed to start trans", K(ret));
+  } else {
+    if (OB_FAIL(report_batch_task_key_(trans, task_keys))) {
+      LOG_WARN("failed to batch report task key", K(ret), K(task_keys));
+    }
+    if (OB_TMP_FAIL(commit_trans_(ret, trans))) {
+      LOG_WARN("failed to commit trans", K(tmp_ret), K(ret));
+      if (OB_SUCCESS == ret) {
+        ret = tmp_ret;
+      }
+    }
+    if (OB_SUCCESS == ret && OB_SUCCESS == tmp_ret) {
+      if (OB_FAIL(batch_del_task_in_mgr_(task_keys))) {
+        LOG_WARN("failed to delete tasks", K(ret), K(task_keys));
+      }
+    }
+  }
+  return ret;
+}
+
+int ObStorageHADiagService::do_report_()
+{
+  int ret = OB_SUCCESS;
+  TaskKeyArray do_report_keys;
+  if (OB_FAIL(deep_copy_keys_(do_report_keys))) {
+    LOG_WARN("failed to copy keys", K(ret), K(do_report_keys));
+  } else {
+    std::sort(do_report_keys.begin(), do_report_keys.end());
+    if (OB_FAIL(report_process_(do_report_keys))) {
+      LOG_WARN("failed to do report", K(ret), K(do_report_keys));
+    }
+  }
+
+  return ret;
+}
+
+int ObStorageHADiagService::do_clean_related_info_(ObLSService *ls_service, const share::ObLSID &ls_id, const uint64_t tenant_id)
+{
+  int ret = OB_SUCCESS;
+  ObLSHandle ls_handle;
+  ObLS *ls = nullptr;
+  share::ObTransferTaskID task_id;
+  int64_t result_count = 0;
+  if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::HA_MOD))) {
+    if (OB_ENTRY_NOT_EXIST == ret) {
+      // overwrite ret
+      ret = OB_SUCCESS;
+    } else {
+      LOG_WARN("failed to get ls", K(ret), K(ls_id));
+    }
+  } else if (OB_ISNULL(ls = ls_handle.get_ls())) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("ls should not be NULL", K(ret), KP(ls), K(ls_id));
+  } else if (OB_FAIL(ls->get_transfer_handler()->get_related_info_task_id(task_id))) {
+    LOG_WARN("failed to get transfer task id", K(ret), K(task_id));
+  } else if (!task_id.is_valid()) {
+    // do nothing
+  } else if (OB_FAIL(op_.check_transfer_task_exist(*sql_proxy_, tenant_id, task_id, result_count))) {
+    if (OB_TABLE_NOT_EXIST == ret) {
+      // overwrite ret
+      ret = OB_SUCCESS;
+    } else {
+      LOG_WARN("failed to check transfer task exist", K(ret), K(tenant_id), K(task_id), K(result_count));
+    }
+  } else if (0 != result_count) {
+    // do not clean related info
+  } else if (OB_FAIL(ls->get_transfer_handler()->reset_related_info(task_id))) {
+    LOG_WARN("failed to reset transfer related info", K(ret), K(task_id));
+  }
+  return ret;
+}
+
+int ObStorageHADiagService::scheduler_clean_related_info_(ObLSService *ls_service, const ObIArray<share::ObLSID> &ls_id_array, const uint64_t tenant_id)
+{
+  int ret = OB_SUCCESS;
+  if (ls_id_array.empty()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("ls id array empty", K(ret), K(ls_id_array));
+  } else {
+    for (int64_t i = 0; OB_SUCC(ret) && i < ls_id_array.count(); ++i) {
+      const share::ObLSID &ls_id = ls_id_array.at(i);
+      if (OB_FAIL(do_clean_related_info_(ls_service, ls_id, tenant_id))) {
+        LOG_WARN("failed to do clean related info", K(ret), K(ls_id), K(tenant_id));
+      }
+    }
+  }
+  return ret;
+}
+
+int ObStorageHADiagService::get_ls_id_array_(ObLSService *ls_service, ObIArray<share::ObLSID> &ls_id_array)
+{
+  int ret = OB_SUCCESS;
+  ls_id_array.reset();
+  common::ObSharedGuard<ObLSIterator> ls_iter_guard;
+  ObLSIterator *ls_iter = nullptr;
+  if (OB_FAIL(ls_service->get_ls_iter(ls_iter_guard, ObLSGetMod::HA_MOD))) {
+    LOG_WARN("failed to get ls iter", K(ret));
+  } else if (OB_ISNULL(ls_iter = ls_iter_guard.get_ptr())) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("ls iter should not be NULL", K(ret));
+  } else {
+    while (OB_SUCC(ret)) {
+      ObLS *ls = nullptr;
+      if (OB_FAIL(ls_iter->get_next(ls))) {
+        if (OB_ITER_END == ret) {
+          ret = OB_SUCCESS;
+          break;
+        }
+      } else if (OB_ISNULL(ls)) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_WARN("ls should not be NULL", K(ret), KP(ls));
+      } else if (OB_FAIL(ls_id_array.push_back(ls->get_ls_id()))) {
+        LOG_WARN("failed to push ls id into array", K(ret), KP(ls));
+      }
+    }
+  }
+  return ret;
+}
+
+int ObStorageHADiagService::do_clean_related_info_in_ls_(ObLSService *ls_service, const uint64_t tenant_id)
+{
+  int ret = OB_SUCCESS;
+  ObArray<share::ObLSID> ls_id_array;
+  if (OB_FAIL(get_ls_id_array_(ls_service, ls_id_array))) {
+    LOG_WARN("failed to get ls id array", K(ret));
+  } else if (ls_id_array.empty()) {
+    // do nothing
+  } else if (OB_FAIL(scheduler_clean_related_info_(ls_service, ls_id_array, tenant_id))) {
+    LOG_WARN("failed to scheduler clean related info", K(ret), K(ls_id_array), K(tenant_id));
+  }
+  return ret;
+}
+
+int ObStorageHADiagService::do_clean_transfer_related_info_()
+{
+  int ret = OB_SUCCESS;
+  common::ObArray<uint64_t> tenant_ids;
+  ObLSService * ls_service = nullptr;
+  if (OB_FAIL(GCTX.omt_->get_mtl_tenant_ids(tenant_ids))) {
+    LOG_WARN("fail to get_mtl_tenant_ids", K(ret));
+  } else {
+    for (int64_t i = 0; i < tenant_ids.size(); i++) {
+      uint64_t tenant_id = tenant_ids.at(i);
+      if (GCTX.omt_->is_available_tenant(tenant_id)) {
+        MAKE_TENANT_SWITCH_SCOPE_GUARD(guard);
+        if (OB_FAIL(guard.switch_to(tenant_id, false))) {
+          if (OB_TENANT_NOT_IN_SERVER == ret) {
+            // overwrite ret
+            ret = OB_SUCCESS;
+            continue;
+          } else {
+            LOG_WARN("fail to switch tenant", K(ret), K(tenant_id));
+          }
+        } else {
+          if (OB_ISNULL(ls_service = (MTL(ObLSService *)))) {
+            ret = OB_ERR_UNEXPECTED;
+            LOG_WARN("ls service should not be NULL", K(ret), KP(ls_service));
+          } else if (OB_FAIL(do_clean_related_info_in_ls_(ls_service, tenant_id))) {
+            LOG_WARN("fail to clean related info", K(ret), K(tenant_id));
+          }
+        }
+      }
+    }
+  }
+  return ret;
+}
+
+}
+}

src/storage/high_availability/ob_storage_ha_diagnose_service.h

@@ -0,0 +1,104 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEABASE_STORAGE_HA_DIAG_SERVICE_
+#define OCEABASE_STORAGE_HA_DIAG_SERVICE_
+
+#include "share/ob_storage_ha_diagnose_struct.h"
+#include "share/ob_storage_ha_diagnose_operator.h"
+
+namespace oceanbase
+{
+using namespace share;
+namespace storage
+{
+
+class ObStorageHADiagService : public lib::ThreadPool
+{
+public:
+  ObStorageHADiagService();
+  virtual ~ObStorageHADiagService() {}
+
+  int init(common::ObMySQLProxy *sql_proxy);
+  void run1() final;
+  void destroy();
+  void wakeup();
+  void stop();
+  void wait();
+  int start();
+
+public:
+  int add_task(const ObStorageHADiagTaskKey &key);
+  static ObStorageHADiagService &instance();
+
+private:
+  typedef ObArray<ObStorageHADiagTaskKey> TaskKeyArray;
+  typedef hash::ObHashMap<ObStorageHADiagTaskKey, int64_t, hash::NoPthreadDefendMode> TaskKeyMap;
+  static const int64_t ONCE_REPORT_KEY_MAX_NUM = 100;
+  static const int64_t REPORT_KEY_MAX_NUM = 1000;
+
+private:
+  int do_clean_history_(const ObStorageHADiagModule module, int64_t &end_timestamp);
+  int do_report_();
+  int get_info_from_type_(const ObStorageHADiagTaskKey &key, ObStorageHADiagInfo *&info,
+                          ObTransferErrorDiagInfo &transfer_err_diag,
+                          ObTransferPerfDiagInfo &transfer_perf_diag,
+                          ObIAllocator &alloc);
+  int insert_inner_table_(
+      ObMySQLTransaction &trans,
+      ObStorageHADiagMgr *mgr,
+      const ObStorageHADiagTaskKey &task_key,
+      ObStorageHADiagInfo &info);
+  int deep_copy_keys_(TaskKeyArray &do_report_keys) const;
+  int report_process_(const TaskKeyArray &task_keys);
+  int add_keys_(
+      const int64_t index,
+      const TaskKeyArray &task_keys,
+      TaskKeyArray &new_task_keys) const;
+  int clean_task_key_without_lock_(const ObStorageHADiagTaskKey &task_key);
+  int report_to_inner_table_(
+      ObMySQLTransaction &trans,
+      ObStorageHADiagMgr *mgr,
+      const ObStorageHADiagTaskKey &task_key);
+  int remove_oldest_without_lock_();
+  int get_oldest_key_without_lock_(ObStorageHADiagTaskKey &key) const;
+  int start_trans_(
+      ObTimeoutCtx &timeout_ctx,
+      ObMySQLTransaction &trans);
+  int commit_trans_(
+      const int32_t result,
+      ObMySQLTransaction &trans);
+  int report_batch_task_key_(ObMySQLTransaction &trans, const TaskKeyArray &task_keys);
+  int batch_del_task_in_mgr_(const TaskKeyArray &task_keys);
+  int get_ls_id_array_(ObLSService *ls_service, ObIArray<share::ObLSID> &ls_id_array);
+  int do_clean_related_info_(ObLSService *ls_service, const share::ObLSID &ls_id, const uint64_t tenant_id);
+  int scheduler_clean_related_info_(ObLSService *ls_service, const ObIArray<share::ObLSID> &ls_id_array, const uint64_t tenant_id);
+  int do_clean_related_info_in_ls_(ObLSService *ls_service, const uint64_t tenant_id);
+  int do_clean_transfer_related_info_();
+  int del_task_in_mgr_(const ObStorageHADiagTaskKey &task_key) const;
+
+private:
+  bool is_inited_;
+  common::ObThreadCond thread_cond_;
+  int64_t wakeup_cnt_;
+  ObStorageHADiagOperator op_;
+  common::ObMySQLProxy *sql_proxy_;
+  TaskKeyMap task_keys_;
+  SpinRWLock lock_;
+  int64_t err_diag_end_timestamp_;
+  int64_t perf_diag_end_timestamp_;
+  DISALLOW_COPY_AND_ASSIGN(ObStorageHADiagService);
+};
+
+}
+}
+#endif

src/storage/high_availability/ob_storage_ha_utils.cpp

@@ -34,7 +34,11 @@
 #include "rootserver/ob_tenant_info_loader.h"
 #include "src/observer/omt/ob_tenant_config.h"
 #include "common/errsim_module/ob_errsim_module_type.h"
+#include "ob_storage_ha_diagnose_mgr.h"
+#include "share/ob_storage_ha_diagnose_struct.h"
 #include "common/ob_role.h"
+#include "share/rc/ob_tenant_base.h"
+#include "observer/omt/ob_tenant.h"
 
 using namespace oceanbase::share;
 
@@ -702,5 +706,447 @@ int ObTransferUtils::check_ls_replay_scn(
   return ret;
 }
 
+void ObTransferUtils::add_transfer_error_diagnose_in_backfill(
+    const share::ObLSID &dest_ls_id,
+    const share::SCN &log_sync_scn,
+    const int result_code,
+    const common::ObTabletID &tablet_id,
+    const share::ObStorageHACostItemName result_msg)
+{
+  int ret = OB_SUCCESS;
+  ObTransferHandler *transfer_handler = nullptr;
+  ObLSHandle ls_handle;
+  if (!dest_ls_id.is_valid()
+      || !log_sync_scn.is_valid()
+      || OB_SUCCESS == result_code
+      || !tablet_id.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(dest_ls_id), K(result_code), K(tablet_id), K(log_sync_scn));
+  } else {
+    ObMigrationStatus migration_status = ObMigrationStatus::OB_MIGRATION_STATUS_MAX;
+    if (OB_FAIL(get_ls_migrate_status_(ls_handle, dest_ls_id, migration_status))) { //in migrate status
+      LOG_WARN("fail to get migration status", K(ret), K(dest_ls_id));
+    } else if (OB_FAIL(get_transfer_handler_(ls_handle, dest_ls_id, transfer_handler))) {
+      LOG_WARN("failed to get ls", K(ret), K(ls_handle), K(dest_ls_id));
+    } else if (OB_FAIL(transfer_handler->record_error_diagnose_info_in_backfill(
+          log_sync_scn, dest_ls_id, result_code, tablet_id, migration_status, result_msg))) {
+      LOG_WARN("failed to record diagnose info", K(ret), K(log_sync_scn),
+          K(dest_ls_id), K(result_code), K(tablet_id), K(migration_status), K(result_msg));
+    }
+  }
+}
+
+void ObTransferUtils::add_transfer_error_diagnose_in_replay(
+    const share::ObTransferTaskID &task_id,
+    const share::ObLSID &dest_ls_id,
+    const int result_code,
+    const bool clean_related_info,
+    const share::ObStorageHADiagTaskType type,
+    const share::ObStorageHACostItemName result_msg)
+{
+  int ret = OB_SUCCESS;
+  ObTransferHandler *transfer_handler = nullptr;
+  ObLSHandle ls_handle;
+  if (!task_id.is_valid()) {
+    // do nothing
+  } else if (!dest_ls_id.is_valid()
+      || type < ObStorageHADiagTaskType::TRANSFER_START
+      || type >= ObStorageHADiagTaskType::MAX_TYPE) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(dest_ls_id), K(type));
+  } else if (OB_FAIL(get_transfer_handler_(ls_handle, dest_ls_id, transfer_handler))) {
+    LOG_WARN("failed to get ls", K(ret), K(ls_handle), K(dest_ls_id));
+  } else if (OB_FAIL(transfer_handler->record_error_diagnose_info_in_replay(
+        task_id, dest_ls_id, result_code, clean_related_info, type, result_msg))) {
+    LOG_WARN("failed to record diagnose info", K(ret), K(dest_ls_id),
+        K(task_id), K(result_code), K(clean_related_info), K(type), K(result_msg));
+  }
+}
+
+void ObTransferUtils::set_transfer_related_info(
+    const share::ObLSID &dest_ls_id,
+    const share::ObTransferTaskID &task_id,
+    const share::SCN &start_scn)
+{
+  int ret = OB_SUCCESS;
+  ObTransferHandler *transfer_handler = nullptr;
+  ObLSHandle ls_handle;
+  if (!dest_ls_id.is_valid() || !task_id.is_valid() || !start_scn.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument!", K(ret), K(dest_ls_id), K(task_id), K(start_scn));
+  } else if (OB_FAIL(get_transfer_handler_(ls_handle, dest_ls_id, transfer_handler))) {
+    LOG_WARN("failed to get ls", K(ret), K(ls_handle), K(dest_ls_id));
+  } else if (OB_FAIL(transfer_handler->set_related_info(task_id, start_scn))) {
+    LOG_WARN("failed to set transfer related info", K(ret), K(dest_ls_id), K(task_id), K(start_scn));
+  }
+}
+
+int ObTransferUtils::get_ls_(
+    ObLSHandle &ls_handle,
+    const share::ObLSID &dest_ls_id,
+    ObLS *&ls)
+{
+  int ret = OB_SUCCESS;
+  ObLSService *ls_service = MTL(ObLSService*);
+  ls = nullptr;
+  if (!dest_ls_id.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument!", K(ret), K(dest_ls_id));
+  } else if (OB_FAIL(ls_service->get_ls(dest_ls_id, ls_handle, ObLSGetMod::MDS_TABLE_MOD))) {
+    LOG_WARN("failed to get ls", K(ret), K(ls_handle), K(dest_ls_id));
+  } else if (OB_ISNULL(ls = ls_handle.get_ls())) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("ls is null", K(ret), K(dest_ls_id), K(ls_handle));
+  }
+  return ret;
+}
+
+int ObTransferUtils::get_ls_migrate_status_(
+    ObLSHandle &ls_handle,
+    const share::ObLSID &dest_ls_id,
+    ObMigrationStatus &migration_status)
+{
+  int ret = OB_SUCCESS;
+  migration_status = ObMigrationStatus::OB_MIGRATION_STATUS_MAX;
+  ObLS *ls = nullptr;
+  if (!dest_ls_id.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument!", K(ret), K(dest_ls_id));
+  } else if (OB_FAIL(get_ls_(ls_handle, dest_ls_id, ls))) {
+    LOG_WARN("failed to get ls", K(ret), K(ls_handle), K(dest_ls_id));
+  } else if (OB_FAIL(ls->get_migration_status(migration_status))) {
+    LOG_WARN("fail to get migration status", K(ret), K(dest_ls_id));
+  }
+
+  return ret;
+}
+
+int ObTransferUtils::get_transfer_handler_(
+    ObLSHandle &ls_handle,
+    const share::ObLSID &dest_ls_id,
+    ObTransferHandler *&transfer_handler)
+{
+  int ret = OB_SUCCESS;
+  ObLS *ls = nullptr;
+  transfer_handler = nullptr;
+  if (!dest_ls_id.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument!", K(ret), K(dest_ls_id));
+  } else if (OB_FAIL(get_ls_(ls_handle, dest_ls_id, ls))) {
+    LOG_WARN("failed to get ls", K(ret), K(ls_handle), K(dest_ls_id));
+  } else {
+    transfer_handler = ls->get_transfer_handler();
+  }
+  return ret;
+}
+
+void ObTransferUtils::reset_related_info(const share::ObLSID &dest_ls_id)
+{
+  int ret = OB_SUCCESS;
+  ObTransferHandler *transfer_handler = nullptr;
+  ObLSHandle ls_handle;
+  if (!dest_ls_id.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument!", K(ret), K(dest_ls_id));
+  } else if (OB_FAIL(get_transfer_handler_(ls_handle, dest_ls_id, transfer_handler))) {
+    LOG_WARN("failed to get ls", K(ls_handle), K(ret), K(dest_ls_id));
+  } else {
+    transfer_handler->reset_related_info();
+  }
+}
+
+void ObTransferUtils::add_transfer_perf_diagnose_in_replay_(
+    const share::ObStorageHAPerfDiagParams &params,
+    const int result,
+    const uint64_t timestamp,
+    const int64_t start_ts,
+    const bool is_report)
+{
+  int ret = OB_SUCCESS;
+  ObTransferHandler *transfer_handler = nullptr;
+  ObLSHandle ls_handle;
+  if (!params.is_valid()
+      || start_ts < 0) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(params), K(start_ts));
+  } else if (OB_FAIL(get_transfer_handler_(ls_handle, params.dest_ls_id_, transfer_handler))) {
+    LOG_WARN("failed to get ls", K(ret), K(ls_handle), K(params.dest_ls_id_));
+  } else if (OB_FAIL(transfer_handler->record_perf_diagnose_info_in_replay(
+        params, result, timestamp, start_ts, is_report))) {
+    LOG_WARN("failed to record perf diagnose info",
+        K(ret), K(params), K(result), K(timestamp), K(start_ts), K(is_report));
+  }
+}
+
+void ObTransferUtils::add_transfer_perf_diagnose_in_backfill(
+    const share::ObStorageHAPerfDiagParams &params,
+    const share::SCN &log_sync_scn,
+    const int result_code,
+    const uint64_t timestamp,
+    const int64_t start_ts,
+    const bool is_report)
+{
+  int ret = OB_SUCCESS;
+  ObTransferHandler *transfer_handler = nullptr;
+  ObLSHandle ls_handle;
+  if (!params.is_valid()
+      || !log_sync_scn.is_valid()
+      || start_ts < 0) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(params), K(log_sync_scn), K(start_ts));
+  } else {
+    ObMigrationStatus migration_status = ObMigrationStatus::OB_MIGRATION_STATUS_MAX;
+    if (OB_FAIL(get_ls_migrate_status_(ls_handle, params.dest_ls_id_, migration_status))) { //in migrate status
+      LOG_WARN("fail to get migration status", K(ret), K(params.dest_ls_id_));
+    } else if (OB_FAIL(get_transfer_handler_(ls_handle, params.dest_ls_id_, transfer_handler))) {
+      LOG_WARN("failed to get ls", K(ret), K(ls_handle), K(params.dest_ls_id_));
+    } else if (OB_FAIL(transfer_handler->record_perf_diagnose_info_in_backfill(
+        params, log_sync_scn, result_code, migration_status, timestamp, start_ts, is_report))) {
+      LOG_WARN("failed to record perf diagnose info",
+          K(ret), K(params), K(result_code), K(timestamp), K(start_ts), K(is_report));
+    }
+  }
+}
+
+void ObTransferUtils::construct_perf_diag_backfill_params_(
+    const share::ObLSID &dest_ls_id,
+    const common::ObTabletID &tablet_id,
+    const share::ObStorageHACostItemType item_type,
+    const share::ObStorageHACostItemName item_name,
+    share::ObStorageHAPerfDiagParams &params)
+{
+  int ret = OB_SUCCESS;
+  if (!dest_ls_id.is_valid()
+      || !tablet_id.is_valid()
+      || item_type >= ObStorageHACostItemType::MAX_TYPE
+      || item_type < ObStorageHACostItemType::ACCUM_COST_TYPE
+      || item_name >= ObStorageHACostItemName::MAX_NAME
+      || item_name < ObStorageHACostItemName::TRANSFER_START_BEGIN) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(dest_ls_id), K(tablet_id), K(item_type), K(item_name));
+  } else {
+    params.dest_ls_id_ = dest_ls_id;
+    params.task_type_ = ObStorageHADiagTaskType::TRANSFER_BACKFILLED;
+    params.item_type_ = item_type;
+    params.name_ = item_name;
+    params.tablet_id_ = tablet_id;
+    params.tablet_count_ = 1;
+  }
+}
+
+void ObTransferUtils::construct_perf_diag_replay_params_(
+    const share::ObLSID &dest_ls_id,
+    const share::ObTransferTaskID &task_id,
+    const share::ObStorageHADiagTaskType task_type,
+    const share::ObStorageHACostItemType item_type,
+    const share::ObStorageHACostItemName item_name,
+    const int64_t tablet_count,
+    share::ObStorageHAPerfDiagParams &params)
+{
+  int ret = OB_SUCCESS;
+  if (!dest_ls_id.is_valid()
+      || !task_id.is_valid()
+      || task_type >= ObStorageHADiagTaskType::MAX_TYPE
+      || task_type < ObStorageHADiagTaskType::TRANSFER_START
+      || item_type >= ObStorageHACostItemType::MAX_TYPE
+      || item_type < ObStorageHACostItemType::ACCUM_COST_TYPE
+      || item_name >= ObStorageHACostItemName::MAX_NAME
+      || item_name < ObStorageHACostItemName::TRANSFER_START_BEGIN
+      || tablet_count < 0) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(task_id), K(dest_ls_id), K(task_type), K(item_type), K(item_name), K(tablet_count));
+  } else {
+    params.dest_ls_id_ = dest_ls_id;
+    params.task_id_ = task_id;
+    params.task_type_ = task_type;
+    params.item_type_ = item_type;
+    params.name_ = item_name;
+    params.tablet_count_ = tablet_count;
+  }
+}
+
+void ObTransferUtils::process_backfill_perf_diag_info(
+    const share::ObLSID &dest_ls_id,
+    const common::ObTabletID &tablet_id,
+    const share::ObStorageHACostItemType item_type,
+    const share::ObStorageHACostItemName name,
+    share::ObStorageHAPerfDiagParams &params)
+{
+  int ret = OB_SUCCESS;
+  params.reset();
+  if (!tablet_id.is_valid()
+      || !dest_ls_id.is_valid()
+      || item_type >= ObStorageHACostItemType::MAX_TYPE
+      || item_type < ObStorageHACostItemType::ACCUM_COST_TYPE
+      || name >= ObStorageHACostItemName::MAX_NAME
+      || name < ObStorageHACostItemName::TRANSFER_START_BEGIN) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(dest_ls_id), K(name), K(item_type), K(tablet_id));
+  } else  {
+    construct_perf_diag_backfill_params_(
+                                  dest_ls_id,
+                                  tablet_id,
+                                  item_type,
+                                  name,
+                                  params);
+  }
+}
+
+void ObTransferUtils::process_start_out_perf_diag_info(
+    const ObTXStartTransferOutInfo &tx_start_transfer_out_info,
+    const share::ObStorageHACostItemType item_type,
+    const share::ObStorageHACostItemName name,
+    const int result,
+    const uint64_t timestamp,
+    const int64_t start_ts,
+    const bool is_report)
+{
+  int ret = OB_SUCCESS;
+  if (!tx_start_transfer_out_info.task_id_.is_valid()) {
+    // do nothing
+  } else if (!tx_start_transfer_out_info.is_valid()
+      || item_type >= ObStorageHACostItemType::MAX_TYPE
+      || item_type < ObStorageHACostItemType::ACCUM_COST_TYPE
+      || name >= ObStorageHACostItemName::MAX_NAME
+      || name < ObStorageHACostItemName::TRANSFER_START_BEGIN
+      || start_ts < 0) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(tx_start_transfer_out_info), K(name), K(item_type), K(start_ts));
+  } else {
+    share::ObStorageHAPerfDiagParams params;
+    construct_perf_diag_replay_params_(
+                                    tx_start_transfer_out_info.dest_ls_id_,
+                                    tx_start_transfer_out_info.task_id_,
+                                    ObStorageHADiagTaskType::TRANSFER_START_OUT,
+                                    item_type,
+                                    name,
+                                    tx_start_transfer_out_info.tablet_list_.count(),
+                                    params);
+    add_transfer_perf_diagnose_in_replay_(
+                                    params,
+                                    result,
+                                    timestamp,
+                                    start_ts,
+                                    is_report);
+  }
+}
+
+void ObTransferUtils::process_start_in_perf_diag_info(
+    const ObTXStartTransferInInfo &tx_start_transfer_in_info,
+    const share::ObStorageHACostItemType item_type,
+    const share::ObStorageHACostItemName name,
+    const int result,
+    const uint64_t timestamp,
+    const int64_t start_ts,
+    const bool is_report)
+{
+  int ret = OB_SUCCESS;
+  if (!tx_start_transfer_in_info.task_id_.is_valid()) {
+    // do nothing
+  } else if (!tx_start_transfer_in_info.is_valid()
+      || item_type >= ObStorageHACostItemType::MAX_TYPE
+      || item_type < ObStorageHACostItemType::ACCUM_COST_TYPE
+      || name >= ObStorageHACostItemName::MAX_NAME
+      || name < ObStorageHACostItemName::TRANSFER_START_BEGIN
+      || start_ts < 0) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(tx_start_transfer_in_info), K(name), K(item_type), K(start_ts));
+  } else {
+    share::ObStorageHAPerfDiagParams params;
+    construct_perf_diag_replay_params_(
+                                    tx_start_transfer_in_info.dest_ls_id_,
+                                    tx_start_transfer_in_info.task_id_,
+                                    ObStorageHADiagTaskType::TRANSFER_START_IN,
+                                    item_type,
+                                    name,
+                                    tx_start_transfer_in_info.tablet_meta_list_.count(),
+                                    params);
+    add_transfer_perf_diagnose_in_replay_(
+                                    params,
+                                    result,
+                                    timestamp,
+                                    start_ts,
+                                    is_report);
+  }
+}
+
+void ObTransferUtils::process_finish_in_perf_diag_info(
+    const ObTXFinishTransferInInfo &tx_finish_transfer_in_info,
+    const share::ObStorageHACostItemType item_type,
+    const share::ObStorageHACostItemName name,
+    const int result,
+    const uint64_t timestamp,
+    const int64_t start_ts,
+    const bool is_report)
+{
+  int ret = OB_SUCCESS;
+  if (!tx_finish_transfer_in_info.task_id_.is_valid()) {
+    // do nothing
+  } else if (!tx_finish_transfer_in_info.is_valid()
+      || item_type >= ObStorageHACostItemType::MAX_TYPE
+      || item_type < ObStorageHACostItemType::ACCUM_COST_TYPE
+      || name >= ObStorageHACostItemName::MAX_NAME
+      || name < ObStorageHACostItemName::TRANSFER_START_BEGIN
+      || start_ts < 0) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(tx_finish_transfer_in_info), K(name), K(item_type), K(start_ts));
+  } else {
+    share::ObStorageHAPerfDiagParams params;
+    construct_perf_diag_replay_params_(
+                                    tx_finish_transfer_in_info.dest_ls_id_,
+                                    tx_finish_transfer_in_info.task_id_,
+                                    ObStorageHADiagTaskType::TRANSFER_FINISH_IN,
+                                    item_type,
+                                    name,
+                                    tx_finish_transfer_in_info.tablet_list_.count(),
+                                    params);
+    add_transfer_perf_diagnose_in_replay_(
+                                    params,
+                                    result,
+                                    timestamp,
+                                    start_ts,
+                                    is_report);
+  }
+}
+
+void ObTransferUtils::process_finish_out_perf_diag_info(
+    const ObTXFinishTransferOutInfo &tx_finish_transfer_out_info,
+    const share::ObStorageHACostItemType item_type,
+    const share::ObStorageHACostItemName name,
+    const int result,
+    const uint64_t timestamp,
+    const int64_t start_ts,
+    const bool is_report)
+{
+  int ret = OB_SUCCESS;
+  if (!tx_finish_transfer_out_info.task_id_.is_valid()) {
+    // do nothing
+  } else if (!tx_finish_transfer_out_info.is_valid()
+      || item_type >= ObStorageHACostItemType::MAX_TYPE
+      || item_type < ObStorageHACostItemType::ACCUM_COST_TYPE
+      || name >= ObStorageHACostItemName::MAX_NAME
+      || name < ObStorageHACostItemName::TRANSFER_START_BEGIN
+      || start_ts < 0) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(tx_finish_transfer_out_info), K(name), K(item_type), K(start_ts));
+  } else {
+    share::ObStorageHAPerfDiagParams params;
+    construct_perf_diag_replay_params_(
+                                    tx_finish_transfer_out_info.dest_ls_id_,
+                                    tx_finish_transfer_out_info.task_id_,
+                                    ObStorageHADiagTaskType::TRANSFER_FINISH_OUT,
+                                    item_type,
+                                    name,
+                                    tx_finish_transfer_out_info.tablet_list_.count(),
+                                    params);
+    add_transfer_perf_diagnose_in_replay_(
+                                    params,
+                                    result,
+                                    timestamp,
+                                    start_ts,
+                                    is_report);
+  }
+}
+
 } // end namespace storage
 } // end namespace oceanbase

src/storage/high_availability/ob_storage_ha_utils.h

@@ -14,6 +14,8 @@
 #include "common/ob_tablet_id.h"
 #include "lib/mysqlclient/ob_isql_client.h"
 #include "ob_storage_ha_struct.h"
+#include "share/ob_storage_ha_diagnose_struct.h"
+#include "ob_transfer_struct.h"
 
 namespace oceanbase
 {
@@ -23,7 +25,8 @@ class SCN;
 }
 namespace storage
 {
-
+class ObBackfillTXCtx;
+class ObTransferHandler;
 class ObStorageHAUtils
 {
 public:
@@ -65,6 +68,7 @@ private:
 
 struct ObTransferUtils
 {
+public:
   static bool is_need_retry_error(const int err);
   static int block_tx(const uint64_t tenant_id, const share::ObLSID &ls_id, const share::SCN &gts_scn);
   static int kill_tx(const uint64_t tenant_id, const share::ObLSID &ls_id, const share::SCN &gts_scn);
@@ -84,6 +88,107 @@ struct ObTransferUtils
       const common::ObIArray<ObAddr> &member_addr_list,
       ObTimeoutCtx &timeout_ctx,
       common::ObIArray<ObAddr> &finished_addr_list);
+  static void add_transfer_error_diagnose_in_replay(
+      const share::ObTransferTaskID &task_id,
+      const share::ObLSID &dest_ls_id,
+      const int result_code,
+      const bool clean_related_info,
+      const share::ObStorageHADiagTaskType type,
+      const share::ObStorageHACostItemName result_msg);
+  static void add_transfer_error_diagnose_in_backfill(
+      const share::ObLSID &dest_ls_id,
+      const share::SCN &log_sync_scn,
+      const int result_code,
+      const common::ObTabletID &tablet_id,
+      const share::ObStorageHACostItemName result_msg);
+  static void set_transfer_related_info(
+      const share::ObLSID &dest_ls_id,
+      const share::ObTransferTaskID &task_id,
+      const share::SCN &start_scn);
+  static void reset_related_info(const share::ObLSID &dest_ls_id);
+
+  static void add_transfer_perf_diagnose_in_backfill(
+              const share::ObStorageHAPerfDiagParams &params,
+              const share::SCN &log_sync_scn,
+              const int result_code,
+              const uint64_t timestamp,
+              const int64_t start_ts,
+              const bool is_report);
+
+  static void process_backfill_perf_diag_info(
+              const share::ObLSID &dest_ls_id,
+              const common::ObTabletID &tablet_id,
+              const share::ObStorageHACostItemType item_type,
+              const share::ObStorageHACostItemName name,
+              share::ObStorageHAPerfDiagParams &params);
+
+  static void process_start_out_perf_diag_info(
+              const ObTXStartTransferOutInfo &tx_start_transfer_out_info,
+              const share::ObStorageHACostItemType item_type,
+              const share::ObStorageHACostItemName name,
+              const int result,
+              const uint64_t timestamp,
+              const int64_t start_ts,
+              const bool is_report);
+  static void process_start_in_perf_diag_info(
+              const ObTXStartTransferInInfo &tx_start_transfer_in_info,
+              const share::ObStorageHACostItemType item_type,
+              const share::ObStorageHACostItemName name,
+              const int result,
+              const uint64_t timestamp,
+              const int64_t start_ts,
+              const bool is_report);
+  static void process_finish_out_perf_diag_info(
+              const ObTXFinishTransferOutInfo &tx_finish_transfer_out_info,
+              const share::ObStorageHACostItemType item_type,
+              const share::ObStorageHACostItemName name,
+              const int result,
+              const uint64_t timestamp,
+              const int64_t start_ts,
+              const bool is_report);
+  static void process_finish_in_perf_diag_info(
+              const ObTXFinishTransferInInfo &tx_finish_transfer_in_info,
+              const share::ObStorageHACostItemType item_type,
+              const share::ObStorageHACostItemName name,
+              const int result,
+              const uint64_t timestamp,
+              const int64_t start_ts,
+              const bool is_report);
+private:
+  static int get_ls_(
+      ObLSHandle &ls_handle,
+      const share::ObLSID &dest_ls_id,
+      ObLS *&ls);
+  static int get_transfer_handler_(
+      ObLSHandle &ls_handle,
+      const share::ObLSID &dest_ls_id,
+      ObTransferHandler *&transfer_handler);
+  static int get_ls_migrate_status_(
+      ObLSHandle &ls_handle,
+      const share::ObLSID &dest_ls_id,
+      ObMigrationStatus &migration_status);
+
+  static void construct_perf_diag_replay_params_(
+              const share::ObLSID &dest_ls_id_,
+              const share::ObTransferTaskID &task_id_,
+              const share::ObStorageHADiagTaskType task_type_,
+              const share::ObStorageHACostItemType item_type_,
+              const share::ObStorageHACostItemName item_name,
+              const int64_t tablet_count,
+              share::ObStorageHAPerfDiagParams &params);
+
+  static void add_transfer_perf_diagnose_in_replay_(
+              const share::ObStorageHAPerfDiagParams &params,
+              const int result,
+              const uint64_t timestamp,
+              const int64_t start_ts,
+              const bool is_report);
+  static void construct_perf_diag_backfill_params_(
+              const share::ObLSID &dest_ls_id,
+              const common::ObTabletID &tablet_id,
+              const share::ObStorageHACostItemType item_type,
+              const share::ObStorageHACostItemName item_name,
+              share::ObStorageHAPerfDiagParams &params);
 };
 
 } // end namespace storage

src/storage/high_availability/ob_tablet_backfill_tx.cpp

@@ -18,6 +18,8 @@
 #include "storage/ob_storage_struct.h"
 #include "storage/tablet/ob_tablet_iterator.h"
 #include "storage/tablet/ob_tablet.h"
+#include "storage/high_availability/ob_storage_ha_diagnose_mgr.h"
+#include "storage/high_availability/ob_storage_ha_utils.h"
 #include "storage/compaction/ob_partition_merger.h"
 
 namespace oceanbase
@@ -442,7 +444,10 @@ int ObTabletBackfillTXTask::process()
 {
   int ret = OB_SUCCESS;
   LOG_INFO("start to do tablet backfill tx task", KPC(ha_dag_net_ctx_), K(tablet_info_), K(ls_id_));
-
+  const int64_t start_ts = ObTimeUtility::current_time();
+  share::ObStorageHACostItemName diagnose_result_msg = share::ObStorageHACostItemName::TRANSFER_BACKFILL_START;
+  process_transfer_perf_diagnose_(start_ts, start_ts, false/*is_report*/,
+      share::ObStorageHACostItemName::TRANSFER_BACKFILL_START, ret);
   if (!is_inited_) {
     ret = OB_NOT_INIT;
     LOG_WARN("tablet backfill tx task task do not init", K(ret));
@@ -455,12 +460,19 @@ int ObTabletBackfillTXTask::process()
   if (OB_FAIL(ret)) {
     int tmp_ret = OB_SUCCESS;
     ObTabletBackfillTXDag *tablet_backfill_tx_dag = static_cast<ObTabletBackfillTXDag *>(this->get_dag());
+    share::ObLSID dest_ls_id;
+    share::SCN log_sync_scn;
     if (OB_ISNULL(tablet_backfill_tx_dag)) {
       tmp_ret = OB_ERR_UNEXPECTED;
       LOG_ERROR("tablet backfill tx dag should not be NULL", K(tmp_ret), KP(tablet_backfill_tx_dag));
     } else if (OB_SUCCESS != (tmp_ret = tablet_backfill_tx_dag->set_result(ret))) {
       LOG_WARN("failed to set result", K(ret), KPC(ha_dag_net_ctx_), K(ls_id_), K(tablet_info_));
     }
+    if (OB_TMP_FAIL(get_diagnose_support_info_(dest_ls_id, log_sync_scn))) {
+      LOG_WARN("failed to get diagnose support info", K(tmp_ret));
+    } else {
+      ObTransferUtils::add_transfer_error_diagnose_in_backfill(dest_ls_id, log_sync_scn, ret, tablet_info_.tablet_id_, diagnose_result_msg);
+    }
   }
   return ret;
 }
@@ -692,6 +704,7 @@ int ObTabletBackfillTXTask::generate_table_backfill_tx_task_(
       ObSSTableMetaHandle sst_meta_hdl;
       ObTabletTableBackfillTXTask *table_backfill_tx_task = nullptr;
       bool is_add_task = false;
+
       if (OB_ISNULL(table)) {
         ret = OB_ERR_UNEXPECTED;
         LOG_WARN("table should not be NULL or table type is unexpected", K(ret), KPC(table));
@@ -740,6 +753,41 @@ int ObTabletBackfillTXTask::generate_table_backfill_tx_task_(
   return ret;
 }
 
+int ObTabletBackfillTXTask::get_diagnose_support_info_(share::ObLSID &dest_ls_id, share::SCN &log_sync_scn) const
+{
+  int ret = OB_SUCCESS;
+  dest_ls_id.reset();
+  log_sync_scn.reset();
+  if (ObIHADagNetCtx::TRANSFER_BACKFILL_TX != ha_dag_net_ctx_->get_dag_net_ctx_type()) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("unexpected type", K(ret), "ctx type", ha_dag_net_ctx_->get_dag_net_ctx_type());
+  } else {
+    dest_ls_id = static_cast<const ObTransferBackfillTXCtx *>(ha_dag_net_ctx_)->get_ls_id();
+    log_sync_scn = backfill_tx_ctx_->log_sync_scn_;
+  }
+  return ret;
+}
+
+void ObTabletBackfillTXTask::process_transfer_perf_diagnose_(
+    const int64_t timestamp,
+    const int64_t start_ts,
+    const bool is_report,
+    const ObStorageHACostItemName name,
+    const int result) const
+{
+  int ret = OB_SUCCESS;
+  share::ObLSID dest_ls_id;
+  share::SCN log_sync_scn;
+  if (OB_FAIL(get_diagnose_support_info_(dest_ls_id, log_sync_scn))) {
+    LOG_WARN("failed to get diagnose support info", K(ret));
+  } else {
+    ObStorageHAPerfDiagParams params;
+    ObTransferUtils::process_backfill_perf_diag_info(dest_ls_id, tablet_info_.tablet_id_,
+        ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, name, params);
+    ObTransferUtils::add_transfer_perf_diagnose_in_backfill(params, log_sync_scn, result, timestamp, start_ts, is_report);
+  }
+}
+
 /******************ObTabletTableBackfillTXTask*********************/
 ObTabletTableBackfillTXTask::ObTabletTableBackfillTXTask()
   : ObITask(TASK_TYPE_MIGRATE_PREPARE),
@@ -802,7 +850,10 @@ int ObTabletTableBackfillTXTask::process()
   int ret = OB_SUCCESS;
   LOG_INFO("start do tablet table backfill tx task", K(ls_id_), K(tablet_id_), K(table_handle_));
   bool need_merge = true;
-
+  const int64_t start_ts = ObTimeUtility::current_time();
+  share::ObStorageHACostItemName diagnose_result_msg = share::ObStorageHACostItemName::MAX_NAME;
+  process_transfer_perf_diagnose_(start_ts, start_ts, false/*is_report*/,
+      ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::TRANSFER_BACKFILLED_TABLE_BEGIN, ret);
   if (!is_inited_) {
     ret = OB_NOT_INIT;
     LOG_WARN("tablet table backfill tx task do not init", K(ret));
@@ -813,11 +864,13 @@ int ObTabletTableBackfillTXTask::process()
   } else if (!need_merge) {
     LOG_INFO("tablet table no need merge", K(ret), K_(tablet_handle));
   } else if (OB_FAIL(prepare_merge_ctx_())) {
+    diagnose_result_msg = share::ObStorageHACostItemName::PREPARE_MERGE_CTX;
     LOG_WARN("failed to prepare merge ctx", K(ret), KPC(this));
   } else if (OB_FAIL(do_backfill_tx_())) {
     if (OB_NO_NEED_MERGE == ret) {
       ret = OB_SUCCESS;
     } else {
+      diagnose_result_msg = share::ObStorageHACostItemName::DO_BACKFILL_TX;
       LOG_WARN("failed to do backfill tx", K(ret), KPC(this));
     }
   }
@@ -832,6 +885,30 @@ int ObTabletTableBackfillTXTask::process()
       LOG_WARN("failed to set result", K(ret), KPC(ha_dag_net_ctx_), K(ls_id_), K(tablet_id_));
     }
   }
+#ifdef ERRSIM
+  if (OB_SUCC(ret)) {
+    ret = EN_TRANSFER_DIAGNOSE_BACKFILL_FAILED ? : OB_SUCCESS;
+    if (OB_FAIL(ret)) {
+      STORAGE_LOG(WARN, "fake EN_TRANSFER_DIAGNOSE_BACKFILL_FAILED", K(ret));
+    }
+  }
+#endif
+  if (OB_FAIL(ret)) {
+    int tmp_ret = OB_SUCCESS;
+    share::ObLSID dest_ls_id;
+    share::SCN log_sync_scn;
+    if (OB_TMP_FAIL(get_diagnose_support_info_(dest_ls_id, log_sync_scn))) {
+      LOG_WARN("failed to get diagnose support info", K(tmp_ret));
+    } else {
+      ObTransferUtils::add_transfer_error_diagnose_in_backfill(dest_ls_id, log_sync_scn, ret, tablet_id_, diagnose_result_msg);
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    const int64_t end_ts = ObTimeUtility::current_time();
+    process_transfer_perf_diagnose_(end_ts, start_ts, false/*is_report*/,
+        ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::TX_BACKFILL, ret);
+  }
   return ret;
 }
 
@@ -851,7 +928,7 @@ int ObTabletTableBackfillTXTask::prepare_merge_ctx_()
   } else {
     // init tablet merge dag param
     static_param.dag_param_.ls_id_ = ls_id_;
-    static_param.dag_param_.merge_type_ = table_handle_.get_table()->is_memtable() ? ObMergeType::MINI_MERGE : ObMergeType::BACKFILL_TX_MERGE;
+    static_param.dag_param_.merge_type_ = table_handle_.get_table()->is_memtable() ? compaction::ObMergeType::MINI_MERGE : compaction::ObMergeType::BACKFILL_TX_MERGE;
     static_param.report_ = nullptr;
     static_param.dag_param_.tablet_id_ = tablet_id_;
     // init version range and sstable
@@ -984,6 +1061,42 @@ int ObTabletTableBackfillTXTask::update_merge_sstable_()
   return ret;
 }
 
+int ObTabletTableBackfillTXTask::get_diagnose_support_info_(share::ObLSID &dest_ls_id, share::SCN &log_sync_scn) const
+{
+  int ret = OB_SUCCESS;
+  dest_ls_id.reset();
+  log_sync_scn.reset();
+  if (ObIHADagNetCtx::TRANSFER_BACKFILL_TX != ha_dag_net_ctx_->get_dag_net_ctx_type()) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("unexpected type", K(ret), "ctx type", ha_dag_net_ctx_->get_dag_net_ctx_type());
+  } else {
+    dest_ls_id = static_cast<const ObTransferBackfillTXCtx *>(ha_dag_net_ctx_)->get_ls_id();
+    log_sync_scn = backfill_tx_ctx_->log_sync_scn_;
+  }
+  return ret;
+}
+
+void ObTabletTableBackfillTXTask::process_transfer_perf_diagnose_(
+    const int64_t timestamp,
+    const int64_t start_ts,
+    const bool is_report,
+    const ObStorageHACostItemType type,
+    const ObStorageHACostItemName name,
+    const int result) const
+{
+  int ret = OB_SUCCESS;
+  share::ObLSID dest_ls_id;
+  share::SCN log_sync_scn;
+  if (OB_FAIL(get_diagnose_support_info_(dest_ls_id, log_sync_scn))) {
+    LOG_WARN("failed to get diagnose support info", K(ret));
+  } else {
+    ObStorageHAPerfDiagParams params;
+    ObTransferUtils::process_backfill_perf_diag_info(dest_ls_id, tablet_id_,
+        type, name, params);
+    ObTransferUtils::add_transfer_perf_diagnose_in_backfill(params, log_sync_scn, result, timestamp, start_ts, is_report);
+  }
+}
+
 /******************ObFinishTabletBackfillTXTask*********************/
 ObFinishTabletBackfillTXTask::ObFinishTabletBackfillTXTask()
   : ObITask(TASK_TYPE_MIGRATE_PREPARE),

src/storage/high_availability/ob_tablet_backfill_tx.h

@@ -121,6 +121,13 @@ private:
   int get_all_backfill_tx_tables_(
       ObTablet *tablet,
       common::ObIArray<ObTableHandleV2> &table_array);
+  int get_diagnose_support_info_(share::ObLSID &dest_ls_id, share::SCN &log_sync_scn) const;
+  void process_transfer_perf_diagnose_(
+      const int64_t timestamp,
+      const int64_t start_ts,
+      const bool is_report,
+      const ObStorageHACostItemName name,
+      const int result) const;
 private:
   bool is_inited_;
   ObBackfillTXCtx *backfill_tx_ctx_;
@@ -148,6 +155,14 @@ private:
   int do_backfill_tx_();
   int prepare_partition_merge_();
   int update_merge_sstable_();
+  int get_diagnose_support_info_(share::ObLSID &dest_ls_id, share::SCN &log_sync_scn) const;
+  void process_transfer_perf_diagnose_(
+      const int64_t timestamp,
+      const int64_t start_ts,
+      const bool is_report,
+      const ObStorageHACostItemType type,
+      const ObStorageHACostItemName name,
+      const int result) const;
 
 private:
   bool is_inited_;

src/storage/high_availability/ob_transfer_backfill_tx.cpp

@@ -24,6 +24,7 @@
 #include "share/ob_debug_sync_point.h"
 #include "lib/utility/ob_tracepoint.h"
 #include "storage/tablet/ob_tablet.h"
+#include "storage/high_availability/ob_storage_ha_utils.h"
 #include "storage/high_availability/ob_transfer_handler.h"
 
 namespace oceanbase
@@ -1692,6 +1693,7 @@ int ObTransferReplaceTableTask::do_replace_logical_tables_(ObLS *ls)
   ObTablet *tablet = nullptr;
   ObTabletCreateDeleteMdsUserData user_data;
   DEBUG_SYNC(TRANSFER_REPLACE_TABLE_BEFORE);
+  share::ObStorageHACostItemName diagnose_result_msg = share::ObStorageHACostItemName::TRANSFER_REPLACE_BEGIN;
   if (OB_ISNULL(ls)) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("ls is nullptr", K(ret));
@@ -1701,6 +1703,9 @@ int ObTransferReplaceTableTask::do_replace_logical_tables_(ObLS *ls)
       const ObTabletBackfillInfo tablet_info = ctx_->tablet_infos_.at(i);
       bool in_migration = false;
       ObMigrationStatus migration_status = ObMigrationStatus::OB_MIGRATION_STATUS_MAX;
+      const int64_t start_ts = ObTimeUtility::current_time();
+      process_transfer_perf_diagnose_(start_ts, start_ts, false/*is_report*/, ls->get_ls_id(),
+          ctx_->backfill_scn_, tablet_info.tablet_id_, share::ObStorageHACostItemName::TRANSFER_REPLACE_BEGIN);
       if (ctx_->is_failed()) {
         int tmp_ret = OB_SUCCESS;
         if (OB_SUCCESS != (tmp_ret = ctx_->get_result(ret))) {
@@ -1739,6 +1744,12 @@ int ObTransferReplaceTableTask::do_replace_logical_tables_(ObLS *ls)
                          "dest_ls_id", ctx_->dest_ls_id_.id(),
                          "tablet_id", tablet_info.tablet_id_.id());
 #endif
+        const int64_t end_ts = ObTimeUtility::current_time();
+        process_transfer_perf_diagnose_(end_ts, start_ts, true/*is_report*/, ls->get_ls_id(),
+            ctx_->backfill_scn_, tablet_info.tablet_id_, share::ObStorageHACostItemName::TRANSFER_REPLACE_END);
+      }
+      if (OB_FAIL(ret)) {
+        ObTransferUtils::add_transfer_error_diagnose_in_backfill(ls->get_ls_id(), ctx_->backfill_scn_, ret, tablet_info.tablet_id_, diagnose_result_msg);
       }
     }
   }
@@ -1820,5 +1831,21 @@ int ObTransferReplaceTableTask::build_migration_param_(
   return ret;
 }
 
+void ObTransferReplaceTableTask::process_transfer_perf_diagnose_(
+    const int64_t timestamp,
+    const int64_t start_ts,
+    const bool is_report,
+    const share::ObLSID &dest_ls_id,
+    const share::SCN &log_sync_scn,
+    const common::ObTabletID &tablet_id,
+    const share::ObStorageHACostItemName name) const
+{
+  int ret = OB_SUCCESS;
+  ObStorageHAPerfDiagParams params;
+  ObTransferUtils::process_backfill_perf_diag_info(dest_ls_id, tablet_id,
+      ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, name, params);
+  ObTransferUtils::add_transfer_perf_diagnose_in_backfill(params, log_sync_scn, ret, timestamp, start_ts, is_report);
+}
+
 }
 }

src/storage/high_availability/ob_transfer_backfill_tx.h

@@ -16,6 +16,7 @@
 #include "ob_storage_ha_struct.h"
 #include "ob_storage_ha_dag.h"
 #include "share/scn.h"
+#include "share/ob_storage_ha_diagnose_struct.h"
 
 namespace oceanbase
 {
@@ -86,6 +87,7 @@ public:
   virtual int fill_comment(char *buf, const int64_t buf_len) const override;
   virtual bool is_valid() const;
   virtual DagNetCtxType get_dag_net_ctx_type() { return ObIHADagNetCtx::TRANSFER_BACKFILL_TX; }
+  const share::ObLSID &get_ls_id() const { return dest_ls_id_; }
 public:
   uint64_t tenant_id_;
 #ifdef ERRSIM
@@ -250,6 +252,14 @@ private:
   int check_major_sstable_(
       const ObTablet *tablet,
       const ObTabletMemberWrapper<ObTabletTableStore> &table_store_wrapper);
+  void process_transfer_perf_diagnose_(
+      const int64_t timestamp,
+      const int64_t start_ts,
+      const bool is_report,
+      const share::ObLSID &dest_ls_id,
+      const share::SCN &log_sync_scn,
+      const common::ObTabletID &tablet_id,
+      const share::ObStorageHACostItemName name) const;
 private:
   bool is_inited_;
   ObTransferBackfillTXCtx *ctx_;

src/storage/high_availability/ob_transfer_handler.cpp

@@ -28,6 +28,8 @@
 #include "storage/compaction/ob_tenant_tablet_scheduler.h"
 #include "ob_rebuild_service.h"
 #include "storage/tablet/ob_tablet.h"
+#include "share/ob_storage_ha_diagnose_struct.h"
+#include "storage/high_availability/ob_storage_ha_diagnose_mgr.h"
 #include "storage/tx/wrs/ob_weak_read_util.h"
 
 using namespace oceanbase::transaction;
@@ -56,6 +58,10 @@ ERRSIM_POINT_DEF(EN_UPDATE_TRANSFER_TASK_FAILED);
 ERRSIM_POINT_DEF(EN_START_CAN_NOT_RETRY);
 ERRSIM_POINT_DEF(EN_MAKE_SRC_LS_REBUILD);
 ERRSIM_POINT_DEF(EN_INSERT_TRANSFER_START_FAILED);
+ERRSIM_POINT_DEF(EN_TRANSFER_DIAGNOSE_START_FAILED);
+ERRSIM_POINT_DEF(EN_TRANSFER_DIAGNOSE_DOING_FAILED);
+ERRSIM_POINT_DEF(EN_TRANSFER_DIAGNOSE_ABORT_FAILED);
+ERRSIM_POINT_DEF(EN_TRANSFER_DIAGNOSE_BACKFILL_FAILED);
 
 ObTransferHandler::ObTransferHandler()
   : is_inited_(false),
@@ -68,6 +74,9 @@ ObTransferHandler::ObTransferHandler()
     transfer_worker_mgr_(),
     round_(0),
     gts_seq_(share::SCN::base_scn()),
+    related_info_(),
+    task_info_(),
+    diagnose_result_msg_(share::ObStorageHACostItemName::MAX_NAME),
     transfer_handler_lock_(),
     transfer_handler_enabled_(true)
 {
@@ -95,6 +104,8 @@ int ObTransferHandler::init(
         KP(svr_rpc_proxy), KP(storage_rpc), KP(sql_proxy));
   } else if (OB_FAIL(transfer_worker_mgr_.init(ls))) {
     LOG_WARN("failed to init transfer worker manager", K(ret), KP(ls));
+  } else if (OB_FAIL(related_info_.init())) {
+    LOG_WARN("failed to init related_info");
   } else {
     ls_ = ls;
     bandwidth_throttle_ = bandwidth_throttle;
@@ -272,6 +283,9 @@ void ObTransferHandler::destroy()
     svr_rpc_proxy_ = nullptr;
     storage_rpc_ = nullptr;
     sql_proxy_ = nullptr;
+    related_info_.destroy();
+    task_info_.reset();
+    diagnose_result_msg_ = share::ObStorageHACostItemName::MAX_NAME;
     is_inited_ = false;
   }
 }
@@ -325,6 +339,7 @@ int ObTransferHandler::flush(share::SCN &scn)
 int ObTransferHandler::process()
 {
   int ret = OB_SUCCESS;
+  int tmp_ret = OB_SUCCESS;
   ObCurTraceId::init(GCONF.self_addr_);
   common::SpinRLockGuard guard(transfer_handler_lock_);
   if (!is_inited_) {
@@ -337,6 +352,9 @@ int ObTransferHandler::process()
   } else if (OB_FAIL(do_worker_transfer_())) {
     LOG_WARN("failed to do worker transfer", K(ret));
   }
+  if (OB_TMP_FAIL(do_clean_diagnose_info_())) {
+    LOG_WARN("failed to clean diagnose info", K(tmp_ret));
+  }
   return ret;
 }
 
@@ -359,10 +377,11 @@ int ObTransferHandler::do_leader_transfer_()
     } else {
       LOG_WARN("failed to get transfer task", K(ret), KPC(ls_));
     }
+  } else if (OB_FAIL(task_info_.assign(task_info))) {
+    LOG_WARN("fail to assign task info", K(ret), K(task_info));
   } else {
     round_++;
     ObCurTraceId::set(task_info.trace_id_);
-
     switch (task_info.status_) {
     case ObTransferStatus::START: {
       if (OB_FAIL(do_with_start_status_(task_info))) {
@@ -437,6 +456,10 @@ int ObTransferHandler::do_with_start_status_(const share::ObTransferTaskInfo &ta
   bool is_leader = true;
   bool succ_block_tx = false;
   int64_t tablet_stop_begin = 0;
+  int64_t tmp_round = round_;
+  diagnose_result_msg_ = share::ObStorageHACostItemName::MAX_NAME;
+  process_perf_diagnose_info_(ObStorageHACostItemName::TRANSFER_START_BEGIN,
+        ObStorageHADiagTaskType::TRANSFER_START, start_ts, round_, false/*is_report*/);
   bool commit_succ = false;
 
   if (!is_inited_) {
@@ -531,8 +554,15 @@ int ObTransferHandler::do_with_start_status_(const share::ObTransferTaskInfo &ta
       LOG_WARN("failed to commit trans", K(tmp_ret), K(ret));
       if (OB_SUCCESS == ret) {
         ret = tmp_ret;
+        diagnose_result_msg_ = share::ObStorageHACostItemName::START_TRANS_COMMIT;
       }
       commit_succ = false;
+    } else if (OB_SUCCESS == ret) {
+      round_ = 0;
+    }
+    if (OB_SUCC(ret)) {
+      process_perf_diagnose_info_(ObStorageHACostItemName::START_TRANS_COMMIT,
+          ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, tmp_round, false/*is_report*/);
     }
     int64_t trans_commit_end = ObTimeUtil::current_time();
     if (new_transfer) {
@@ -551,9 +581,13 @@ int ObTransferHandler::do_with_start_status_(const share::ObTransferTaskInfo &ta
     } else if (can_retry_(task_info, ret)) {
       LOG_INFO("transfer task can retry", K(ret), K(task_info));
       if (!new_transfer && OB_TMP_FAIL(unblock_tx_(task_info.tenant_id_, task_info.src_ls_id_, gts_seq_))) {
+        diagnose_result_msg_ = share::ObStorageHACostItemName::UNLOCK_MEMBER_LIST_IN_START;
         LOG_WARN("failed to unblock tx", K(ret));
       } else if (OB_TMP_FAIL(unlock_src_and_dest_ls_member_list_(task_info))) {
         LOG_WARN("failed to unlock src and dest ls member list", K(tmp_ret), K(ret), K(task_info));
+      } else {
+        process_perf_diagnose_info_(ObStorageHACostItemName::UNLOCK_MEMBER_LIST_IN_START,
+          ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, tmp_round, false/*is_report*/);
       }
       ob_usleep(INTERVAL_US);
     } else if (OB_SUCCESS != (tmp_ret = update_transfer_status_aborted_(task_info, ret))) {
@@ -581,6 +615,26 @@ int ObTransferHandler::do_with_start_status_(const share::ObTransferTaskInfo &ta
       LOG_WARN("failed to wakeup dest ls leader", K(tmp_ret), K(task_info));
     }
   }
+#ifdef ERRSIM
+  if (OB_SUCC(ret)) {
+    ret = EN_TRANSFER_DIAGNOSE_START_FAILED ? : OB_SUCCESS;
+    if (OB_FAIL(ret)) {
+      STORAGE_LOG(WARN, "fake EN_TRANSFER_DIAGNOSE_START_FAILED", K(ret));
+    }
+  }
+#endif
+  if (OB_FAIL(ret) && task_info.is_valid()) {
+    if (OB_TMP_FAIL(ObStorageHADiagMgr::add_transfer_error_diagnose_info(task_info.task_id_, task_info.dest_ls_id_,
+        share::ObStorageHADiagTaskType::TRANSFER_START, tmp_round, ret, diagnose_result_msg_))) {
+      LOG_WARN("failed to add error diagnose info", K(tmp_ret), K(ret),
+          K(task_info.task_id_), K(task_info.dest_ls_id_), K(tmp_round));
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    process_perf_diagnose_info_(ObStorageHACostItemName::TRANSFER_START_END,
+        ObStorageHADiagTaskType::TRANSFER_START, start_ts, tmp_round, true/*is_report*/);
+  }
   LOG_INFO("[TRANSFER] finish do with start status", K(ret), K(task_info), "cost_ts", ObTimeUtil::current_time() - start_ts);
   return ret;
 }
@@ -653,7 +707,12 @@ int ObTransferHandler::lock_src_and_dest_ls_member_list_(
                         "member_list_is_same", is_same);
 #endif
   DEBUG_SYNC(AFTER_TRANSFER_START_LOCK_MEMBER_LIST);
-
+  if (OB_SUCC(ret)) {
+    process_perf_diagnose_info_(ObStorageHACostItemName::LOCK_MEMBER_LIST,
+        ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/);
+  } else {
+    diagnose_result_msg_ = share::ObStorageHACostItemName::LOCK_MEMBER_LIST;
+  }
   return ret;
 }
 
@@ -828,6 +887,12 @@ int ObTransferHandler::check_src_ls_has_active_trans_(
 #endif
     LOG_INFO("get active trans count", K(src_ls_id), K(active_trans_count));
   }
+  if (OB_SUCC(ret)) {
+    process_perf_diagnose_info_(ObStorageHACostItemName::CHECK_SRC_LS_HAS_ACTIVE_TRANS,
+        ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/);
+  } else {
+    diagnose_result_msg_ = share::ObStorageHACostItemName::CHECK_SRC_LS_HAS_ACTIVE_TRANS;
+  }
   return ret;
 }
 
@@ -927,6 +992,12 @@ int ObTransferHandler::check_start_status_transfer_tablets_(
 #endif
 
   }
+  if (OB_SUCC(ret)) {
+    process_perf_diagnose_info_(ObStorageHACostItemName::CHECK_START_STATUS_TRANSFER_TABLETS,
+        ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/);
+  } else {
+    diagnose_result_msg_ = share::ObStorageHACostItemName::CHECK_START_STATUS_TRANSFER_TABLETS;
+  }
   return ret;
 }
 
@@ -1307,6 +1378,7 @@ int ObTransferHandler::do_tx_start_transfer_out_(
     start_transfer_out_info.data_end_scn_ = data_end_scn;
     // TODO lana optimise transfer_epoch value
     start_transfer_out_info.transfer_epoch_ = task_info.task_id_.id();
+    start_transfer_out_info.task_id_ = task_info.task_id_;
     start_transfer_out_info.data_version_ = DEFAULT_MIN_DATA_VERSION;
     if (OB_FAIL(start_transfer_out_info.tablet_list_.assign(task_info.tablet_list_))) {
       LOG_WARN("failed to assign transfer tablet list", K(ret), K(task_info));
@@ -1347,6 +1419,12 @@ int ObTransferHandler::do_tx_start_transfer_out_(
     DEBUG_SYNC(AFTER_START_TRANSFER_OUT);
 
   }
+  if (OB_SUCC(ret)) {
+    process_perf_diagnose_info_(ObStorageHACostItemName::REGISTER_TRANSFER_START_OUT,
+        ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/);
+  } else {
+    diagnose_result_msg_ = share::ObStorageHACostItemName::REGISTER_TRANSFER_START_OUT;
+  }
   return ret;
 }
 
@@ -1401,6 +1479,12 @@ int ObTransferHandler::get_start_transfer_out_scn_(
 #endif
     DEBUG_SYNC(AFTER_START_TRANSFER_GET_START_SCN);
   }
+  if (OB_SUCC(ret)) {
+    process_perf_diagnose_info_(ObStorageHACostItemName::DEST_LS_GET_START_SCN,
+        ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/);
+  } else {
+    diagnose_result_msg_ = share::ObStorageHACostItemName::DEST_LS_GET_START_SCN;
+  }
   return ret;
 }
 
@@ -1512,7 +1596,12 @@ int ObTransferHandler::wait_src_ls_replay_to_start_scn_(
     }
   }
 #endif
-
+  if (OB_SUCC(ret)) {
+    process_perf_diagnose_info_(ObStorageHACostItemName::WAIT_SRC_LS_REPLAY_TO_START_SCN,
+        ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/);
+  } else {
+    diagnose_result_msg_ = share::ObStorageHACostItemName::WAIT_SRC_LS_REPLAY_TO_START_SCN;
+  }
   return ret;
 }
 
@@ -1555,6 +1644,13 @@ int ObTransferHandler::precheck_ls_replay_scn_(const share::ObTransferTaskInfo &
     ret = OB_TRANSFER_CANNOT_START;
     LOG_WARN("transfer precheck timeout, cannot start transfer in", K(ret), K(task_info));
   }
+
+  if (OB_SUCC(ret)) {
+    process_perf_diagnose_info_(ObStorageHACostItemName::PRECHECK_LS_REPALY_SCN,
+        ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/);
+  } else {
+    diagnose_result_msg_ = share::ObStorageHACostItemName::PRECHECK_LS_REPALY_SCN;
+  }
   return ret;
 }
 
@@ -1680,7 +1776,12 @@ int ObTransferHandler::get_transfer_tablets_meta_(
 
   }
   DEBUG_SYNC(AFTER_START_TRANSFER_GET_TABLET_META);
-
+  if (OB_SUCC(ret)) {
+    process_perf_diagnose_info_(ObStorageHACostItemName::SRC_LS_GET_TABLET_META,
+        ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/);
+  } else {
+    diagnose_result_msg_ = share::ObStorageHACostItemName::SRC_LS_GET_TABLET_META;
+  }
   return ret;
 }
 
@@ -1752,6 +1853,7 @@ int ObTransferHandler::do_tx_start_transfer_in_(
       start_transfer_in_info.src_ls_id_ = task_info.src_ls_id_;
       start_transfer_in_info.dest_ls_id_ = task_info.dest_ls_id_;
       start_transfer_in_info.start_scn_ = start_scn;
+      start_transfer_in_info.task_id_ = task_info.task_id_;
       start_transfer_in_info.data_version_ = DEFAULT_MIN_DATA_VERSION;
 
       if (timeout_ctx.is_timeouted()) {
@@ -1796,6 +1898,12 @@ int ObTransferHandler::do_tx_start_transfer_in_(
     LOG_INFO("[TRANSFER_BLOCK_TX] do tx start transfer in", K(ret), "cost", ObTimeUtil::current_time() - start_ts);
     DEBUG_SYNC(AFTER_START_TRANSFER_IN);
   }
+  if (OB_SUCC(ret)) {
+    process_perf_diagnose_info_(ObStorageHACostItemName::REGISTER_TRANSFER_START_IN,
+        ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/);
+  } else {
+    diagnose_result_msg_ = share::ObStorageHACostItemName::REGISTER_TRANSFER_START_IN;
+  }
   return ret;
 }
 
@@ -1885,6 +1993,12 @@ int ObTransferHandler::update_all_tablet_to_ls_(
     }
 
   }
+  if (OB_SUCC(ret)) {
+    process_perf_diagnose_info_(ObStorageHACostItemName::UPDATE_ALL_TABLET_TO_LS,
+        ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/);
+  } else {
+    diagnose_result_msg_ = share::ObStorageHACostItemName::UPDATE_ALL_TABLET_TO_LS;
+  }
   return ret;
 }
 
@@ -1910,7 +2024,12 @@ int ObTransferHandler::lock_tablet_on_dest_ls_for_table_lock_(
   } else {
     LOG_INFO("[TRANSFER] success lock tablet on dest ls for table lock", "cost", ObTimeUtil::current_time() - start_ts);
   }
-
+  if (OB_SUCC(ret)) {
+    process_perf_diagnose_info_(ObStorageHACostItemName::LOCK_TABLET_ON_DEST_LS_FOR_TABLE_LOCK,
+        ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/);
+  } else {
+    diagnose_result_msg_ = share::ObStorageHACostItemName::LOCK_TABLET_ON_DEST_LS_FOR_TABLE_LOCK;
+  }
   return ret;
 }
 
@@ -1974,6 +2093,7 @@ int ObTransferHandler::update_transfer_status_aborted_(
 {
   int ret = OB_SUCCESS;
   int tmp_ret = OB_SUCCESS;
+  int64_t tmp_round = round_;
   const share::ObTransferStatus next_status(ObTransferStatus::ABORTED);
   ObTimeoutCtx timeout_ctx;
   ObMySQLTransaction trans;
@@ -2010,8 +2130,16 @@ int ObTransferHandler::update_transfer_status_aborted_(
       if (OB_SUCCESS == ret) {
         ret = tmp_ret;
       }
+    } else if (OB_SUCCESS == ret) {
+      round_ = 0;
     }
   }
+  if (OB_SUCC(ret)) {
+    process_perf_diagnose_info_(ObStorageHACostItemName::TRANSFER_START_END,
+        ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, tmp_round, true/*is_report*/);
+  } else {
+    diagnose_result_msg_ = share::ObStorageHACostItemName::TRANSFER_START_END;
+  }
   return ret;
 }
 
@@ -2128,6 +2256,10 @@ int ObTransferHandler::do_with_aborted_status_(
   ObTimeoutCtx timeout_ctx;
   ObMySQLTransaction trans;
   const int64_t tmp_round = round_;
+  const int64_t start_ts = ObTimeUtil::current_time();
+  diagnose_result_msg_ = share::ObStorageHACostItemName::MAX_NAME;
+  process_perf_diagnose_info_(ObStorageHACostItemName::TRANSFER_ABORT_BEGIN,
+      ObStorageHADiagTaskType::TRANSFER_ABORT, start_ts, round_, false/*is_report*/);
   if (!is_inited_) {
     ret = OB_NOT_INIT;
     LOG_WARN("transfer handler do not init", K(ret));
@@ -2145,17 +2277,26 @@ int ObTransferHandler::do_with_aborted_status_(
       } else if (OB_FAIL(update_transfer_status_(task_info, next_status, scn, result, trans))) {
         LOG_WARN("failed to update transfer status", K(ret), K(task_info), K(next_status));
       } else if (OB_FAIL(unlock_src_and_dest_ls_member_list_(task_info))) {
+        diagnose_result_msg_ = share::ObStorageHACostItemName::UNLOCK_MEMBER_LIST_IN_ABORT;
         LOG_WARN("failed to unlock src and dest ls member list", K(ret), K(task_info));
+      } else {
+        process_perf_diagnose_info_(ObStorageHACostItemName::UNLOCK_MEMBER_LIST_IN_ABORT,
+            ObStorageHADiagTaskType::TRANSFER_ABORT, start_ts, round_, false/*is_report*/);
       }
 
       if (OB_TMP_FAIL(commit_trans_(ret, trans))) {
         LOG_WARN("failed to commit trans", K(tmp_ret), K(ret));
         if (OB_SUCCESS == ret) {
           ret = tmp_ret;
+          diagnose_result_msg_ = share::ObStorageHACostItemName::ABORT_TRANS_COMMIT;
         }
       } else if (OB_SUCCESS == ret) {
         round_ = 0;
       }
+      if (OB_SUCC(ret)) {
+        process_perf_diagnose_info_(ObStorageHACostItemName::ABORT_TRANS_COMMIT,
+            ObStorageHADiagTaskType::TRANSFER_ABORT, start_ts, tmp_round, false/*is_report*/);
+      }
     }
   }
 
@@ -2173,6 +2314,26 @@ int ObTransferHandler::do_with_aborted_status_(
   } else if (OB_SUCCESS != (tmp_ret = record_server_event_(ret, tmp_round, task_info))) {
     LOG_WARN("failed to record server event", K(tmp_ret), K(ret), K(retry_count_), K(task_info));
   }
+#ifdef ERRSIM
+  if (OB_SUCC(ret)) {
+    ret = EN_TRANSFER_DIAGNOSE_ABORT_FAILED ? : OB_SUCCESS;
+    if (OB_FAIL(ret)) {
+      STORAGE_LOG(WARN, "fake EN_TRANSFER_DIAGNOSE_ABORT_FAILED", K(ret));
+    }
+  }
+#endif
+  if (OB_FAIL(ret)) {
+    if (OB_TMP_FAIL(ObStorageHADiagMgr::add_transfer_error_diagnose_info(task_info.task_id_, task_info.dest_ls_id_,
+        share::ObStorageHADiagTaskType::TRANSFER_ABORT, tmp_round, ret, diagnose_result_msg_))) {
+      LOG_WARN("failed to add error diagnose info", K(tmp_ret), K(ret),
+          K(task_info.task_id_), K(task_info.dest_ls_id_), K(tmp_round));
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    process_perf_diagnose_info_(ObStorageHACostItemName::TRANSFER_ABORT_END,
+        ObStorageHADiagTaskType::TRANSFER_ABORT, start_ts, tmp_round, true/*is_report*/);
+  }
   return ret;
 }
 
@@ -2244,6 +2405,12 @@ int ObTransferHandler::block_and_kill_tx_(
   SERVER_EVENT_SYNC_ADD("TRANSFER", "AFTER_TRANSFER_BLOCK_AND_KILL_TX");
 #endif
   DEBUG_SYNC(AFTER_TRANSFER_BLOCK_AND_KILL_TX);
+  if (OB_SUCC(ret)) {
+    process_perf_diagnose_info_(ObStorageHACostItemName::BLOCK_AND_KILL_TX,
+        ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/);
+  } else {
+    diagnose_result_msg_ = share::ObStorageHACostItemName::BLOCK_AND_KILL_TX;
+  }
   return ret;
 }
 
@@ -2337,6 +2504,12 @@ int ObTransferHandler::unblock_tx_(
   SERVER_EVENT_SYNC_ADD("TRANSFER", "AFTER_TRANSFER_UNBLOCK_TX");
 #endif
   DEBUG_SYNC(AFTER_TRANSFER_UNBLOCK_TX);
+  if (OB_SUCC(ret)) {
+    process_perf_diagnose_info_(ObStorageHACostItemName::UNBLOCK_TX,
+        ObStorageHADiagTaskType::TRANSFER_START, 0/*start_ts*/, round_, false/*is_report*/);
+  } else {
+    diagnose_result_msg_ = share::ObStorageHACostItemName::UNBLOCK_TX;
+  }
   return ret;
 }
 
@@ -2436,6 +2609,9 @@ int ObTransferHandler::stop_tablets_schedule_medium_(const ObIArray<ObTabletID>
   } else {
     succ_stop = true;
   }
+  if (OB_FAIL(ret)) {
+    diagnose_result_msg_ = share::ObStorageHACostItemName::STOP_LS_SCHEDULE_MEMDIUM;
+  }
   return ret;
 }
 
@@ -2783,6 +2959,159 @@ int ObTransferHandler::check_task_exist_(
   return ret;
 }
 
+int ObTransferHandler::set_related_info(
+    const share::ObTransferTaskID &task_id,
+    const share::SCN &start_scn)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("ls transfer handler do not init", K(ret));
+  } else if (!task_id.is_valid() || !start_scn.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(task_id), K(start_scn));
+  } else if (OB_FAIL(related_info_.set_info(task_id, start_scn))) {
+    LOG_WARN("fail to set info", K(ret), K(task_id), K(start_scn));
+  }
+  return ret;
+}
+
+int ObTransferHandler::get_related_info_task_id(share::ObTransferTaskID &task_id) const
+{
+  int ret = OB_SUCCESS;
+  task_id.reset();
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("ls transfer handler do not init", K(ret));
+  } else if (OB_FAIL(related_info_.get_related_info_task_id(task_id))) {
+    LOG_WARN("failed to get task id", K(ret));
+  }
+  return ret;
+}
+
+int ObTransferHandler::record_error_diagnose_info_in_replay(
+    const share::ObTransferTaskID &task_id,
+    const share::ObLSID &dest_ls_id,
+    const int result_code,
+    const bool clean_related_info,
+    const share::ObStorageHADiagTaskType type,
+    const share::ObStorageHACostItemName result_msg)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("ls transfer handler do not init", K(ret));
+  } else if (!task_id.is_valid()
+      || !dest_ls_id.is_valid()
+      || type < share::ObStorageHADiagTaskType::TRANSFER_START
+      || type >= share::ObStorageHADiagTaskType::MAX_TYPE) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(task_id), K(dest_ls_id), K(type));
+  } else if (OB_FAIL(related_info_.record_error_diagnose_info_in_replay(
+        task_id, dest_ls_id, result_code, clean_related_info, type, result_msg))) {
+    LOG_WARN("failed to record diagnose info in replay", K(ret), K(task_id), K(dest_ls_id),
+        K(result_code), K(clean_related_info), K(type), K(result_msg));
+  }
+  return ret;
+}
+
+int ObTransferHandler::record_error_diagnose_info_in_backfill(
+  const share::SCN &log_sync_scn,
+  const share::ObLSID &dest_ls_id,
+  const int result_code,
+  const ObTabletID &tablet_id,
+  const ObMigrationStatus &migration_status,
+  const share::ObStorageHACostItemName result_msg)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("ls transfer handler do not init", K(ret));
+  } else if (!log_sync_scn.is_valid()
+      || !dest_ls_id.is_valid()
+      || !tablet_id.is_valid()
+      || OB_SUCCESS == result_code
+      || migration_status < ObMigrationStatus::OB_MIGRATION_STATUS_NONE
+      || migration_status >= ObMigrationStatus::OB_MIGRATION_STATUS_MAX) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(log_sync_scn), K(dest_ls_id), K(result_code), K(tablet_id), K(migration_status));
+  } else if (OB_FAIL(related_info_.record_error_diagnose_info_in_backfill(
+        log_sync_scn, dest_ls_id, result_code, tablet_id, migration_status, result_msg))) {
+    LOG_WARN("failed to record diagnose info in worker", K(ret), K(log_sync_scn),
+        K(dest_ls_id), K(result_code), K(tablet_id), K(migration_status), K(result_msg));
+  }
+  return ret;
+}
+
+void ObTransferHandler::reset_related_info()
+{
+  related_info_.reset();
+}
+
+int ObTransferHandler::reset_related_info(const share::ObTransferTaskID &task_id)
+{
+  int ret = OB_SUCCESS;
+  if (!task_id.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(task_id));
+  } else {
+    related_info_.reset(task_id);
+  }
+  return ret;
+}
+
+void ObTransferHandler::process_perf_diagnose_info_(
+      const ObStorageHACostItemName name,
+      const ObStorageHADiagTaskType task_type,
+      const int64_t start_ts,
+      const int64_t round, const bool is_report) const
+{
+  int ret = OB_SUCCESS;
+  ObArenaAllocator alloc;
+  ObTransferPerfDiagInfo info;
+  ObStorageHATimestampItem item;
+  item.name_ = name;
+  item.type_ = ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE;
+  item.retry_id_ = round;
+  item.timestamp_ = ObTimeUtil::current_time();
+  common::ObTabletID tablet_id;
+  share::ObStorageHADiagTaskKey key;
+  if (OB_FAIL(info.init(&alloc, MTL_ID()))) {
+    LOG_WARN("fail to init info", K(ret));
+  } else if (OB_FAIL(info.add_item(item))) {
+    LOG_WARN("fail to add item", K(ret), K(item));
+  } else if (OB_FAIL(ObStorageHADiagMgr::construct_diagnose_info_key(task_info_.task_id_, ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE,
+      task_type, ObStorageHADiagType::PERF_DIAGNOSE, round, tablet_id, key))) {
+    LOG_WARN("failed to construct error diagnose info key", K(ret), K(task_info_.task_id_), K(round), K(tablet_id));
+  } else if (OB_FAIL(ObStorageHADiagMgr::construct_diagnose_info(task_info_.task_id_, task_info_.dest_ls_id_,
+      task_type, round, OB_SUCCESS, ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE, info))) {
+    LOG_WARN("failed to construct diagnose info", K(ret), K(task_info_), K(round), K(task_type));
+  } else if (OB_FAIL(ObStorageHADiagMgr::add_transfer_perf_diagnose_info(key, start_ts, task_info_.tablet_list_.count(), is_report, info))) {
+    LOG_WARN("failed to add perf diagnose info", K(ret), K(key), K(info), K(start_ts), K(task_info_), K(is_report));
+  }
+}
+
+int ObTransferHandler::record_perf_diagnose_info_in_replay(
+    const share::ObStorageHAPerfDiagParams &params,
+    const int result,
+    const uint64_t timestamp,
+    const int64_t start_ts,
+    const bool is_report)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("ls transfer handler do not init", K(ret));
+  } else if (!params.is_valid()
+      || start_ts < 0) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(params), K(start_ts));
+  } else if (OB_FAIL(related_info_.record_perf_diagnose_info_in_replay(params, result, timestamp, start_ts, is_report))) {
+    LOG_WARN("failed to record diagnose info in replay", K(ret), K(params), K(result), K(start_ts), K(timestamp), K(is_report));
+  }
+  return ret;
+}
+
 int ObTransferHandler::get_src_ls_member_list_(
     common::ObMemberList &member_list)
 {
@@ -2807,6 +3136,34 @@ int ObTransferHandler::get_src_ls_member_list_(
   return ret;
 }
 
+int ObTransferHandler::record_perf_diagnose_info_in_backfill(
+    const share::ObStorageHAPerfDiagParams &params,
+    const share::SCN &log_sync_scn,
+    const int result_code,
+    const ObMigrationStatus &migration_status,
+    const uint64_t timestamp,
+    const int64_t start_ts,
+    const bool is_report)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("ls transfer handler do not init", K(ret));
+  } else if (!params.is_valid()
+      ||!log_sync_scn.is_valid()
+      || migration_status < ObMigrationStatus::OB_MIGRATION_STATUS_NONE
+      || migration_status >= ObMigrationStatus::OB_MIGRATION_STATUS_MAX
+      || start_ts < 0) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(log_sync_scn), K(start_ts), K(params), K(migration_status));
+  } else if (OB_FAIL(related_info_.record_perf_diagnose_info_in_backfill(params, log_sync_scn,
+      result_code, migration_status, timestamp, start_ts, is_report))) {
+    LOG_WARN("failed to record diagnose info in backfill", K(ret), K(params), K(log_sync_scn),
+        K(result_code), K(migration_status), K(start_ts), K(timestamp), K(is_report));
+  }
+  return ret;
+}
+
 int ObTransferHandler::broadcast_tablet_location_(const ObTransferTaskInfo &task_info)
 {
   int ret = OB_SUCCESS;
@@ -2848,6 +3205,28 @@ int ObTransferHandler::broadcast_tablet_location_(const ObTransferTaskInfo &task
   return ret;
 }
 
-}
+int ObTransferHandler::do_clean_diagnose_info_()
+{
+  int ret = OB_SUCCESS;
+  bool is_leader = false;
+  ObStorageHADiagMgr *mgr = nullptr;
+  if (!is_inited_) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("transfer handler do not init", K(ret));
+  } else if (OB_FAIL(check_self_is_leader_(is_leader))) {
+    LOG_WARN("failed to check self is leader", K(ret), KPC(ls_));
+  } else if (is_leader) {
+    // clean follower diagnose info
+  } else if (!task_info_.is_valid()) {
+    // unset error ret code
+  } else if (OB_ISNULL(mgr = MTL(ObStorageHADiagMgr *))) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("failed to get ObStorageHADiagMgr from MTL", K(ret));
+  } else if (OB_FAIL(mgr->report_task(task_info_.task_id_))) {
+    LOG_WARN("fail to report task", K(ret), K(task_info_.task_id_));
+  }
+  return ret;
 }
 
+}
+}

src/storage/high_availability/ob_transfer_handler.h

@@ -31,6 +31,7 @@
 #include "ob_transfer_backfill_tx.h"
 #include "lib/thread/thread_mgr_interface.h"
 #include "logservice/ob_log_base_type.h"
+#include "share/ob_storage_ha_diagnose_struct.h"
 
 namespace oceanbase
 {
@@ -68,6 +69,40 @@ public:
   int safe_to_destroy(bool &is_safe);
   int offline();
   void online();
+  int set_related_info(
+      const share::ObTransferTaskID &task_id,
+      const share::SCN &start_scn);
+  int get_related_info_task_id(share::ObTransferTaskID &task_id) const;
+  int reset_related_info(const share::ObTransferTaskID &task_id);
+  int record_error_diagnose_info_in_replay(
+      const share::ObTransferTaskID &task_id,
+      const share::ObLSID &dest_ls_id,
+      const int result_code,
+      const bool clean_related_info,
+      const share::ObStorageHADiagTaskType type,
+      const share::ObStorageHACostItemName result_msg);
+  int record_error_diagnose_info_in_backfill(
+      const share::SCN &log_sync_scn,
+      const share::ObLSID &dest_ls_id,
+      const int result_code,
+      const ObTabletID &tablet_id,
+      const ObMigrationStatus &migration_status,
+      const share::ObStorageHACostItemName result_msg);
+  void reset_related_info();
+  int record_perf_diagnose_info_in_replay(
+      const share::ObStorageHAPerfDiagParams &params,
+      const int result,
+      const uint64_t timestamp,
+      const int64_t start_ts,
+      const bool is_report);
+  int record_perf_diagnose_info_in_backfill(
+      const share::ObStorageHAPerfDiagParams &params,
+      const share::SCN &log_sync_scn,
+      const int result_code,
+      const ObMigrationStatus &migration_status,
+      const uint64_t timestamp,
+      const int64_t start_ts,
+      const bool is_report);
 private:
   int get_transfer_task_(share::ObTransferTaskInfo &task_info);
   int get_transfer_task_from_inner_table_(
@@ -284,6 +319,12 @@ private:
   int get_src_ls_member_list_(
       common::ObMemberList &member_list);
   int broadcast_tablet_location_(const share::ObTransferTaskInfo &task_info);
+  void process_perf_diagnose_info_(
+      const ObStorageHACostItemName name,
+      const ObStorageHADiagTaskType task_type,
+      const int64_t start_ts,
+      const int64_t round, const bool is_report) const;
+  int do_clean_diagnose_info_();
 
   int register_move_tx_ctx_batch_(const share::ObTransferTaskInfo &task_info,
                                   const SCN transfer_scn,
@@ -306,6 +347,9 @@ private:
   ObTransferWorkerMgr transfer_worker_mgr_;
   int64_t round_;
   share::SCN gts_seq_;
+  ObTransferRelatedInfo related_info_;
+  ObTransferTaskInfo task_info_;
+  share::ObStorageHACostItemName diagnose_result_msg_;
   common::SpinRWLock transfer_handler_lock_;
   bool transfer_handler_enabled_;
   DISALLOW_COPY_AND_ASSIGN(ObTransferHandler);

src/storage/high_availability/ob_transfer_struct.cpp

@@ -20,6 +20,7 @@
 #include "storage/tx_storage/ob_ls_service.h"
 #include "storage/high_availability/ob_storage_ha_utils.h"
 #include "storage/tx/ob_ts_mgr.h"
+#include "storage/high_availability/ob_storage_ha_diagnose_mgr.h"
 
 using namespace oceanbase;
 using namespace share;
@@ -591,3 +592,418 @@ int ObTransferTaskLockInfo::set(const uint64_t tenant_id, const share::ObLSID &l
   }
   return ret;
 }
+
+ObTransferRelatedInfo::ObTransferRelatedInfo()
+  : is_inited_(false),
+    task_id_(),
+    start_scn_(),
+    start_out_log_replay_num_(0),
+    start_in_log_replay_num_(0),
+    finish_out_log_replay_num_(0),
+    finish_in_log_replay_num_(0),
+    map_(),
+    lock_()
+{
+}
+
+ObTransferRelatedInfo::~ObTransferRelatedInfo()
+{
+  destroy();
+}
+
+int ObTransferRelatedInfo::init()
+{
+  int ret = OB_SUCCESS;
+  const int64_t BUCKET_NUM = 100;
+  ObMemAttr attr(MTL_ID(), "TXBFRetryId");
+  if (IS_INIT) {
+    ret = OB_INIT_TWICE;
+    LOG_WARN("init twice", K(ret));
+  } else if (OB_FAIL(map_.create(BUCKET_NUM, attr))) {
+    LOG_WARN("fail to create hash map", K(ret), K(BUCKET_NUM));
+  } else {
+    start_out_log_replay_num_ = 0;
+    start_in_log_replay_num_ = 0;
+    finish_in_log_replay_num_ = 0;
+    finish_out_log_replay_num_ = 0;
+    is_inited_ = true;
+  }
+  return ret;
+}
+
+bool ObTransferRelatedInfo::is_valid() const
+{
+  return task_id_.is_valid()
+         && start_scn_.is_valid()
+         && start_out_log_replay_num_ >= 0
+         && start_in_log_replay_num_ >= 0
+         && finish_in_log_replay_num_ >= 0
+         && finish_out_log_replay_num_ >= 0
+         && is_inited_;
+}
+
+void ObTransferRelatedInfo::reset_()
+{
+  task_id_.reset();
+  start_scn_.reset();
+  start_out_log_replay_num_ = 0;
+  start_in_log_replay_num_ = 0;
+  finish_in_log_replay_num_ = 0;
+  finish_out_log_replay_num_ = 0;
+}
+
+void ObTransferRelatedInfo::destroy()
+{
+  common::SpinWLockGuard guard(lock_);
+  is_inited_ = false;
+  reset_();
+  map_.destroy();
+}
+
+void ObTransferRelatedInfo::reset()
+{
+  common::SpinWLockGuard guard(lock_);
+  reset_();
+  map_.reuse();
+}
+
+int ObTransferRelatedInfo::reset(const share::ObTransferTaskID &task_id)
+{
+  int ret = OB_SUCCESS;
+  common::SpinWLockGuard guard(lock_);
+  if (task_id != get_task_id_()) {
+    ret = OB_STATE_NOT_MATCH;
+    LOG_WARN("infos already are clean", K(ret), K(task_id), K(task_id_));
+  } else {
+    reset_();
+    map_.reuse();
+  }
+  return ret;
+}
+
+int ObTransferRelatedInfo::inc_tx_backfill_retry_num_(const ObTabletID &id, int64_t &retry_num)
+{
+  int ret = OB_SUCCESS;
+  retry_num = 0;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not inited", K(ret));
+  } else if (!id.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument!", K(ret), K(id));
+  } else {
+    if (OB_FAIL(map_.get_refactored(id, retry_num))) {
+      if (OB_HASH_NOT_EXIST == ret) {
+        ret = OB_SUCCESS;
+      } else {
+        LOG_WARN("fail to get retry id from map", K(ret), K(id));
+      }
+    }
+    if (OB_FAIL(ret)) {
+      // do nothing
+    } else if (OB_FAIL(map_.set_refactored(id, ++retry_num, 1/*overwrite*/))) {
+      LOG_WARN("fail to set retry_num", K(ret), K(id), K(retry_num));
+    }
+  }
+  return ret;
+}
+
+int ObTransferRelatedInfo::set_info(
+    const share::ObTransferTaskID &task_id,
+    const share::SCN &start_scn)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not inited", K(ret));
+  } else if (!task_id.is_valid() || !start_scn.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(task_id), K(start_scn));
+  } else {
+    common::SpinWLockGuard guard(lock_);
+    reset_();
+    map_.reuse();
+    start_scn_ = start_scn;
+    task_id_ = task_id;
+  }
+  return ret;
+}
+
+const share::ObTransferTaskID &ObTransferRelatedInfo::get_task_id_() const
+{
+  return task_id_;
+}
+
+const share::SCN &ObTransferRelatedInfo::get_start_scn_() const
+{
+  return start_scn_;
+}
+
+int ObTransferRelatedInfo::get_replay_retry_num_(
+    const share::ObStorageHADiagTaskType type, const bool inc_retry_num, int64_t &retry_num)
+{
+  int ret = OB_SUCCESS;
+  retry_num = 0;
+  switch (type) {
+    case ObStorageHADiagTaskType::TRANSFER_START_IN: {
+      if (inc_retry_num) {
+        start_out_log_replay_num_++;
+      }
+      retry_num = start_out_log_replay_num_;
+      break;
+    }
+    case ObStorageHADiagTaskType::TRANSFER_START_OUT: {
+      if (inc_retry_num) {
+        start_in_log_replay_num_++;
+      }
+      retry_num = start_in_log_replay_num_;
+      break;
+    }
+    case ObStorageHADiagTaskType::TRANSFER_FINISH_IN: {
+      if (inc_retry_num) {
+        finish_in_log_replay_num_++;
+      }
+      retry_num = finish_in_log_replay_num_;
+      break;
+    }
+    case ObStorageHADiagTaskType::TRANSFER_FINISH_OUT: {
+      if (inc_retry_num) {
+        finish_out_log_replay_num_++;
+      }
+      retry_num = finish_out_log_replay_num_;
+      break;
+    }
+    default: {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_WARN("unexpected type", K(ret), K(type));
+    }
+  }
+  return ret;
+}
+
+int ObTransferRelatedInfo::record_error_diagnose_info_in_backfill(
+    const share::SCN &log_sync_scn,
+    const share::ObLSID &dest_ls_id,
+    const int result_code,
+    const ObTabletID &tablet_id,
+    const ObMigrationStatus &migration_status,
+    const share::ObStorageHACostItemName result_msg)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not inited", K(ret));
+  } else if (!log_sync_scn.is_valid()
+      || !dest_ls_id.is_valid()
+      || !tablet_id.is_valid()
+      || OB_SUCCESS == result_code
+      || migration_status < ObMigrationStatus::OB_MIGRATION_STATUS_NONE
+      || migration_status >= ObMigrationStatus::OB_MIGRATION_STATUS_MAX) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(log_sync_scn), K(dest_ls_id),
+        K(result_code), K(tablet_id), K(migration_status));
+  } else {
+    common::SpinWLockGuard guard(lock_);
+    if (log_sync_scn != get_start_scn_()) {
+      if (ObMigrationStatus::OB_MIGRATION_STATUS_NONE != migration_status) {
+        ret = OB_STATE_NOT_MATCH;
+        LOG_WARN("dest ls in migration", K(ret), K(dest_ls_id), K(migration_status));
+      } else {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_WARN("related_info already change", K(ret), K(start_scn_), K(log_sync_scn));
+      }
+    } else {
+      int64_t retry_num = 0;
+      if (OB_FAIL(inc_tx_backfill_retry_num_(tablet_id, retry_num))) {
+        LOG_WARN("fail to get retry id", K(ret), K(tablet_id));
+      } else if (OB_FAIL(ObStorageHADiagMgr::add_transfer_error_diagnose_info(get_task_id_(), dest_ls_id,
+          share::ObStorageHADiagTaskType::TRANSFER_BACKFILLED, retry_num, result_code, tablet_id, result_msg))) {
+        LOG_WARN("failed to add error diagnose info", K(ret), K(get_task_id_()), K(dest_ls_id),
+            K(retry_num), K(result_code), K(tablet_id), K(result_msg));
+      }
+    }
+  }
+  return ret;
+}
+
+int ObTransferRelatedInfo::record_error_diagnose_info_in_replay(
+    const share::ObTransferTaskID &task_id,
+    const share::ObLSID &dest_ls_id,
+    const int result_code,
+    const bool clean_related_info,
+    const share::ObStorageHADiagTaskType type,
+    const share::ObStorageHACostItemName result_msg)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not inited", K(ret));
+  } else if (!task_id.is_valid()
+      || !dest_ls_id.is_valid()
+      || type < share::ObStorageHADiagTaskType::TRANSFER_START
+      || type >= share::ObStorageHADiagTaskType::MAX_TYPE) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(task_id), K(dest_ls_id), K(type));
+  } else {
+    common::SpinWLockGuard guard(lock_);
+    if (OB_SUCCESS == result_code) {
+      // do nothing
+    } else {
+      int64_t log_replay_num = 0;
+      if (OB_FAIL(get_replay_retry_num_(type, true/*inc_retry_num*/, log_replay_num))) {
+        LOG_WARN("fail to inc retry num", K(ret), K(type), K(log_replay_num));
+      } else if (OB_FAIL(ObStorageHADiagMgr::add_transfer_error_diagnose_info(task_id, dest_ls_id,
+          type, log_replay_num, result_code, result_msg))) {
+        LOG_WARN("failed to add error diagnose info", K(ret),
+            K(task_id), K(dest_ls_id), K(type), K(log_replay_num), K(result_code), K(result_msg));
+      }
+    }
+    if (OB_SUCCESS == result_code && clean_related_info) {
+      reset_();
+      map_.reuse();
+    }
+  }
+  return ret;
+}
+
+int ObTransferRelatedInfo::construct_perf_diag_info_(
+    const share::ObStorageHAPerfDiagParams &params,
+    const uint64_t timestamp,
+    const int64_t retry_num,
+    const share::ObTransferTaskID &task_id,
+    const int result,
+    const int64_t start_ts,
+    const bool is_report)
+{
+  int ret = OB_SUCCESS;
+  ObArenaAllocator alloc;
+  ObTransferPerfDiagInfo info;
+  ObStorageHACostAccumItem accum_item;
+  ObStorageHATimestampItem ts_item;
+  ObIStorageHACostItem *item = nullptr;
+  if (ObStorageHACostItemType::ACCUM_COST_TYPE == params.item_type_) {
+    item = &accum_item;
+    static_cast<ObStorageHACostAccumItem *>(item)->cost_time_ = timestamp;
+  } else {
+    item = &ts_item;
+    static_cast<ObStorageHATimestampItem *>(item)->timestamp_ = timestamp;
+  }
+  item->name_ = params.name_;
+  item->type_ = params.item_type_;
+  item->retry_id_ = retry_num + 1;
+  share::ObStorageHADiagTaskKey key;
+  if (OB_FAIL(info.init(&alloc, MTL_ID()))) {
+    LOG_WARN("fail to init info", K(ret));
+  } else if (OB_FAIL(info.add_item(*item))) {
+    LOG_WARN("fail to add item", K(ret), KPC(item));
+  } else if (OB_FAIL(ObStorageHADiagMgr::construct_diagnose_info_key(
+      task_id, ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE,
+      params.task_type_, ObStorageHADiagType::PERF_DIAGNOSE, retry_num + 1, params.tablet_id_, key))) {
+    LOG_WARN("failed to construct error diagnose info key",
+        K(ret), K(task_id), K(retry_num), K(params.tablet_id_));
+  } else if (OB_FAIL(ObStorageHADiagMgr::construct_diagnose_info(task_id, params.dest_ls_id_,
+      params.task_type_, retry_num + 1, result, ObStorageHADiagModule::TRANSFER_PERF_DIAGNOSE, info))) {
+    LOG_WARN("failed to construct diagnose info", K(ret), K(params), K(retry_num), K(result));
+  } else if (OB_FAIL(ObStorageHADiagMgr::append_perf_diagnose_info(params.tablet_id_, info))) {
+    LOG_WARN("fail to append diagnose info", K(ret), K(params.tablet_id_));
+  } else if (OB_FAIL(ObStorageHADiagMgr::add_transfer_perf_diagnose_info(
+        key, start_ts, params.tablet_count_, is_report, info))) {
+    LOG_WARN("failed to add perf diagnose info", K(ret),
+        K(key), K(info), K(timestamp), K(start_ts), K(is_report));
+  }
+  return ret;
+}
+
+int ObTransferRelatedInfo::record_perf_diagnose_info_in_replay(
+    const share::ObStorageHAPerfDiagParams &params,
+    const int result,
+    const uint64_t timestamp,
+    const int64_t start_ts,
+    const bool is_report)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not inited", K(ret));
+  } else if (!params.is_valid()
+      || start_ts < 0) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(params), K(start_ts));
+  } else {
+    common::SpinRLockGuard guard(lock_);
+    int64_t log_replay_num = 0;
+    if (OB_FAIL(get_replay_retry_num_(params.task_type_, false/*inc_retry_num*/, log_replay_num))) {
+      LOG_WARN("fail to inc retry num", K(ret), K(params.task_type_), K(log_replay_num));
+    } else if (OB_FAIL(construct_perf_diag_info_(params, timestamp,
+        log_replay_num, params.task_id_, result, start_ts, is_report))) {
+      LOG_WARN("fail to construct perf diag info", K(ret), K(params), K(timestamp),
+          K(log_replay_num), K(result), K(start_ts), K(is_report));
+    }
+  }
+  return ret;
+}
+
+int ObTransferRelatedInfo::record_perf_diagnose_info_in_backfill(
+    const share::ObStorageHAPerfDiagParams &params,
+    const share::SCN &log_sync_scn,
+    const int result_code,
+    const ObMigrationStatus &migration_status,
+    const uint64_t timestamp,
+    const int64_t start_ts,
+    const bool is_report)
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not inited", K(ret));
+  } else if (!params.is_valid()
+      ||!log_sync_scn.is_valid()
+      || migration_status < ObMigrationStatus::OB_MIGRATION_STATUS_NONE
+      || migration_status >= ObMigrationStatus::OB_MIGRATION_STATUS_MAX
+      || start_ts < 0) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", K(ret), K(log_sync_scn), K(start_ts), K(params), K(migration_status));
+  } else {
+    common::SpinWLockGuard guard(lock_);
+    if (log_sync_scn != get_start_scn_()) {
+      if (ObMigrationStatus::OB_MIGRATION_STATUS_NONE != migration_status) {
+        ret = OB_STATE_NOT_MATCH;
+        LOG_WARN("dest ls in migration", K(ret), K(migration_status));
+      } else {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_WARN("related_info already change", K(ret), K(start_scn_), K(log_sync_scn));
+      }
+    } else {
+      int64_t retry_num = 0;
+      if (OB_FAIL(map_.get_refactored(params.tablet_id_, retry_num))) {
+        if (OB_HASH_NOT_EXIST == ret) {
+          ret = OB_SUCCESS;
+        } else {
+          LOG_WARN("fail to get retry id from map", K(ret), K(params.tablet_id_));
+        }
+      }
+      if (OB_FAIL(ret)) {
+        //do nothing
+      } else if (OB_FAIL(construct_perf_diag_info_(params, timestamp,
+          retry_num, get_task_id_(), result_code, start_ts, is_report))) {
+        LOG_WARN("fail to construct perf diag info", K(ret), K(params), K(timestamp),
+            K(retry_num), K(result_code), K(start_ts), K(is_report));
+      }
+    }
+  }
+  return ret;
+}
+
+int ObTransferRelatedInfo::get_related_info_task_id(share::ObTransferTaskID &task_id) const
+{
+  int ret = OB_SUCCESS;
+  task_id.reset();
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("ls transfer handler do not init", K(ret));
+  } else {
+    common::SpinRLockGuard guard(lock_);
+    task_id = get_task_id_();
+  }
+  return ret;
+}

src/storage/high_availability/ob_transfer_struct.h

@@ -21,6 +21,7 @@
 #include "storage/memtable/ob_memtable.h"
 #include "storage/tablet/ob_tablet_meta.h"
 #include "storage/tablet/ob_tablet_create_delete_mds_user_data.h"
+#include "share/ob_storage_ha_diagnose_struct.h"
 
 namespace oceanbase
 {
@@ -230,6 +231,84 @@ public:
   ObSqlString comment_;
 };
 
+class ObTransferRelatedInfo final
+{
+public:
+  ObTransferRelatedInfo();
+  ~ObTransferRelatedInfo();
+  int init();
+  bool is_valid() const;
+  void reset();
+  void destroy();
+  int set_info(
+      const share::ObTransferTaskID &task_id,
+      const share::SCN &start_scn);
+  int record_error_diagnose_info_in_replay(
+      const share::ObTransferTaskID &task_id,
+      const share::ObLSID &dest_ls_id,
+      const int result_code,
+      const bool clean_related_info,
+      const share::ObStorageHADiagTaskType type,
+      const share::ObStorageHACostItemName result_msg);
+  int record_error_diagnose_info_in_backfill(
+      const share::SCN &log_sync_scn,
+      const share::ObLSID &dest_ls_id,
+      const int result_code,
+      const ObTabletID &tablet_id,
+      const ObMigrationStatus &migration_status,
+      const share::ObStorageHACostItemName result_msg);
+
+  int record_perf_diagnose_info_in_replay(
+      const share::ObStorageHAPerfDiagParams &params,
+      const int result,
+      const uint64_t timestamp,
+      const int64_t start_ts,
+      const bool is_report);
+
+  int record_perf_diagnose_info_in_backfill(
+      const share::ObStorageHAPerfDiagParams &params,
+      const share::SCN &log_sync_scn,
+      const int result_code,
+      const ObMigrationStatus &migration_status,
+      const uint64_t timestamp,
+      const int64_t start_ts,
+      const bool is_report);
+  int get_related_info_task_id(share::ObTransferTaskID &task_id) const;
+  int reset(const share::ObTransferTaskID &task_id);
+  TO_STRING_KV(K_(task_id), K_(start_scn), K_(start_out_log_replay_num),
+      K_(start_in_log_replay_num), K_(finish_out_log_replay_num), K_(finish_in_log_replay_num));
+  typedef hash::ObHashMap<common::ObTabletID, int64_t, hash::NoPthreadDefendMode> TxBackfillStatMap;
+
+private:
+  void reset_();
+  int inc_tx_backfill_retry_num_(const ObTabletID &id, int64_t &retry_num);
+  const share::ObTransferTaskID &get_task_id_() const;
+  const share::SCN &get_start_scn_() const;
+  int get_replay_retry_num_(
+      const share::ObStorageHADiagTaskType type, const bool inc_retry_num, int64_t &retry_num);
+  int construct_perf_diag_info_(
+      const share::ObStorageHAPerfDiagParams &params,
+      const uint64_t timestamp,
+      const int64_t retry_num,
+      const share::ObTransferTaskID &task_id,
+      const int result,
+      const int64_t start_ts,
+      const bool is_report);
+
+private:
+  bool is_inited_;
+  share::ObTransferTaskID task_id_;
+  share::SCN start_scn_;
+  int64_t start_out_log_replay_num_;
+  int64_t start_in_log_replay_num_;
+  int64_t finish_out_log_replay_num_;
+  int64_t finish_in_log_replay_num_;
+  TxBackfillStatMap map_;
+  SpinRWLock lock_;
+
+  DISALLOW_COPY_AND_ASSIGN(ObTransferRelatedInfo);
+};
+
 }
 }
 #endif

src/storage/tablet/ob_tablet_finish_transfer_mds_helper.cpp

@@ -30,6 +30,8 @@
 #include "storage/high_availability/ob_rebuild_service.h"
 #include "storage/high_availability/ob_storage_ha_utils.h"
 #include "storage/high_availability/ob_transfer_service.h"
+#include "share/ob_storage_ha_diagnose_struct.h"
+#include "storage/high_availability/ob_storage_ha_diagnose_mgr.h"
 
 #define USING_LOG_PREFIX MDS
 
@@ -39,6 +41,7 @@ namespace storage
 {
 
 ERRSIM_POINT_DEF(EN_TRANSFER_NEED_REBUILD);
+ERRSIM_POINT_DEF(EN_TRANSFER_DIAGNOSE_FINISH_REPLAY_FAILED);
 
 int ObTabletFinishTransferUtil::check_transfer_table_replaced(
     ObTabletHandle &tablet_handle,
@@ -283,7 +286,8 @@ int ObTabletFinishTransferOutHelper::on_register(
   ObTXFinishTransferOutInfo tx_finish_transfer_out_info;
   int64_t pos = 0;
   ObTransferUtils::set_transfer_module();
-
+  const int64_t start_ts = ObTimeUtility::current_time();
+  share::ObStorageHACostItemName diagnose_result_msg = share::ObStorageHACostItemName::MAX_NAME;
   if (OB_ISNULL(buf) || len < 0) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("on register finish transfer out get invalid argument", K(ret), KP(buf), K(len));
@@ -293,11 +297,32 @@ int ObTabletFinishTransferOutHelper::on_register(
     ret = OB_ERR_UNEXPECTED;
     LOG_WARN("tx finish transfer out info is unexpected", K(ret), K(tx_finish_transfer_out_info));
   } else if (CLICK_FAIL(on_register_success_(tx_finish_transfer_out_info, ctx))) {
+    diagnose_result_msg = share::ObStorageHACostItemName::ON_REGISTER_SUCCESS;
     LOG_WARN("failed to on register", K(ret), K(tx_finish_transfer_out_info));
   } else if (CLICK_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_empty_shell_trigger(tx_finish_transfer_out_info.src_ls_id_))) {
     LOG_WARN("failed to set_tablet_empty_shell_trigger", K(ret), K(tx_finish_transfer_out_info));
   }
   ObTransferUtils::clear_transfer_module();
+#ifdef ERRSIM
+  if (OB_SUCC(ret)) {
+    ret = EN_TRANSFER_DIAGNOSE_FINISH_REPLAY_FAILED ? : OB_SUCCESS;
+    if (OB_FAIL(ret)) {
+      STORAGE_LOG(WARN, "fake EN_TRANSFER_DIAGNOSE_FINISH_REPLAY_FAILED", K(ret));
+    }
+  }
+#endif
+  const int64_t end_ts = ObTimeUtility::current_time();
+  bool is_report = OB_SUCCESS == ret ? true : false;
+  ObTransferUtils::process_finish_out_perf_diag_info(tx_finish_transfer_out_info,
+      ObStorageHACostItemType::ACCUM_COST_TYPE, ObStorageHACostItemName::FINISH_TRANSFER_OUT_REPLAY_COST,
+      ret, end_ts - start_ts, start_ts, is_report);
+  ObTransferUtils::add_transfer_error_diagnose_in_replay(
+                                   tx_finish_transfer_out_info.task_id_,
+                                   tx_finish_transfer_out_info.dest_ls_id_,
+                                   ret,
+                                   false/*clean_related_info*/,
+                                   ObStorageHADiagTaskType::TRANSFER_FINISH_OUT,
+                                   diagnose_result_msg);
   return ret;
 }
 
@@ -312,6 +337,9 @@ int ObTabletFinishTransferOutHelper::on_register_success_(
   ObLS *ls = NULL;
   const share::ObLSID &src_ls_id = tx_finish_transfer_out_info.src_ls_id_;
   const int64_t start_ts = ObTimeUtil::current_time();
+  ObTransferUtils::process_finish_out_perf_diag_info(tx_finish_transfer_out_info,
+      ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::FINISH_TRANSFER_OUT_FIRST_REPALY_LOG_TIMESTAMP,
+      ret, start_ts, start_ts, false/*is_report*/);
   LOG_INFO("[TRANSFER] start tx finish transfer out on_register_success_", K(tx_finish_transfer_out_info));
 #ifdef ERRSIM
   SERVER_EVENT_ADD("transfer", "tx_finish_transfer_out",
@@ -464,7 +492,8 @@ int ObTabletFinishTransferOutHelper::on_replay(
   ObTXFinishTransferOutInfo tx_finish_transfer_out_info;
   int64_t pos = 0;
   ObTransferUtils::set_transfer_module();
-
+  const int64_t start_ts = ObTimeUtility::current_time();
+  share::ObStorageHACostItemName diagnose_result_msg = share::ObStorageHACostItemName::MAX_NAME;
   if (OB_ISNULL(buf) || len < 0 || !scn.is_valid()) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("on replay finish transfer out get invalid argument", K(ret), KP(buf), K(len), K(scn));
@@ -474,11 +503,32 @@ int ObTabletFinishTransferOutHelper::on_replay(
     ret = OB_ERR_UNEXPECTED;
     LOG_WARN("tx finish transfer out info is unexpected", K(ret), K(tx_finish_transfer_out_info));
   } else if (CLICK_FAIL(on_replay_success_(scn, tx_finish_transfer_out_info, ctx))) {
+    diagnose_result_msg = share::ObStorageHACostItemName::ON_REPLAY_SUCCESS;
     LOG_WARN("failed to do on_replay_success_", K(ret), K(tx_finish_transfer_out_info));
   } else if (CLICK_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_empty_shell_trigger(tx_finish_transfer_out_info.src_ls_id_))) {
     LOG_WARN("failed to set_tablet_empty_shell_trigger", K(ret), K(tx_finish_transfer_out_info));
   }
   ObTransferUtils::clear_transfer_module();
+#ifdef ERRSIM
+  if (OB_SUCC(ret)) {
+    ret = EN_TRANSFER_DIAGNOSE_FINISH_REPLAY_FAILED ? : OB_SUCCESS;
+    if (OB_FAIL(ret)) {
+      STORAGE_LOG(WARN, "fake EN_TRANSFER_DIAGNOSE_FINISH_REPLAY_FAILED", K(ret));
+    }
+  }
+#endif
+  const int64_t end_ts = ObTimeUtility::current_time();
+  bool is_report = OB_SUCCESS == ret ? true : false;
+  ObTransferUtils::process_finish_out_perf_diag_info(tx_finish_transfer_out_info,
+      ObStorageHACostItemType::ACCUM_COST_TYPE, ObStorageHACostItemName::FINISH_TRANSFER_OUT_REPLAY_COST,
+      ret, end_ts - start_ts, start_ts, is_report);
+  ObTransferUtils::add_transfer_error_diagnose_in_replay(
+                                   tx_finish_transfer_out_info.task_id_,
+                                   tx_finish_transfer_out_info.dest_ls_id_,
+                                   ret,
+                                   true/*clean_related_info*/,
+                                   ObStorageHADiagTaskType::TRANSFER_FINISH_OUT,
+                                   diagnose_result_msg);
   return ret;
 }
 
@@ -493,7 +543,12 @@ int ObTabletFinishTransferOutHelper::on_replay_success_(
   ObLSService *ls_svr = NULL;
   ObLSHandle ls_handle;
   ObLS *ls = NULL;
-
+  ObTransferUtils::process_finish_out_perf_diag_info(tx_finish_transfer_out_info,
+      ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::FINISH_TRANSFER_OUT_LOG_SCN,
+      ret, scn.get_val_for_logservice(), start_ts, false/*is_report*/);
+  ObTransferUtils::process_finish_out_perf_diag_info(tx_finish_transfer_out_info,
+      ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::FINISH_TRANSFER_OUT_FIRST_REPALY_LOG_TIMESTAMP,
+      ret, start_ts, start_ts, false/*is_report*/);
   FLOG_INFO("[TRANSFER] start tx finish transfer out on_replay_success_", K(scn), K(tx_finish_transfer_out_info));
 #ifdef ERRSIM
   SERVER_EVENT_ADD("transfer", "tx_finish_transfer_out",
@@ -867,7 +922,8 @@ int ObTabletFinishTransferInHelper::on_register(
   int64_t pos = 0;
   const bool for_replay = false;
   ObTransferUtils::set_transfer_module();
-
+  const int64_t start_ts = ObTimeUtility::current_time();
+  share::ObStorageHACostItemName diagnose_result_msg = ObStorageHACostItemName::MAX_NAME;
   if (OB_ISNULL(buf) || len < 0) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("on register finish transfer in get invalid argument", K(ret), KP(buf), K(len));
@@ -877,10 +933,31 @@ int ObTabletFinishTransferInHelper::on_register(
     ret = OB_ERR_UNEXPECTED;
     LOG_WARN("tx finish transfer in info is unexpected", K(ret), K(tx_finish_transfer_in_info));
   } else if (CLICK_FAIL(on_register_success_(tx_finish_transfer_in_info, ctx))) {
+    diagnose_result_msg = share::ObStorageHACostItemName::ON_REGISTER_SUCCESS;
     LOG_WARN("failed to do on register success", K(ret), K(tx_finish_transfer_in_info));
   }
 
   ObTransferUtils::clear_transfer_module();
+#ifdef ERRSIM
+  if (OB_SUCC(ret)) {
+    ret = EN_TRANSFER_DIAGNOSE_FINISH_REPLAY_FAILED ? : OB_SUCCESS;
+    if (OB_FAIL(ret)) {
+      STORAGE_LOG(WARN, "fake EN_TRANSFER_DIAGNOSE_FINISH_REPLAY_FAILED", K(ret));
+    }
+  }
+#endif
+  const int64_t end_ts = ObTimeUtility::current_time();
+  bool is_report = OB_SUCCESS == ret ? true : false;
+  ObTransferUtils::process_finish_in_perf_diag_info(tx_finish_transfer_in_info,
+      ObStorageHACostItemType::ACCUM_COST_TYPE, ObStorageHACostItemName::FINISH_TRANSFER_IN_REPLAY_COST,
+      ret, end_ts - start_ts, 0/*start_ts*/, is_report);
+  ObTransferUtils::add_transfer_error_diagnose_in_replay(
+                                   tx_finish_transfer_in_info.task_id_,
+                                   tx_finish_transfer_in_info.dest_ls_id_,
+                                   ret,
+                                   false/*clean_related_info*/,
+                                   ObStorageHADiagTaskType::TRANSFER_FINISH_IN,
+                                   diagnose_result_msg);
   return ret;
 }
 
@@ -895,6 +972,9 @@ int ObTabletFinishTransferInHelper::on_register_success_(
   ObLS *ls = NULL;
   const share::ObLSID &dest_ls_id = tx_finish_transfer_in_info.dest_ls_id_;
   const int64_t start_ts = ObTimeUtil::current_time();
+  ObTransferUtils::process_finish_in_perf_diag_info(tx_finish_transfer_in_info,
+      ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::FINISH_TRANSFER_IN_FIRST_REPALY_LOG_TIMESTAMP,
+      ret, start_ts, start_ts, false/*is_report*/);
   LOG_INFO("[TRANSFER] start tx finish transfer in on_register_success_", K(tx_finish_transfer_in_info));
 #ifdef ERRSIM
   SERVER_EVENT_ADD("transfer", "tx_finish_transfer_in",
@@ -1083,7 +1163,8 @@ int ObTabletFinishTransferInHelper::on_replay(
   bool skip_replay = false;
   const bool for_replay = true;
   ObTransferUtils::set_transfer_module();
-
+  const int64_t start_ts = ObTimeUtility::current_time();
+  share::ObStorageHACostItemName diagnose_result_msg = share::ObStorageHACostItemName::MAX_NAME;
   if (OB_ISNULL(buf) || len < 0 || !scn.is_valid()) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("on_replay_success_ finish transfer in get invalid argument", K(ret), KP(buf), K(len), K(scn));
@@ -1093,9 +1174,30 @@ int ObTabletFinishTransferInHelper::on_replay(
     ret = OB_ERR_UNEXPECTED;
     LOG_WARN("tx finish transfer in info is unexpected", K(ret), K(tx_finish_transfer_in_info));
   } else if (CLICK_FAIL(on_replay_success_(scn, tx_finish_transfer_in_info, ctx))) {
+    diagnose_result_msg = share::ObStorageHACostItemName::ON_REPLAY_SUCCESS;
     LOG_WARN("failed to do on_replay_success_", K(ret), K(tx_finish_transfer_in_info));
   }
   ObTransferUtils::clear_transfer_module();
+#ifdef ERRSIM
+  if (OB_SUCC(ret)) {
+    ret = EN_TRANSFER_DIAGNOSE_FINISH_REPLAY_FAILED ? : OB_SUCCESS;
+    if (OB_FAIL(ret)) {
+      STORAGE_LOG(WARN, "fake EN_TRANSFER_DIAGNOSE_FINISH_REPLAY_FAILED", K(ret));
+    }
+  }
+#endif
+  const int64_t end_ts = ObTimeUtility::current_time();
+  bool is_report = OB_SUCCESS == ret ? true : false;
+  ObTransferUtils::process_finish_in_perf_diag_info(tx_finish_transfer_in_info,
+      ObStorageHACostItemType::ACCUM_COST_TYPE, ObStorageHACostItemName::FINISH_TRANSFER_IN_REPLAY_COST,
+      ret, end_ts - start_ts, 0/*start_ts*/, is_report);
+  ObTransferUtils::add_transfer_error_diagnose_in_replay(
+                                   tx_finish_transfer_in_info.task_id_,
+                                   tx_finish_transfer_in_info.dest_ls_id_,
+                                   ret,
+                                   false/*clean_related_info*/,
+                                   ObStorageHADiagTaskType::TRANSFER_FINISH_IN,
+                                   diagnose_result_msg);
   return ret;
 }
 
@@ -1107,6 +1209,12 @@ int ObTabletFinishTransferInHelper::on_replay_success_(
   MDS_TG(1_s);
   int ret = OB_SUCCESS;
   const int64_t start_ts = ObTimeUtil::current_time();
+  ObTransferUtils::process_finish_in_perf_diag_info(tx_finish_transfer_in_info,
+      ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::FINISH_TRANSFER_IN_LOG_SCN,
+      ret, scn.get_val_for_logservice(), start_ts, false/*is_report*/);
+  ObTransferUtils::process_finish_in_perf_diag_info(tx_finish_transfer_in_info,
+      ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::FINISH_TRANSFER_IN_FIRST_REPALY_LOG_TIMESTAMP,
+      ret, start_ts, start_ts, false/*is_report*/);
   LOG_INFO("[TRANSFER] start tx finish transfer in on_replay_success_", K(scn), K(tx_finish_transfer_in_info));
 #ifdef ERRSIM
   SERVER_EVENT_ADD("transfer", "tx_finish_transfer_in",

src/storage/tablet/ob_tablet_start_transfer_mds_helper.cpp

@@ -30,6 +30,8 @@
 #include "storage/high_availability/ob_transfer_service.h"
 #include "storage/high_availability/ob_rebuild_service.h"
 #include "storage/high_availability/ob_storage_ha_utils.h"
+#include "share/ob_storage_ha_diagnose_struct.h"
+#include "storage/high_availability/ob_storage_ha_diagnose_mgr.h"
 #include "storage/tx/ob_multi_data_source.h"
 #define USING_LOG_PREFIX MDS
 
@@ -41,6 +43,7 @@ using namespace oceanbase::transaction;
 
 ERRSIM_POINT_DEF(EN_CREATE_TRANSFER_IN_TABLET_FAILED);
 
+ERRSIM_POINT_DEF(EN_TRANSFER_DIAGNOSE_START_REPLAY_FAILED);
 /******************ObTabletStartTransferOutReplayExecutor*********************/
 class ObTabletStartTransferOutReplayExecutor final : public logservice::ObTabletReplayExecutor
 {
@@ -219,7 +222,8 @@ int ObTabletStartTransferOutHelper::on_register(
   int64_t pos = 0;
   const bool for_replay = false;
   ObTransferUtils::set_transfer_module();
-
+  const int64_t start_ts = ObTimeUtility::current_time();
+  share::ObStorageHACostItemName diagnose_result_msg = share::ObStorageHACostItemName::MAX_NAME;
   if (OB_ISNULL(buf) || len < 0) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("on register start transfer out get invalid argument", K(ret), KP(buf), K(len));
@@ -229,9 +233,30 @@ int ObTabletStartTransferOutHelper::on_register(
     ret = OB_ERR_UNEXPECTED;
     LOG_WARN("tx start transfer out info is unexpected", K(ret), K(tx_start_transfer_out_info));
   } else if (CLICK_FAIL(on_register_success_(tx_start_transfer_out_info, ctx))) {
+    diagnose_result_msg = share::ObStorageHACostItemName::ON_REGISTER_SUCCESS;
     LOG_WARN("failed to on register", K(ret), K(tx_start_transfer_out_info));
   }
   ObTransferUtils::clear_transfer_module();
+#ifdef ERRSIM
+  if (OB_SUCC(ret)) {
+    ret = EN_TRANSFER_DIAGNOSE_START_REPLAY_FAILED ? : OB_SUCCESS;
+    if (OB_FAIL(ret)) {
+      STORAGE_LOG(WARN, "fake EN_TRANSFER_DIAGNOSE_START_REPLAY_FAILED", K(ret));
+    }
+  }
+#endif
+  const int64_t end_ts = ObTimeUtility::current_time();
+  bool is_report = OB_SUCCESS == ret ? true : false;
+  ObTransferUtils::process_start_out_perf_diag_info(tx_start_transfer_out_info,
+      ObStorageHACostItemType::ACCUM_COST_TYPE, ObStorageHACostItemName::START_TRANSFER_OUT_REPLAY_COST,
+      ret, end_ts - start_ts, start_ts, is_report);
+  ObTransferUtils::add_transfer_error_diagnose_in_replay(
+                                   tx_start_transfer_out_info.task_id_,
+                                   tx_start_transfer_out_info.dest_ls_id_,
+                                   ret,
+                                   false/*clean_related_info*/,
+                                   ObStorageHADiagTaskType::TRANSFER_START_OUT,
+                                   diagnose_result_msg);
   return ret;
 }
 
@@ -246,6 +271,9 @@ int ObTabletStartTransferOutHelper::on_register_success_(
   ObLS *ls = nullptr;
   const int64_t start_ts = ObTimeUtil::current_time();
   LOG_INFO("[TRANSFER] start tx start transfer out on_register_success_", K(tx_start_transfer_out_info));
+  ObTransferUtils::process_start_out_perf_diag_info(tx_start_transfer_out_info,
+      ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::START_TRANSFER_OUT_FIRST_REPALY_LOG_TIMESTAMP,
+      ret, start_ts, start_ts, false/*is_report*/);
 #ifdef ERRSIM
   SERVER_EVENT_ADD("transfer", "tx_start_transfer_out",
                    "stage", "on_register_success",
@@ -463,7 +491,8 @@ int ObTabletStartTransferOutHelper::on_replay(
   ObTxDataSourceType mds_op_type = ObTxDataSourceType::START_TRANSFER_OUT;
   ObTabletStartTransferOutCommonHelper transfer_out_helper(mds_op_type);
   ObTransferUtils::set_transfer_module();
-
+  const int64_t start_ts = ObTimeUtility::current_time();
+  share::ObStorageHACostItemName diagnose_result_msg = share::ObStorageHACostItemName::MAX_NAME;
   if (OB_ISNULL(buf) || len < 0 || !scn.is_valid()) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("on replay start transfer out get invalid argument", K(ret), KP(buf), K(len), K(scn));
@@ -484,6 +513,9 @@ int ObTabletStartTransferOutHelper::on_replay(
     }
 #endif
   }
+  ObTransferUtils::process_start_out_perf_diag_info(tx_start_transfer_out_info,
+      ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::START_TRANSFER_OUT_LOG_SCN,
+      ret, scn.get_val_for_logservice(), start_ts, false/*is_report*/);
 #ifdef ERRSIM
   SERVER_EVENT_SYNC_ADD("TRANSFER", "BEFORE_ON_REDO_START_TRANSFER_OUT",
                         "tenant_id", MTL_ID(),
@@ -494,6 +526,7 @@ int ObTabletStartTransferOutHelper::on_replay(
 #endif
   DEBUG_SYNC(BEFORE_ON_REDO_START_TRANSFER_OUT);
   if (CLICK() && FAILEDx(transfer_out_helper.on_replay_success_(scn, tx_start_transfer_out_info, ctx))) {
+    diagnose_result_msg = share::ObStorageHACostItemName::ON_REPLAY_SUCCESS;
     LOG_WARN("failed to on register_success_", K(ret), K(scn), K(tx_start_transfer_out_info));
   }
 #ifdef ERRSIM
@@ -506,6 +539,26 @@ int ObTabletStartTransferOutHelper::on_replay(
 #endif
   DEBUG_SYNC(AFTER_ON_REDO_START_TRANSFER_OUT);
   ObTransferUtils::clear_transfer_module();
+#ifdef ERRSIM
+  if (OB_SUCC(ret)) {
+    ret = EN_TRANSFER_DIAGNOSE_START_REPLAY_FAILED ? : OB_SUCCESS;
+    if (OB_FAIL(ret)) {
+      STORAGE_LOG(WARN, "fake EN_TRANSFER_DIAGNOSE_START_REPLAY_FAILED", K(ret));
+    }
+  }
+#endif
+  const int64_t end_ts = ObTimeUtility::current_time();
+  bool is_report = OB_SUCCESS == ret ? true : false;
+  ObTransferUtils::process_start_out_perf_diag_info(tx_start_transfer_out_info,
+      ObStorageHACostItemType::ACCUM_COST_TYPE, ObStorageHACostItemName::START_TRANSFER_OUT_REPLAY_COST,
+      ret, end_ts - start_ts, start_ts, is_report);
+  ObTransferUtils::add_transfer_error_diagnose_in_replay(
+                                   tx_start_transfer_out_info.task_id_,
+                                   tx_start_transfer_out_info.dest_ls_id_,
+                                   ret,
+                                   false/*clean_related_info*/,
+                                   ObStorageHADiagTaskType::TRANSFER_START_OUT,
+                                   diagnose_result_msg);
   return ret;
 }
 
@@ -623,6 +676,9 @@ int ObTabletStartTransferOutCommonHelper::on_replay_success_(
   ObLS *ls = nullptr;
   const int64_t start_ts = ObTimeUtil::current_time();
   LOG_INFO("[TRANSFER] start tx start transfer out on_replay_success_", K(scn), K(tx_start_transfer_out_info));
+  ObTransferUtils::process_start_out_perf_diag_info(tx_start_transfer_out_info,
+      ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::START_TRANSFER_OUT_FIRST_REPALY_LOG_TIMESTAMP,
+      ret, start_ts, start_ts, false/*is_report*/);
 #ifdef ERRSIM
   SERVER_EVENT_ADD("transfer", "tx_start_transfer_out",
                    "stage", "on_replay_success",
@@ -922,8 +978,9 @@ int ObTabletStartTransferInHelper::on_register(
   int ret = OB_SUCCESS;
   ObTXStartTransferInInfo tx_start_transfer_in_info;
   int64_t pos = 0;
-
+  share::ObStorageHACostItemName diagnose_result_msg = share::ObStorageHACostItemName::MAX_NAME;
   ObTransferUtils::set_transfer_module();
+  const int64_t start_ts = ObTimeUtility::current_time();
   if (OB_ISNULL(buf) || len < 0) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("on register start transfer in get invalid argument", K(ret), KP(buf), K(len));
@@ -932,12 +989,39 @@ int ObTabletStartTransferInHelper::on_register(
   } else if (!tx_start_transfer_in_info.is_valid()) {
     ret = OB_ERR_UNEXPECTED;
     LOG_WARN("tx start transfer in info is unexpected", K(ret), K(tx_start_transfer_in_info));
-  } else if (CLICK_FAIL(on_register_success_(tx_start_transfer_in_info, ctx))) {
-    LOG_WARN("failed to do on register success", K(ret), K(tx_start_transfer_in_info));
-  } else if (CLICK_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_gc_trigger(tx_start_transfer_in_info.dest_ls_id_))) {
-    LOG_WARN("failed to set_tablet_gc_trigger", K(ret), K(tx_start_transfer_in_info));
+  } else {
+    ObTransferUtils::set_transfer_related_info(
+                                     tx_start_transfer_in_info.dest_ls_id_,
+                                     tx_start_transfer_in_info.task_id_,
+                                     tx_start_transfer_in_info.start_scn_);
+    if (CLICK_FAIL(on_register_success_(tx_start_transfer_in_info, ctx))) {
+      diagnose_result_msg = share::ObStorageHACostItemName::ON_REGISTER_SUCCESS;
+      LOG_WARN("failed to do on register success", K(ret), K(tx_start_transfer_in_info));
+    } else if (CLICK_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_gc_trigger(tx_start_transfer_in_info.dest_ls_id_))) {
+      LOG_WARN("failed to set_tablet_gc_trigger", K(ret), K(tx_start_transfer_in_info));
+    }
   }
   ObTransferUtils::clear_transfer_module();
+#ifdef ERRSIM
+  if (OB_SUCC(ret)) {
+    ret = EN_TRANSFER_DIAGNOSE_START_REPLAY_FAILED ? : OB_SUCCESS;
+    if (OB_FAIL(ret)) {
+      STORAGE_LOG(WARN, "fake EN_TRANSFER_DIAGNOSE_START_REPLAY_FAILED", K(ret));
+    }
+  }
+#endif
+  const int64_t end_ts = ObTimeUtility::current_time();
+  bool is_report = OB_SUCCESS == ret ? true : false;
+  ObTransferUtils::process_start_in_perf_diag_info(tx_start_transfer_in_info,
+      ObStorageHACostItemType::ACCUM_COST_TYPE, ObStorageHACostItemName::START_TRANSFER_IN_REPLAY_COST,
+      ret, end_ts - start_ts, start_ts, is_report);
+  ObTransferUtils::add_transfer_error_diagnose_in_replay(
+                                   tx_start_transfer_in_info.task_id_,
+                                   tx_start_transfer_in_info.dest_ls_id_,
+                                   ret,
+                                   false/*clean_related_info*/,
+                                   ObStorageHADiagTaskType::TRANSFER_START_IN,
+                                   diagnose_result_msg);
   return ret;
 }
 
@@ -955,6 +1039,9 @@ int ObTabletStartTransferInHelper::on_register_success_(
   const int64_t start_ts = ObTimeUtil::current_time();
   LOG_INFO("[TRANSFER] start tx start transfer in on_register_success_",
       K(tx_start_transfer_in_info));
+  ObTransferUtils::process_start_in_perf_diag_info(tx_start_transfer_in_info,
+      ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::START_TRANSFER_IN_FIRST_REPALY_LOG_TIMESTAMP,
+      ret, start_ts, start_ts, false/*is_report*/);
   if (!tx_start_transfer_in_info.is_valid()) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("on register success get invalid argument", K(ret), K(tx_start_transfer_in_info));
@@ -1103,7 +1190,6 @@ int ObTabletStartTransferInHelper::check_can_skip_replay_(
   ObLSHandle ls_handle;
   ObLSService *ls_service = nullptr;
   ObLS *ls = nullptr;
-
   if (!scn.is_valid() || !tx_start_transfer_in_info.is_valid()) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("check can skip replay start transfer in get invalid argument", K(ret), K(scn), K(tx_start_transfer_in_info));
@@ -1494,7 +1580,8 @@ int ObTabletStartTransferInHelper::on_replay(
   bool skip_replay = false;
   ObTransferService *transfer_service = nullptr;
   ObTransferUtils::set_transfer_module();
-
+  const int64_t start_ts = ObTimeUtility::current_time();
+  share::ObStorageHACostItemName diagnose_result_msg = share::ObStorageHACostItemName::MAX_NAME;
   if (OB_ISNULL(buf) || len < 0) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("on replay start transfer in get invalid argument", K(ret), KP(buf), K(len));
@@ -1510,18 +1597,45 @@ int ObTabletStartTransferInHelper::on_replay(
     LOG_WARN("failed to check can skip replay", K(ret), K(tx_start_transfer_in_info));
   } else if (skip_replay) {
     FLOG_INFO("skip replay start transfer in", K(scn), K(tx_start_transfer_in_info));
-  } else if (CLICK_FAIL(on_replay_success_(scn, tx_start_transfer_in_info, ctx))) {
-    LOG_WARN("failed to do on_replay_success_", K(ret), K(tx_start_transfer_in_info));
-  } else if (CLICK_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_gc_trigger(tx_start_transfer_in_info.dest_ls_id_))) {
-    LOG_WARN("failed to set_tablet_gc_trigger", K(ret), K(tx_start_transfer_in_info));
   } else {
-    transfer_service->wakeup();
+    ObTransferUtils::set_transfer_related_info(
+                                     tx_start_transfer_in_info.dest_ls_id_,
+                                     tx_start_transfer_in_info.task_id_,
+                                     tx_start_transfer_in_info.start_scn_);
+    if (CLICK_FAIL(on_replay_success_(scn, tx_start_transfer_in_info, ctx))) {
+      diagnose_result_msg = share::ObStorageHACostItemName::ON_REPLAY_SUCCESS;
+      LOG_WARN("failed to do on_replay_success_", K(ret), K(tx_start_transfer_in_info));
+    } else if (CLICK_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_gc_trigger(tx_start_transfer_in_info.dest_ls_id_))) {
+      LOG_WARN("failed to set_tablet_gc_trigger", K(ret), K(tx_start_transfer_in_info));
+    } else {
+      transfer_service->wakeup();
+    }
   }
 #ifdef ERRSIM
   SERVER_EVENT_SYNC_ADD("TRANSFER", "AFTER_ON_REDO_START_TRANSFER_IN");
 #endif
   DEBUG_SYNC(AFTER_ON_REDO_START_TRANSFER_IN);
   ObTransferUtils::clear_transfer_module();
+#ifdef ERRSIM
+  if (OB_SUCC(ret)) {
+    ret = EN_TRANSFER_DIAGNOSE_START_REPLAY_FAILED ? : OB_SUCCESS;
+    if (OB_FAIL(ret)) {
+      STORAGE_LOG(WARN, "fake EN_TRANSFER_DIAGNOSE_START_REPLAY_FAILED", K(ret));
+    }
+  }
+#endif
+  const int64_t end_ts = ObTimeUtility::current_time();
+  bool is_report = OB_SUCCESS == ret ? true : false;
+  ObTransferUtils::process_start_in_perf_diag_info(tx_start_transfer_in_info,
+      ObStorageHACostItemType::ACCUM_COST_TYPE, ObStorageHACostItemName::START_TRANSFER_IN_REPLAY_COST,
+      ret, end_ts - start_ts, start_ts, is_report);
+  ObTransferUtils::add_transfer_error_diagnose_in_replay(
+                                   tx_start_transfer_in_info.task_id_,
+                                   tx_start_transfer_in_info.dest_ls_id_,
+                                   ret,
+                                   false/*clean_related_info*/,
+                                   ObStorageHADiagTaskType::TRANSFER_START_IN,
+                                   diagnose_result_msg);
   return ret;
 }
 
@@ -1541,6 +1655,12 @@ int ObTabletStartTransferInHelper::on_replay_success_(
   const int64_t start_ts = ObTimeUtil::current_time();
   LOG_INFO("[TRANSFER] start tx start transfer in on_replay_success_",
       K(tx_start_transfer_in_info));
+  ObTransferUtils::process_start_in_perf_diag_info(tx_start_transfer_in_info,
+      ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::START_TRANSFER_IN_LOG_SCN,
+      ret, scn.get_val_for_logservice(), start_ts, false/*is_report*/);
+  ObTransferUtils::process_start_in_perf_diag_info(tx_start_transfer_in_info,
+      ObStorageHACostItemType::FLUENT_TIMESTAMP_TYPE, ObStorageHACostItemName::START_TRANSFER_IN_FIRST_REPALY_LOG_TIMESTAMP,
+      ret, start_ts, start_ts, false/*is_report*/);
 #ifdef ERRSIM
   SERVER_EVENT_ADD("transfer", "tx_start_transfer_in",
                    "stage", "on_replay_success",

tools/deploy/mysql_test/r/mysql/information_schema.result

@@ -2886,6 +2886,12 @@ select * from information_schema.statistics where table_schema in ('oceanbase',
 | def           | oceanbase    | __all_spatial_reference_systems           |          0 | oceanbase    | PRIMARY                                     |            1 | srs_id                  | A         |        NULL |     NULL | NULL   |          | BTREE      | VALID   |               | YES        | NULL       |
 | def           | oceanbase    | __all_spm_config                          |          0 | oceanbase    | PRIMARY                                     |            1 | tenant_id               | A         |        NULL |     NULL | NULL   |          | BTREE      | VALID   |               | YES        | NULL       |
 | def           | oceanbase    | __all_spm_config                          |          0 | oceanbase    | PRIMARY                                     |            2 | name                    | A         |        NULL |     NULL | NULL   |          | BTREE      | VALID   |               | YES        | NULL       |
+| def           | oceanbase    | __all_storage_ha_error_diagnose_history   |          0 | oceanbase    | PRIMARY                                     |            1 | gmt_create              | A         |        NULL |     NULL | NULL   |          | BTREE      | VALID   |               | YES        | NULL       |
+| def           | oceanbase    | __all_storage_ha_error_diagnose_history   |          0 | oceanbase    | PRIMARY                                     |            2 | svr_ip                  | A         |        NULL |     NULL | NULL   |          | BTREE      | VALID   |               | YES        | NULL       |
+| def           | oceanbase    | __all_storage_ha_error_diagnose_history   |          0 | oceanbase    | PRIMARY                                     |            3 | svr_port                | A         |        NULL |     NULL | NULL   |          | BTREE      | VALID   |               | YES        | NULL       |
+| def           | oceanbase    | __all_storage_ha_perf_diagnose_history    |          0 | oceanbase    | PRIMARY                                     |            1 | gmt_create              | A         |        NULL |     NULL | NULL   |          | BTREE      | VALID   |               | YES        | NULL       |
+| def           | oceanbase    | __all_storage_ha_perf_diagnose_history    |          0 | oceanbase    | PRIMARY                                     |            2 | svr_ip                  | A         |        NULL |     NULL | NULL   |          | BTREE      | VALID   |               | YES        | NULL       |
+| def           | oceanbase    | __all_storage_ha_perf_diagnose_history    |          0 | oceanbase    | PRIMARY                                     |            3 | svr_port                | A         |        NULL |     NULL | NULL   |          | BTREE      | VALID   |               | YES        | NULL       |
 | def           | oceanbase    | __all_sub_part                            |          0 | oceanbase    | PRIMARY                                     |            1 | tenant_id               | A         |        NULL |     NULL | NULL   |          | BTREE      | VALID   |               | YES        | NULL       |
 | def           | oceanbase    | __all_sub_part                            |          0 | oceanbase    | PRIMARY                                     |            2 | table_id                | A         |        NULL |     NULL | NULL   |          | BTREE      | VALID   |               | YES        | NULL       |
 | def           | oceanbase    | __all_sub_part                            |          0 | oceanbase    | PRIMARY                                     |            3 | part_id                 | A         |        NULL |     NULL | NULL   |          | BTREE      | VALID   |               | YES        | NULL       |

tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result

@@ -336,6 +336,7 @@ _force_skip_encoding_partition_id
 _global_enable_rich_vector_format
 _hash_area_size
 _hash_join_enabled
+_ha_diagnose_history_recycle_interval
 _ha_rpc_timeout
 _ha_tablet_info_batch_count
 _hidden_sys_tenant_memory

tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result

@@ -9011,6 +9011,49 @@ is_deleted	bigint(20)	NO		NULL
 select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_column_group_mapping_history;
 IF(count(*) >= 0, 1, 0)
 1
+desc oceanbase.__all_virtual_storage_ha_error_diagnose;
+Field	Type	Null	Key	Default	Extra
+tenant_id	bigint(20)	NO		NULL	
+ls_id	bigint(20)	NO		NULL	
+module	varchar(64)	NO		NULL	
+type	varchar(64)	NO		NULL	
+task_id	varchar(64)	NO		NULL	
+svr_ip	varchar(46)	NO		NULL	
+svr_port	bigint(20)	NO		NULL	
+retry_id	bigint(20)	NO		NULL	
+create_time	timestamp(6)	NO		NULL	
+result_code	bigint(20)	NO		NULL	
+result_msg	varchar(1024)	NO		NULL	
+info	longtext	NO		NULL	
+select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_storage_ha_error_diagnose;
+IF(count(*) >= 0, 1, 0)
+1
+"oceanbase.__all_virtual_storage_ha_error_diagnose runs in single server"
+IF(count(*) >= 0, 1, 0)
+1
+desc oceanbase.__all_virtual_storage_ha_perf_diagnose;
+Field	Type	Null	Key	Default	Extra
+tenant_id	bigint(20)	NO		NULL	
+ls_id	bigint(20)	NO		NULL	
+module	varchar(64)	NO		NULL	
+type	varchar(64)	NO		NULL	
+task_id	varchar(64)	NO		NULL	
+svr_ip	varchar(46)	NO		NULL	
+svr_port	bigint(20)	NO		NULL	
+retry_id	bigint(20)	NO		NULL	
+start_timestamp	timestamp(6)	NO		NULL	
+end_timestamp	timestamp(6)	NO		NULL	
+tablet_id	bigint(20)	NO		NULL	
+tablet_count	bigint(20)	NO		NULL	
+result_code	bigint(20)	NO		NULL	
+result_msg	varchar(1024)	NO		NULL	
+info	longtext	NO		NULL	
+select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_storage_ha_perf_diagnose;
+IF(count(*) >= 0, 1, 0)
+1
+"oceanbase.__all_virtual_storage_ha_perf_diagnose runs in single server"
+IF(count(*) >= 0, 1, 0)
+1
 desc oceanbase.__all_virtual_clone_job;
 Field	Type	Null	Key	Default	Extra
 tenant_id	bigint(20)	NO	PRI	NULL	

tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result

@@ -275,6 +275,8 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr
 478	__all_import_table_job_history	0	201001	1
 479	__all_import_table_task	0	201001	1
 480	__all_import_table_task_history	0	201001	1
+483	__all_storage_ha_error_diagnose_history	0	201001	1
+484	__all_storage_ha_perf_diagnose_history	0	201001	1
 485	__all_clone_job	0	201001	1
 486	__all_clone_job_history	0	201001	1
 493	__all_ncomp_dll	0	201001	1
@@ -704,6 +706,8 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr
 12430	__all_virtual_column_group_mapping	2	201001	1
 12431	__all_virtual_column_group_history	2	201001	1
 12432	__all_virtual_column_group_mapping_history	2	201001	1
+12433	__all_virtual_storage_ha_error_diagnose	2	201001	1
+12434	__all_virtual_storage_ha_perf_diagnose	2	201001	1
 12435	__all_virtual_clone_job	2	201001	1
 12436	__all_virtual_clone_job_history	2	201001	1
 12437	__all_virtual_checkpoint_diagnose_memtable_info	2	201001	1