[Net Standby] Modify standby db fetch log rpc timeout

src/logservice/restoreservice/ob_log_restore_handler.cpp

@@ -795,6 +795,11 @@ int ObLogRestoreHandler::check_restore_to_newest_from_service_(
       CLOG_LOG(INFO, "check_restore_to_newest succ", K(id_), K(archive_scn), K(end_scn));
     }
   }
+
+  // if connect to source tenant denied, rewrite ret_code
+  if (-ER_ACCESS_DENIED_ERROR == ret) {
+    ret = OB_PASSWORD_WRONG;
+  }
   return ret;
 }
 

src/logservice/restoreservice/ob_log_restore_net_driver.cpp

@@ -26,6 +26,7 @@
 #include "logservice/ob_log_service.h"      // ObLogService
 #include "share/restore/ob_log_restore_source.h"   // ObLogRestoreSourceType
 #include "logservice/logfetcher/ob_log_fetcher.h"  // ObLogFetcher
+#include "observer/omt/ob_tenant_config_mgr.h"  // tenant_config
 
 #include "lib/mysqlclient/ob_mysql_proxy.h"
 #include "lib/string/ob_sql_string.h"    // ObSqlString
@@ -216,6 +217,7 @@ int ObLogRestoreNetDriver::scan_ls(const share::ObLogRestoreSourceType &type)
     }
   }
   delete_fetcher_if_needed_with_lock_();
+  update_config_();
   return ret;
 }
 
@@ -393,6 +395,7 @@ int ObLogRestoreNetDriver::init_fetcher_if_needed_(const int64_t cluster_id, con
     const logfetcher::ClientFetchingMode fetching_mode = logfetcher::ClientFetchingMode::FETCHING_MODE_INTEGRATED;
     share::ObBackupPathString archive_dest;
     common::ObMySQLProxy *proxy = NULL;
+    cfg_.fetch_log_rpc_timeout_sec = get_rpc_timeout_sec_();
 
     if (OB_FAIL(proxy_.get_sql_proxy(proxy))) {
       LOG_WARN("get sql proxy failed");
@@ -432,6 +435,30 @@ void ObLogRestoreNetDriver::delete_fetcher_if_needed_with_lock_()
   }
 }
 
+void ObLogRestoreNetDriver::update_config_()
+{
+  if (NULL != fetcher_) {
+    const int64_t fetch_log_rpc_timeout_sec = get_rpc_timeout_sec_();
+    if (fetch_log_rpc_timeout_sec != cfg_.fetch_log_rpc_timeout_sec) {
+      cfg_.fetch_log_rpc_timeout_sec = fetch_log_rpc_timeout_sec;
+      fetcher_->configure(cfg_);
+    }
+  }
+}
+
+int64_t ObLogRestoreNetDriver::get_rpc_timeout_sec_()
+{
+  int64_t rpc_timeout = 0;
+  const int64_t DEFAULT_FETECH_LOG_RPC_TIMEOUT = 15;   // 15s
+  omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id_));
+  if (!tenant_config.is_valid()) {
+    rpc_timeout = DEFAULT_FETECH_LOG_RPC_TIMEOUT;
+  } else {
+    rpc_timeout = tenant_config->standby_db_fetch_log_rpc_timeout / 1000 / 1000L;
+  }
+  return rpc_timeout;
+}
+
 bool ObLogRestoreNetDriver::is_fetcher_stale_(const int64_t cluster_id, const uint64_t tenant_id)
 {
   bool bret = false;

src/logservice/restoreservice/ob_log_restore_net_driver.h

@@ -89,6 +89,8 @@ private:
   int refresh_fetcher_if_needed_(const share::ObRestoreSourceServiceAttr &source);
   int init_fetcher_if_needed_(const int64_t cluster_id, const uint64_t tenant_id);
   void delete_fetcher_if_needed_with_lock_();
+  void update_config_();
+  int64_t get_rpc_timeout_sec_();
   int refresh_proxy_(const share::ObRestoreSourceServiceAttr &source);
 
 

src/share/ob_errno.cpp

@@ -21735,11 +21735,11 @@ static const _error _error_OB_SOURCE_TENANT_STATE_NOT_MATCH = {
       .error_solution        = "Contact OceanBase Support",
       .mysql_errno           = -1,
       .sqlstate              = "HY000",
-      .str_error             = "log restore source tenant state not match when switchover to primary",
+      .str_error             = "log restore source tenant state not match, switchover to primary not allowed",
-      .str_user_error        = "log restore source tenant state not match when switchover to primary",
+      .str_user_error        = "log restore source tenant state not match, switchover to primary not allowed",
       .oracle_errno          = 600,
-      .oracle_str_error      = "ORA-00600: internal error code, arguments: -9088, log restore source tenant state not match when switchover to primary",
+      .oracle_str_error      = "ORA-00600: internal error code, arguments: -9088, log restore source tenant state not match, switchover to primary not allowed",
-      .oracle_str_user_error = "ORA-00600: internal error code, arguments: -9088, log restore source tenant state not match when switchover to primary"
+      .oracle_str_user_error = "ORA-00600: internal error code, arguments: -9088, log restore source tenant state not match, switchover to primary not allowed"
 };
 static const _error _error_OB_SOURCE_LS_STATE_NOT_MATCH = {
       .error_name            = "OB_SOURCE_LS_STATE_NOT_MATCH",
@@ -21747,11 +21747,11 @@ static const _error _error_OB_SOURCE_LS_STATE_NOT_MATCH = {
       .error_solution        = "Contact OceanBase Support",
       .mysql_errno           = -1,
       .sqlstate              = "HY000",
-      .str_error             = "log restore source ls state not match when switchover to primary",
+      .str_error             = "log restore source ls state not match, switchover to primary not allowed",
-      .str_user_error        = "log restore source ls state not match when switchover to primary",
+      .str_user_error        = "log restore source ls state not match, switchover to primary not allowed",
       .oracle_errno          = 600,
-      .oracle_str_error      = "ORA-00600: internal error code, arguments: -9089, log restore source ls state not match when switchover to primary",
+      .oracle_str_error      = "ORA-00600: internal error code, arguments: -9089, log restore source ls state not match, switchover to primary not allowed",
-      .oracle_str_user_error = "ORA-00600: internal error code, arguments: -9089, log restore source ls state not match when switchover to primary"
+      .oracle_str_user_error = "ORA-00600: internal error code, arguments: -9089, log restore source ls state not match, switchover to primary not allowed"
 };
 static const _error _error_OB_ESI_SESSION_NOT_EXIST = {
       .error_name            = "OB_ESI_SESSION_NOT_EXIST",

src/share/ob_errno.def

@@ -1993,8 +1993,8 @@ DEFINE_ERROR_DEP(OB_BACKUP_PWRITE_CONTENT_NOT_MATCH, -9084, -1, "HY000", "the co
 DEFINE_ERROR(OB_BACKUP_MAJOR_NOT_COVER_MINOR, -9085, -1, "HY000", "backup major sstable range not cover minor sstable");
 DEFINE_ERROR(OB_BACKUP_ADVANCE_CHECKPOINT_TIMEOUT, -9086, -1, "HY000", "backup advance checkpoint by flush timeout");
 DEFINE_ERROR(OB_CLOG_RECYCLE_BEFORE_ARCHIVE, -9087, -1, "HY000", "observer clog is recycled before archive");
-DEFINE_ERROR(OB_SOURCE_TENANT_STATE_NOT_MATCH, -9088, -1, "HY000", "log restore source tenant state not match when switchover to primary");
+DEFINE_ERROR(OB_SOURCE_TENANT_STATE_NOT_MATCH, -9088, -1, "HY000", "log restore source tenant state not match, switchover to primary not allowed");
-DEFINE_ERROR(OB_SOURCE_LS_STATE_NOT_MATCH, -9089, -1, "HY000", "log restore source ls state not match when switchover to primary");
+DEFINE_ERROR(OB_SOURCE_LS_STATE_NOT_MATCH, -9089, -1, "HY000", "log restore source ls state not match, switchover to primary not allowed");
 
 
 DEFINE_ERROR(OB_ESI_SESSION_NOT_EXIST, -9090, -1, "HY000", "obesi process session not exist");

src/share/ob_errno.h

@@ -3554,8 +3554,8 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219;
 #define OB_BACKUP_MAJOR_NOT_COVER_MINOR__USER_ERROR_MSG "backup major sstable range not cover minor sstable"
 #define OB_BACKUP_ADVANCE_CHECKPOINT_TIMEOUT__USER_ERROR_MSG "backup advance checkpoint by flush timeout"
 #define OB_CLOG_RECYCLE_BEFORE_ARCHIVE__USER_ERROR_MSG "observer clog is recycled before archive"
-#define OB_SOURCE_TENANT_STATE_NOT_MATCH__USER_ERROR_MSG "log restore source tenant state not match when switchover to primary"
+#define OB_SOURCE_TENANT_STATE_NOT_MATCH__USER_ERROR_MSG "log restore source tenant state not match, switchover to primary not allowed"
-#define OB_SOURCE_LS_STATE_NOT_MATCH__USER_ERROR_MSG "log restore source ls state not match when switchover to primary"
+#define OB_SOURCE_LS_STATE_NOT_MATCH__USER_ERROR_MSG "log restore source ls state not match, switchover to primary not allowed"
 #define OB_ESI_SESSION_NOT_EXIST__USER_ERROR_MSG "obesi process session not exist"
 #define OB_ALREADY_IN_ARCHIVE_MODE__USER_ERROR_MSG "Already in ARCHIVELOG mode"
 #define OB_ALREADY_IN_NOARCHIVE_MODE__USER_ERROR_MSG "Already in NOARCHIVELOG mode"
@@ -5633,8 +5633,8 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219;
 #define OB_BACKUP_MAJOR_NOT_COVER_MINOR__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9085, backup major sstable range not cover minor sstable"
 #define OB_BACKUP_ADVANCE_CHECKPOINT_TIMEOUT__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9086, backup advance checkpoint by flush timeout"
 #define OB_CLOG_RECYCLE_BEFORE_ARCHIVE__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9087, observer clog is recycled before archive"
-#define OB_SOURCE_TENANT_STATE_NOT_MATCH__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9088, log restore source tenant state not match when switchover to primary"
+#define OB_SOURCE_TENANT_STATE_NOT_MATCH__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9088, log restore source tenant state not match, switchover to primary not allowed"
-#define OB_SOURCE_LS_STATE_NOT_MATCH__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9089, log restore source ls state not match when switchover to primary"
+#define OB_SOURCE_LS_STATE_NOT_MATCH__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9089, log restore source ls state not match, switchover to primary not allowed"
 #define OB_ESI_SESSION_NOT_EXIST__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9090, obesi process session not exist"
 #define OB_ALREADY_IN_ARCHIVE_MODE__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9091, Already in ARCHIVELOG mode"
 #define OB_ALREADY_IN_NOARCHIVE_MODE__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9092, Already in NOARCHIVELOG mode"

src/share/parameter/ob_parameter_seed.ipp

@@ -591,6 +591,13 @@ DEF_TIME(log_storage_warning_tolerance_time, OB_CLUSTER_PARAMETER, "5s",
         "Range: [1s,300s]",
         ObParameterAttr(Section::LOGSERVICE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
 
+DEF_TIME(standby_db_fetch_log_rpc_timeout, OB_TENANT_PARAMETER, "15s",
+        "[2s,)",
+        "The threshold for detecting the RPC timeout for the standby tenant to fetch log from the log restore source tenant. "
+        "When the rpc timeout, the log transport service switches to another server of the log restore source tenant to fetch logs. "
+        "Range: [2s, +∞)",
+        ObParameterAttr(Section::LOGSERVICE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
+
 // ========================= LogService Config End   =====================
 DEF_INT(resource_hard_limit, OB_CLUSTER_PARAMETER, "100", "[100, 10000]",
         "system utilization should not be large than resource_hard_limit",

tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result

@@ -202,6 +202,7 @@ sql_work_area
 ssl_client_authentication
 ssl_external_kms_info
 stack_size
+standby_db_fetch_log_rpc_timeout
 standby_fetch_log_bandwidth_limit
 syslog_io_bandwidth_limit
 syslog_level