all_server_list improvement

src/rootserver/ob_root_service.cpp

@@ -648,6 +648,7 @@ ObRootService::ObRootService()
     snapshot_manager_(),
     core_meta_table_version_(0),
     update_rs_list_timer_task_(*this),
+    update_all_server_config_task_(*this),
     baseline_schema_version_(0),
     backup_service_(),
     backup_task_scheduler_(),
@@ -1606,6 +1607,26 @@ int ObRootService::schedule_update_rs_list_task()
   }
   return ret;
 }
+ERRSIM_POINT_DEF(ALL_SERVER_SCHEDULE_ERROR);
+int ObRootService::schedule_update_all_server_config_task()
+{
+  int ret = OB_SUCCESS;
+  const bool did_repeat = true;
+  if (OB_UNLIKELY(!inited_)) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not init", KR(ret), K(inited_));
+  } else if (task_queue_.exist_timer_task(update_all_server_config_task_)) {
+    LOG_WARN("already have one update rs list timer task , ignore this");
+  } else if (OB_FAIL(task_queue_.add_timer_task(
+      update_all_server_config_task_,
+      ALL_SERVER_SCHEDULE_ERROR ? (ObUpdateAllServerConfigTask::RETRY_INTERVAL / 2) : ObUpdateAllServerConfigTask::RETRY_INTERVAL,
+      did_repeat))) {
+    LOG_WARN("fail to add timer task", KR(ret));
+  } else {
+    LOG_INFO("add update server config task success");
+  }
+  return ret;
+}
 
 int ObRootService::schedule_load_ddl_task()
 {
@@ -5226,6 +5247,14 @@ int ObRootService::start_timer_tasks()
     }
   }
 
+  if (OB_SUCC(ret) && !task_queue_.exist_timer_task(update_all_server_config_task_)) {
+    if (OB_FAIL(schedule_update_all_server_config_task())) {
+      LOG_WARN("fail to schedule update_all_server_config_task", KR(ret));
+    } else {
+      LOG_INFO("add update_all_server_config_task");
+    }
+  }
+
   if (OB_SUCC(ret)) {
     if (OB_FAIL(schedule_inspector_task())) {
       LOG_WARN("start inspector fail", K(ret));
@@ -5270,6 +5299,7 @@ int ObRootService::stop_timer_tasks()
     task_queue_.cancel_timer_task(event_table_clear_task_);
     task_queue_.cancel_timer_task(self_check_task_);
     task_queue_.cancel_timer_task(update_rs_list_timer_task_);
+    task_queue_.cancel_timer_task(update_all_server_config_task_);
     inspect_task_queue_.cancel_timer_task(inspector_task_);
     inspect_task_queue_.cancel_timer_task(purge_recyclebin_task_);
   }
@@ -8598,17 +8628,33 @@ int ObRootService::update_all_server_config()
   int ret = OB_SUCCESS;
   ObZone empty_zone;
   ObArray<ObAddr> server_list;
+  ObArray<ObAddr> config_all_server_list;
+  ObArray<ObAddr> empty_excluded_server_list;
+  bool need_update = true;
   HEAP_VAR(ObAdminSetConfigItem, all_server_config) {
     auto &value = all_server_config.value_;
     int64_t pos = 0;
     if (!inited_) {
       ret = OB_NOT_INIT;
       LOG_WARN("not init", K(ret));
+    } else if (OB_UNLIKELY(!SVR_TRACER.has_build())) {
+      need_update = false;
     } else if (OB_FAIL(SVR_TRACER.get_servers_of_zone(empty_zone, server_list))) {
       LOG_WARN("fail to get server", K(ret));
+    } else if (OB_UNLIKELY(0 == server_list.size())) {
+      need_update = false;
+      LOG_WARN("no servers in all_server_tracer");
     } else if (OB_FAIL(all_server_config.name_.assign(config_->all_server_list.name()))) {
       LOG_WARN("fail to assign name", K(ret));
+    } else if (OB_FAIL(ObShareUtil::parse_all_server_list(empty_excluded_server_list, config_all_server_list))) {
+      LOG_WARN("fail to parse all_server_list from GCONF", KR(ret));
+    } else if (ObRootUtils::is_subset(server_list, config_all_server_list)
+        && ObRootUtils::is_subset(config_all_server_list, server_list)) {
+      need_update = false;
+      LOG_TRACE("server_list is the same as config_all_server_list, no need to update GCONF.all_server_list",
+          K(server_list), K(config_all_server_list));
     } else {
+      LOG_INFO("GCONF.all_server_list should be updated", K(config_all_server_list), K(server_list));
       char ip_port_buf[MAX_IP_PORT_LENGTH];
       for (int64_t i = 0; i < server_list.count() - 1; i++) {
         if (OB_FAIL(server_list.at(i).ip_port_to_string(ip_port_buf, MAX_IP_PORT_LENGTH))) {
@@ -8630,7 +8676,7 @@ int ObRootService::update_all_server_config()
     if (OB_SIZE_OVERFLOW == ret) {
       LOG_ERROR("can't print server addr to buffer, size overflow", K(ret), K(server_list));
     }
-    if (OB_SUCC(ret)) {
+    if (need_update && OB_SUCC(ret)) {
       ObAdminSetConfigArg arg;
       arg.is_inner_ = true;
       if (OB_FAIL(arg.items_.push_back(all_server_config))) {
@@ -8644,7 +8690,6 @@ int ObRootService::update_all_server_config()
   }
   return ret;
 }
-
 /////////////////////////
 ObRootService::ObReportCoreTableReplicaTask::ObReportCoreTableReplicaTask(ObRootService &root_service)
 : ObAsyncTimerTask(root_service.task_queue_),

src/rootserver/ob_root_service.h

@@ -290,6 +290,7 @@ public:
   class ObUpdateAllServerConfigTask : public common::ObAsyncTimerTask
   {
   public:
+    const static int64_t RETRY_INTERVAL = 600 * 1000L * 1000L;  // 10min
     explicit ObUpdateAllServerConfigTask(ObRootService &root_service);
     virtual ~ObUpdateAllServerConfigTask() {}
   public:
@@ -743,6 +744,7 @@ public:
   // @see ObInspector
   int schedule_inspector_task();
   int schedule_update_rs_list_task();
+  int schedule_update_all_server_config_task();
   //update statistic cache
   int update_stat_cache(const obrpc::ObUpdateStatCacheArg &arg);
 
@@ -949,6 +951,7 @@ private:
   ObSnapshotInfoManager snapshot_manager_;
   int64_t core_meta_table_version_;
   ObUpdateRsListTimerTask update_rs_list_timer_task_;
+  ObUpdateAllServerConfigTask update_all_server_config_task_;
   int64_t baseline_schema_version_;
 
   // backup

src/rootserver/ob_root_utils.cpp

@@ -1962,21 +1962,6 @@ bool ObRootUtils::has_intersection(const common::ObIArray<T> &this_array,
   return bret;
 }
 
-template<class T>
-bool ObRootUtils::is_subset(const common::ObIArray<T> &superset_array,
-                            const common::ObIArray<T> &array)
-{
-  bool bret = true;
-  for (int64_t i = 0; i < array.count() && bret; i++) {
-    if (has_exist_in_array(superset_array, array.at(i))) {
-      //nothing todo
-    } else {
-      bret = false;
-    }
-  }
-  return bret;
-}
-
 //iter the tenant, table's primary_zone is covered by zone_list
 int ObRootUtils::check_primary_region_in_zonelist(ObMultiVersionSchemaService *schema_service,
                                                   ObDDLService *ddl_service,

src/rootserver/ob_root_utils.h

@@ -676,6 +676,21 @@ public:
 
 };
 
+template<class T>
+bool ObRootUtils::is_subset(const common::ObIArray<T> &superset_array,
+                            const common::ObIArray<T> &array)
+{
+  bool bret = true;
+  for (int64_t i = 0; i < array.count() && bret; i++) {
+    if (has_exist_in_array(superset_array, array.at(i))) {
+      //nothing todo
+    } else {
+      bret = false;
+    }
+  }
+  return bret;
+}
+
 class ObClusterInfoGetter
 {
 public:

src/rootserver/ob_server_zone_op_service.cpp

@@ -18,6 +18,7 @@
 #include "share/ob_service_epoch_proxy.h"
 #include "share/ob_max_id_fetcher.h"
 #include "lib/mysqlclient/ob_mysql_transaction.h"  // ObMySQLTransaction
+#include "lib/utility/ob_tracepoint.h" // ERRSIM
 #include "rootserver/ob_root_service.h" // callback
 #include "share/ob_all_server_tracer.h"
 #include "rootserver/ob_server_manager.h"
@@ -134,10 +135,6 @@ int ObServerZoneOpService::add_servers(const ObIArray<ObAddr> &servers, const Ob
       } else {}
     }
   }
-  int tmp_ret = OB_SUCCESS;
-  if (OB_TMP_FAIL(SVR_TRACER.refresh())) {
-    LOG_WARN("fail to refresh server tracer", KR(ret), KR(tmp_ret));
-  }
   return ret;
 }
 int ObServerZoneOpService::delete_servers(
@@ -165,10 +162,6 @@ int ObServerZoneOpService::delete_servers(
       }
     }
   }
-  int tmp_ret = OB_SUCCESS;
-  if (OB_TMP_FAIL(SVR_TRACER.refresh())) {
-    LOG_WARN("fail to refresh server tracer", KR(ret), KR(tmp_ret));
-  }
   return ret;
 }
 int ObServerZoneOpService::cancel_delete_servers(
@@ -207,10 +200,6 @@ int ObServerZoneOpService::cancel_delete_servers(
       (void) end_trans_and_on_server_change_(ret, trans, "cancel_delete_server", server, server_info_in_table.get_zone(), now);
     }
   }
-  int tmp_ret = OB_SUCCESS;
-  if (OB_TMP_FAIL(SVR_TRACER.refresh())) {
-    LOG_WARN("fail to refresh server tracer", KR(ret), KR(tmp_ret));
-  }
   return ret;
 }
 int ObServerZoneOpService::finish_delete_server(
@@ -240,10 +229,6 @@ int ObServerZoneOpService::finish_delete_server(
     LOG_WARN("fail to remove this server from __all_server table", KR(ret), K(server));
   }
   (void) end_trans_and_on_server_change_(ret, trans, "finish_delete_server", server, server_info_in_table.get_zone(), now);
-  int tmp_ret = OB_SUCCESS;
-  if (OB_TMP_FAIL(SVR_TRACER.refresh())) {
-    LOG_WARN("fail to refresh server tracer", KR(ret), KR(tmp_ret));
-  }
   return ret;
 }
 int ObServerZoneOpService::stop_servers(
@@ -265,10 +250,6 @@ int ObServerZoneOpService::stop_servers(
       }
     }
   }
-  int tmp_ret = OB_SUCCESS;
-  if (OB_TMP_FAIL(SVR_TRACER.refresh())) {
-    LOG_WARN("fail to refresh server tracer", KR(ret), KR(tmp_ret));
-  }
   return ret;
 }
 int ObServerZoneOpService::start_servers(
@@ -290,10 +271,6 @@ int ObServerZoneOpService::start_servers(
       }
     }
   }
-  int tmp_ret = OB_SUCCESS;
-  if (OB_TMP_FAIL(SVR_TRACER.refresh())) {
-    LOG_WARN("fail to refresh server tracer", KR(ret), KR(tmp_ret));
-  }
   return ret;
 }
 int ObServerZoneOpService::stop_server_precheck(
@@ -739,6 +716,7 @@ int ObServerZoneOpService::check_zone_and_server_(
   }
   return ret;
 }
+ERRSIM_POINT_DEF(ALL_SERVER_LIST_ERROR);
 void ObServerZoneOpService::end_trans_and_on_server_change_(
     int &ret,
     common::ObMySQLTransaction &trans,
@@ -758,10 +736,15 @@ void ObServerZoneOpService::end_trans_and_on_server_change_(
       ret = OB_SUCC(ret) ? tmp_ret : ret;
     }
   }
+  if (OB_TMP_FAIL(SVR_TRACER.refresh())) {
+    LOG_WARN("fail to refresh server tracer", KR(ret), KR(tmp_ret));
+  }
+  bool no_on_server_change = ALL_SERVER_LIST_ERROR ? true : false;
   if (OB_ISNULL(server_change_callback_)) {
     tmp_ret = OB_ERR_UNEXPECTED;
     LOG_WARN("server_change_callback_ is null", KR(ret), KR(tmp_ret), KP(server_change_callback_));
     ret = OB_SUCC(ret) ? tmp_ret : ret;
+  } else if (no_on_server_change) {
   } else if (OB_TMP_FAIL(server_change_callback_->on_server_change())) {
     LOG_WARN("fail to callback on server change", KR(ret), KR(tmp_ret));
   }

src/share/config/ob_config.cpp

@@ -701,6 +701,17 @@ int ObConfigStringItem::copy(char *buf, const int64_t buf_len)
   return ret;
 }
 
+int ObConfigStringItem::deep_copy_value_string(ObIAllocator &allocator, ObString &dst)
+{
+  int ret = OB_SUCCESS;
+  ObLatchRGuard rd_guard(const_cast<ObLatch&>(lock_), ObLatchIds::CONFIG_LOCK);
+  ObString src = ObString::make_string(value_str_);
+  if (OB_FAIL(ob_write_string(allocator, src, dst))) {
+    OB_LOG(WARN, "fail to deep copy", KR(ret), K(src));
+  }
+  return ret;
+}
+
 ObConfigLogArchiveOptionsItem::ObConfigLogArchiveOptionsItem(ObConfigContainer *container,
                                                              Scope::ScopeInfo scope_info,
                                                              const char *name,

src/share/config/ob_config.h

@@ -719,6 +719,7 @@ public:
     return ObString::make_string(value_str_);
   }
   int copy(char *buf, const int64_t buf_len); // '\0' will be added
+  int deep_copy_value_string(ObIAllocator &allocator, ObString &dst);
   virtual ObConfigItemType get_config_item_type() const {
     return ObConfigItemType::OB_CONF_ITEM_TYPE_STRING;
   }

src/share/location_cache/ob_ls_location_service.cpp

@@ -809,8 +809,8 @@ int ObLSLocationService::construct_rpc_dests_(
     LOG_WARN("fail to check inner stat", KR(ret));
   } else if (OB_FAIL(rs_mgr_->construct_initial_server_list(check_ls_service, rs_list))) {
     LOG_WARN("fail to get rs list", KR(ret));
-  } else if (OB_FAIL(rs_mgr_->construct_all_server_list(rs_list, all_server_list))) {
+  } else if (OB_FAIL(ObShareUtil::parse_all_server_list(rs_list, all_server_list))) {
-    LOG_WARN("fail to get all server list", KR(ret));
+    LOG_WARN("fail to get all server list", KR(ret), K(rs_list));
   } else if (OB_FAIL(dests.assign(rs_list))) {
     LOG_WARN("fail to assign rs_list", KR(ret));
   } else if (OB_FAIL(append(dests, all_server_list))) {

src/share/ls/ob_rpc_ls_table.cpp

@@ -144,8 +144,8 @@ int ObRpcLSTable::get_ls_info_(ObLSInfo &ls_info)
     // case 4: try use all_server_list from local configure
     if (need_retry) { // overwrite ret
       ObArray<ObAddr> server_list;
-      if (OB_FAIL(rs_mgr_->construct_all_server_list(rs_list, server_list))) {
+      if (OB_FAIL(ObShareUtil::parse_all_server_list(rs_list, server_list))) {
-        LOG_WARN("fail to construct all server list", KR(ret));
+        LOG_WARN("fail to construct all server list", KR(ret), K(rs_list));
       } else if (server_list.empty()) {
         // server_list is empty, do nothing
         LOG_INFO("server_list is empty, do nothing", KR(ret), K(server_list));

src/share/ob_all_server_tracer.cpp

@@ -573,7 +573,7 @@ int ObServerTraceMap::refresh()
         LOG_WARN("fail to push back", K(server_info_i), KR(ret));
       }
     }
-    if (OB_SUCC(ret)) {
+    if (OB_SUCC(ret) && !has_build_) {
       has_build_ = true;
     }
   }

src/share/ob_rs_mgr.cpp

@@ -450,48 +450,6 @@ int ObRsMgr::construct_initial_server_list(
   return ret;
 }
 
-// case 4: try use all_server_list from local configure
-int ObRsMgr::construct_all_server_list(
-    const ObIArray<ObAddr> &rs_list,
-    ObIArray<ObAddr> &server_list)
-{
-  int ret = OB_SUCCESS;
-  server_list.reset();
-  if (OB_FAIL(check_inner_stat())) {
-    LOG_WARN("fail to check inner stat", KR(ret));
-  } else {
-    bool split_end = false;
-    ObString sub_string;
-    ObString trimed_string;
-    ObString all_server_list(strlen(static_cast<common::ObServerConfig *>(config_)->all_server_list.str()),
-                                    static_cast<common::ObServerConfig *>(config_)->all_server_list.str());
-    char buf[OB_IP_PORT_STR_BUFF];
-    ObAddr addr;
-    while (!split_end && OB_SUCCESS == ret) {
-      sub_string = all_server_list.split_on(',');
-      if (sub_string.empty() && NULL == sub_string.ptr()) {
-        split_end = true;
-        sub_string = all_server_list;
-      }
-      trimed_string = sub_string.trim();
-      if (trimed_string.empty()) {
-        //nothing todo
-      } else if (0 > snprintf(buf, OB_IP_PORT_STR_BUFF, "%.*s", trimed_string.length(), trimed_string.ptr())) {
-        ret = OB_ERR_UNEXPECTED;
-        LOG_WARN("fail to snprintf", KR(ret), K(trimed_string));
-      } else if (OB_FAIL(addr.parse_from_cstring(buf))) {
-        LOG_WARN("fail to parser addr from cstring", KR(ret));
-      } else if (has_exist_in_array(rs_list, addr)) {
-        //nothing todo
-      } else if (OB_FAIL(server_list.push_back(addr))) {
-        LOG_WARN("fail to push back", KR(ret), K(addr));
-      }
-    } // end while
-  } //end else
-  return ret;
-}
-
-
 int ObRsMgr::renew_remote_master_rootserver()
 {
   return OB_NOT_SUPPORTED;

src/share/ob_rs_mgr.h

@@ -133,12 +133,6 @@ public:
   int construct_initial_server_list(
       const bool check_ls_service,
       common::ObIArray<common::ObAddr> &server_list);
-  // build a broader list to ask informations
-  // @param [in] rs_list, servers belongs to rs
-  // @param [out] server_list, all servers except rs_list
-  int construct_all_server_list(
-      const common::ObIArray<common::ObAddr> &rs_list,
-      common::ObIArray<common::ObAddr> &server_list);
 
 private:
   class ObRemoteClusterIdGetter

src/share/ob_share_util.cpp

@@ -216,5 +216,47 @@ int ObShareUtil::fetch_current_data_version(
   } // end SMART_VAR
   return ret;
 }
+
+int ObShareUtil::parse_all_server_list(
+    const ObArray<ObAddr> &excluded_server_list,
+    ObArray<ObAddr> &config_all_server_list)
+{
+  int ret = OB_SUCCESS;
+  config_all_server_list.reset();
+  common::ObArenaAllocator allocator(lib::ObLabel("AllSvrList"));
+  ObString all_server_list;
+  LOG_TRACE("get all_server_list from GCONF", K(GCONF.all_server_list));
+  if (OB_FAIL(GCONF.all_server_list.deep_copy_value_string(allocator, all_server_list))) {
+    LOG_WARN("fail to deep copy GCONF.all_server_list", KR(ret), K(GCONF.all_server_list));
+  } else {
+    bool split_end = false;
+    ObAddr addr;
+    ObString sub_string;
+    ObString trimed_string;
+    while (!split_end && OB_SUCCESS == ret) {
+      sub_string.reset();
+      trimed_string.reset();
+      addr.reset();
+      sub_string = all_server_list.split_on(',');
+      if (sub_string.empty() && NULL == sub_string.ptr()) {
+        split_end = true;
+        sub_string = all_server_list;
+      }
+      trimed_string = sub_string.trim();
+      if (trimed_string.empty()) {
+        //nothing todo
+      } else if (OB_FAIL(addr.parse_from_string(trimed_string))) {
+        LOG_WARN("fail to parser addr from string", KR(ret), K(trimed_string));
+      } else if (has_exist_in_array(excluded_server_list, addr)) {
+        // nothing todo
+      } else if (has_exist_in_array(config_all_server_list, addr)) {
+        //nothing todo
+      } else if (OB_FAIL(config_all_server_list.push_back(addr))) {
+        LOG_WARN("fail to push back", KR(ret), K(addr));
+      }
+    } // end while
+  }
+  return ret;
+}
 } //end namespace share
 } //end namespace oceanbase

src/share/ob_share_util.h

@@ -59,6 +59,12 @@ public:
              common::ObISQLClient &client,
              const uint64_t tenant_id,
              uint64_t &data_version);
+  // parse GCONF.all_server_list
+  // @params[in]  excluded_server_list, servers which will not be included in the output
+  // @params[out] config_all_server_list, servers in (GCONF.all_server_list - excluded_server_list)
+  static int parse_all_server_list(
+    const ObArray<ObAddr> &excluded_server_list,
+    ObArray<ObAddr> &config_all_server_list);
 };
 }//end namespace share
 }//end namespace oceanbase