[CP] [UPGRADE] Fix min_observer_version unchanged problem

src/rootserver/ob_system_admin_util.cpp

@@ -1578,11 +1578,41 @@ int ObAdminRollingUpgradeCmd::execute(const obrpc::ObAdminRollingUpgradeArg &arg
     } else if (OB_FAIL(set_config_arg.items_.push_back(item))) {
       LOG_WARN("add _upgrade_stage config item failed", KR(ret), K(arg));
     } else if (obrpc::OB_UPGRADE_STAGE_POSTUPGRADE == arg.stage_) {
+      // wait min_observer_version to report to inner table
+      ObTimeoutCtx ctx;
+      if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, GCONF.rpc_timeout))) {
+        LOG_WARN("fail to set default timeout", KR(ret));
+      } else {
+        const int64_t CHECK_INTERVAL = 100 * 1000L; // 100ms
+        while (OB_SUCC(ret)) {
+          uint64_t min_observer_version = 0;
+          if (ctx.is_timeouted()) {
+            ret = OB_TIMEOUT;
+            LOG_WARN("wait min_server_version report to inner table failed",
+                     KR(ret), "abs_timeout", ctx.get_abs_timeout());
+          } else if (OB_FAIL(SVR_TRACER.get_min_server_version(
+                     min_server_version, min_observer_version))) {
+            LOG_WARN("failed to get the min server version", KR(ret));
+          } else if (min_observer_version > CLUSTER_CURRENT_VERSION) {
+            ret = OB_ERR_UNEXPECTED;
+            LOG_WARN("min_observer_version is larger than CLUSTER_CURRENT_VERSION",
+                     KR(ret), "min_server_version", min_server_version,
+                     K(min_observer_version), "CLUSTER_CURRENT_VERSION", CLUSTER_CURRENT_VERSION);
+          } else if (min_observer_version < CLUSTER_CURRENT_VERSION) {
+            if (REACH_TIME_INTERVAL(1 * 1000 * 1000L)) { // 1s
+              LOG_INFO("min_observer_version is not reported yet, just wait",
+                       KR(ret), "min_server_version", min_server_version,
+                       K(min_observer_version), "CLUSTER_CURRENT_VERSION", CLUSTER_CURRENT_VERSION);
+            }
+            ob_usleep(CHECK_INTERVAL);
+          } else {
+            break;
+          }
+        } // end while
+      }
       // end rolling upgrade, should raise min_observer_version
       const char *min_obs_version_name = "min_observer_version";
-      if (OB_FAIL(SVR_TRACER.get_min_server_version(min_server_version))) {
+      if (FAILEDx(item.name_.assign(min_obs_version_name))) {
-        LOG_WARN("failed to get the min server version", KR(ret));
-      } else if (OB_FAIL(item.name_.assign(min_obs_version_name))) {
         LOG_WARN("assign min_observer_version config name failed",
                  KR(ret), K(min_obs_version_name));
       } else if (OB_FAIL(item.value_.assign(min_server_version))) {

src/share/ob_all_server_tracer.cpp

@@ -502,11 +502,14 @@ int ObServerTraceMap::get_servers_by_status(
   return ret;
 }
 
-int ObServerTraceMap::get_min_server_version(char min_server_version[OB_SERVER_VERSION_LENGTH])
+int ObServerTraceMap::get_min_server_version(
+    char min_server_version[OB_SERVER_VERSION_LENGTH],
+    uint64_t &min_observer_version)
 {
   int ret = OB_SUCCESS;
   ObZone zone; // empty zone, get all server statuses
   ObArray<ObServerInfoInTable> servers_info;
+  min_observer_version = 0;
   if (IS_NOT_INIT) {
     ret = OB_NOT_INIT;
     LOG_WARN("server trace map has not inited", KR(ret));
@@ -539,6 +542,7 @@ int ObServerTraceMap::get_min_server_version(char min_server_version[OB_SERVER_V
           MEMCPY(min_server_version, version, len);
           min_server_version[len] = '\0';
           cur_min_version = version_parser.get_cluster_version();
+          min_observer_version = cur_min_version;
         }
       }
       if (OB_SUCC(ret) && UINT64_MAX == cur_min_version) {
@@ -766,9 +770,11 @@ int ObAllServerTracer::get_servers_by_status(
   return trace_map_.get_servers_by_status(zone, alive_server_list, not_alive_server_list);
 }
 
-int ObAllServerTracer::get_min_server_version(char min_server_version[OB_SERVER_VERSION_LENGTH])
+int ObAllServerTracer::get_min_server_version(
+    char min_server_version[OB_SERVER_VERSION_LENGTH],
+    uint64_t &min_observer_version)
 {
-  return trace_map_.get_min_server_version(min_server_version);
+  return trace_map_.get_min_server_version(min_server_version, min_observer_version);
 }
 
 bool ObAllServerTracer::has_build() const

src/share/ob_all_server_tracer.h

@@ -62,7 +62,9 @@ public:
       const ObZone &zone,
       common::ObIArray<common::ObAddr> &alive_server_list,
       common::ObIArray<common::ObAddr> &not_alive_server_list) const;
-  virtual int get_min_server_version(char min_server_version[OB_SERVER_VERSION_LENGTH]);
+  virtual int get_min_server_version(
+              char min_server_version_str[OB_SERVER_VERSION_LENGTH],
+              uint64_t &min_observer_version);
   bool has_build() const {return has_build_; };
   int refresh();
   int for_each_server_info(const ObFunction<int(const ObServerInfoInTable &server_info)> &functor);
@@ -138,7 +140,9 @@ public:
       const ObZone &zone,
       common::ObIArray<common::ObAddr> &alive_server_list,
       common::ObIArray<common::ObAddr> &not_alive_server_list) const;
-  virtual int get_min_server_version(char min_server_version[OB_SERVER_VERSION_LENGTH]);
+  virtual int get_min_server_version(
+              char min_server_version_str[OB_SERVER_VERSION_LENGTH],
+              uint64_t &min_observer_version);
   bool has_build() const;
 private:
   ObAllServerTracer();