From c50b0cf6072881fc9c0846a1abf2bb5d407f7b53 Mon Sep 17 00:00:00 2001 From: tino247 Date: Thu, 8 Feb 2024 07:34:04 +0000 Subject: [PATCH] [CP] [UPGRADE] Fix min_observer_version unchanged problem --- src/rootserver/ob_system_admin_util.cpp | 36 ++++++++++++++++++++++--- src/share/ob_all_server_tracer.cpp | 12 ++++++--- src/share/ob_all_server_tracer.h | 8 ++++-- 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/src/rootserver/ob_system_admin_util.cpp b/src/rootserver/ob_system_admin_util.cpp index eacc5713f..35096ddaf 100644 --- a/src/rootserver/ob_system_admin_util.cpp +++ b/src/rootserver/ob_system_admin_util.cpp @@ -1578,11 +1578,41 @@ int ObAdminRollingUpgradeCmd::execute(const obrpc::ObAdminRollingUpgradeArg &arg } else if (OB_FAIL(set_config_arg.items_.push_back(item))) { LOG_WARN("add _upgrade_stage config item failed", KR(ret), K(arg)); } else if (obrpc::OB_UPGRADE_STAGE_POSTUPGRADE == arg.stage_) { + // wait min_observer_version to report to inner table + ObTimeoutCtx ctx; + if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, GCONF.rpc_timeout))) { + LOG_WARN("fail to set default timeout", KR(ret)); + } else { + const int64_t CHECK_INTERVAL = 100 * 1000L; // 100ms + while (OB_SUCC(ret)) { + uint64_t min_observer_version = 0; + if (ctx.is_timeouted()) { + ret = OB_TIMEOUT; + LOG_WARN("wait min_server_version report to inner table failed", + KR(ret), "abs_timeout", ctx.get_abs_timeout()); + } else if (OB_FAIL(SVR_TRACER.get_min_server_version( + min_server_version, min_observer_version))) { + LOG_WARN("failed to get the min server version", KR(ret)); + } else if (min_observer_version > CLUSTER_CURRENT_VERSION) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("min_observer_version is larger than CLUSTER_CURRENT_VERSION", + KR(ret), "min_server_version", min_server_version, + K(min_observer_version), "CLUSTER_CURRENT_VERSION", CLUSTER_CURRENT_VERSION); + } else if (min_observer_version < CLUSTER_CURRENT_VERSION) { + if (REACH_TIME_INTERVAL(1 * 1000 * 1000L)) { // 1s + LOG_INFO("min_observer_version is not reported yet, just wait", + KR(ret), "min_server_version", min_server_version, + K(min_observer_version), "CLUSTER_CURRENT_VERSION", CLUSTER_CURRENT_VERSION); + } + ob_usleep(CHECK_INTERVAL); + } else { + break; + } + } // end while + } // end rolling upgrade, should raise min_observer_version const char *min_obs_version_name = "min_observer_version"; - if (OB_FAIL(SVR_TRACER.get_min_server_version(min_server_version))) { - LOG_WARN("failed to get the min server version", KR(ret)); - } else if (OB_FAIL(item.name_.assign(min_obs_version_name))) { + if (FAILEDx(item.name_.assign(min_obs_version_name))) { LOG_WARN("assign min_observer_version config name failed", KR(ret), K(min_obs_version_name)); } else if (OB_FAIL(item.value_.assign(min_server_version))) { diff --git a/src/share/ob_all_server_tracer.cpp b/src/share/ob_all_server_tracer.cpp index 6c538aecb..0eabbced9 100644 --- a/src/share/ob_all_server_tracer.cpp +++ b/src/share/ob_all_server_tracer.cpp @@ -502,11 +502,14 @@ int ObServerTraceMap::get_servers_by_status( return ret; } -int ObServerTraceMap::get_min_server_version(char min_server_version[OB_SERVER_VERSION_LENGTH]) +int ObServerTraceMap::get_min_server_version( + char min_server_version[OB_SERVER_VERSION_LENGTH], + uint64_t &min_observer_version) { int ret = OB_SUCCESS; ObZone zone; // empty zone, get all server statuses ObArray servers_info; + min_observer_version = 0; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("server trace map has not inited", KR(ret)); @@ -539,6 +542,7 @@ int ObServerTraceMap::get_min_server_version(char min_server_version[OB_SERVER_V MEMCPY(min_server_version, version, len); min_server_version[len] = '\0'; cur_min_version = version_parser.get_cluster_version(); + min_observer_version = cur_min_version; } } if (OB_SUCC(ret) && UINT64_MAX == cur_min_version) { @@ -766,9 +770,11 @@ int ObAllServerTracer::get_servers_by_status( return trace_map_.get_servers_by_status(zone, alive_server_list, not_alive_server_list); } -int ObAllServerTracer::get_min_server_version(char min_server_version[OB_SERVER_VERSION_LENGTH]) +int ObAllServerTracer::get_min_server_version( + char min_server_version[OB_SERVER_VERSION_LENGTH], + uint64_t &min_observer_version) { - return trace_map_.get_min_server_version(min_server_version); + return trace_map_.get_min_server_version(min_server_version, min_observer_version); } bool ObAllServerTracer::has_build() const diff --git a/src/share/ob_all_server_tracer.h b/src/share/ob_all_server_tracer.h index 0aeab385c..a562609fa 100644 --- a/src/share/ob_all_server_tracer.h +++ b/src/share/ob_all_server_tracer.h @@ -62,7 +62,9 @@ public: const ObZone &zone, common::ObIArray &alive_server_list, common::ObIArray ¬_alive_server_list) const; - virtual int get_min_server_version(char min_server_version[OB_SERVER_VERSION_LENGTH]); + virtual int get_min_server_version( + char min_server_version_str[OB_SERVER_VERSION_LENGTH], + uint64_t &min_observer_version); bool has_build() const {return has_build_; }; int refresh(); int for_each_server_info(const ObFunction &functor); @@ -138,7 +140,9 @@ public: const ObZone &zone, common::ObIArray &alive_server_list, common::ObIArray ¬_alive_server_list) const; - virtual int get_min_server_version(char min_server_version[OB_SERVER_VERSION_LENGTH]); + virtual int get_min_server_version( + char min_server_version_str[OB_SERVER_VERSION_LENGTH], + uint64_t &min_observer_version); bool has_build() const; private: ObAllServerTracer();