[FEAT MERGE] New upgrade framework for 4.x

cmake/RPM.cmake

@@ -64,9 +64,8 @@ install(FILES
   tools/default_srs_data_mysql.sql
   tools/upgrade/upgrade_pre.py
   tools/upgrade/upgrade_post.py
-  tools/upgrade/upgrade_post_checker.py
   tools/upgrade/upgrade_checker.py
-  tools/upgrade/upgrade_cluster_health_checker.py
+  tools/upgrade/upgrade_health_checker.py
   DESTINATION etc
   COMPONENT server)
 

deps/oblib/src/lib/ob_define.h

@@ -131,6 +131,8 @@ const int64_t OB_MAX_CHILD_MEMBER_NUMBER_IN_FOLLOWER = 5;
 const int64_t OB_DEFAULT_MEMBER_NUMBER = 3;
 const int64_t MAX_VALUE_LENGTH = 4096;
 const int64_t MAX_LLC_BITMAP_LENGTH = 4096;
+const int64_t MAX_CLUSTER_EVENT_NAME_LENGTH = 256;
+const int64_t MAX_CLUSTER_EVENT_VALUE_LENGTH = 4096;
 const int64_t MAX_ROOTSERVICE_EVENT_NAME_LENGTH = 256;
 const int64_t MAX_ROOTSERVICE_EVENT_VALUE_LENGTH = 256;
 const int64_t MAX_ROOTSERVICE_EVENT_DESC_LENGTH = 64;

deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h

@@ -453,7 +453,7 @@ PCODE_DEF(OB_HA_FETCH_LS_MEMBER_LIST, 0x4A5)
 PCODE_DEF(OB_HA_FETCH_LS_META_INFO, 0x4A6)
 PCODE_DEF(OB_NOTIFY_CREATE_TENANT_USER_LS, 0x4A7)
 
-//PCODE_DEF(OB_INIT_TENANT_CONFIG, 0x4A8)
+PCODE_DEF(OB_INIT_TENANT_CONFIG, 0x4A8)
 // sql, including executor
 
 // sql

src/observer/ob_rpc_processor_simple.cpp

@@ -433,6 +433,18 @@ int ObGetMinSSTableSchemaVersionP::process()
   return ret;
 }
 
+int ObInitTenantConfigP::process()
+{
+  int ret = OB_SUCCESS;
+  if (OB_ISNULL(gctx_.ob_service_)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_ERROR("invalid argument", KR(ret), KP(gctx_.ob_service_));
+  } else {
+    ret = gctx_.ob_service_->init_tenant_config(arg_, result_);
+  }
+  return ret;
+}
+
 int ObCalcColumnChecksumRequestP::process()
 {
   int ret = OB_SUCCESS;

src/observer/ob_rpc_processor_simple.h

@@ -109,6 +109,7 @@ OB_DEFINE_PROCESSOR_OBADMIN(Srv, OB_SET_TENANT_CONFIG, ObRpcSetTenantConfigP);
 OB_DEFINE_PROCESSOR_S(Srv, OB_NOTIFY_TENANT_SERVER_UNIT_RESOURCE, ObRpcNotifyTenantServerUnitResourceP);
 OB_DEFINE_PROCESSOR_S(Srv, OB_CHECK_FROZEN_SCN, ObCheckFrozenVersionP);
 OB_DEFINE_PROCESSOR_S(Srv, OB_GET_MIN_SSTABLE_SCHEMA_VERSION, ObGetMinSSTableSchemaVersionP);
+OB_DEFINE_PROCESSOR_S(Srv, OB_INIT_TENANT_CONFIG, ObInitTenantConfigP);
 
 // oceanbase service provied
 OB_DEFINE_PROCESSOR_S(Srv, OB_FETCH_SYS_LS, ObRpcFetchSysLSP);

src/observer/ob_server.cpp

@@ -217,8 +217,8 @@ int ObServer::init(const ObServerOptions &opts, const ObPLogWriterCfg &log_cfg)
     LOG_ERROR("init pre setting failed", KR(ret));
   } else if (OB_FAIL(init_global_context())) {
     LOG_ERROR("init global context failed", KR(ret));
-  } else if (OB_FAIL(init_cluster_version())) {
-    LOG_ERROR("init cluster version failed", KR(ret));
+  } else if (OB_FAIL(init_version())) {
+    LOG_ERROR("init version failed", KR(ret));
   } else if (OB_FAIL(init_sql_proxy())) {
     LOG_ERROR("init sql connection pool failed", KR(ret));
   } else if (OB_FAIL(init_io())) {
@@ -2028,9 +2028,9 @@ int ObServer::init_global_context()
   return ret;
 }
 
-int ObServer::init_cluster_version()
+int ObServer::init_version()
 {
-  return ObClusterVersion::get_instance().init(&config_);
+  return ObClusterVersion::get_instance().init(&config_, &tenant_config_mgr_);
 }
 
 int ObServer::init_ts_mgr()

src/observer/ob_server.h

@@ -248,7 +248,7 @@ private:
   int init_sequence();
   int init_pl();
   int init_global_context();
-  int init_cluster_version();
+  int init_version();
   int init_ts_mgr();
   int init_px_target_mgr();
   int init_storage();

src/observer/ob_service.cpp

@@ -70,6 +70,7 @@
 #include "share/backup/ob_backup_connectivity.h"
 #include "observer/report/ob_tenant_meta_checker.h"//ObTenantMetaChecker
 #include "storage/compaction/ob_tenant_tablet_scheduler.h"
+#include "share/ob_cluster_event_history_table_operator.h"//CLUSTER_EVENT_INSTANCE
 #include "storage/ddl/ob_tablet_ddl_kv_mgr.h"
 
 namespace oceanbase
@@ -192,6 +193,8 @@ int ObService::init(common::ObMySQLProxy &sql_proxy,
     FLOG_WARN("heartbeat_process_.init failed", KR(ret));
   } else if (OB_FAIL(schema_updater_.init(gctx_.self_addr(), gctx_.schema_service_))) {
     FLOG_WARN("client_manager_.initialize failed", "self_addr", gctx_.self_addr(), KR(ret));
+  } else if (OB_FAIL(CLUSTER_EVENT_INSTANCE.init(sql_proxy))) {
+    FLOG_WARN("init cluster event history table failed", KR(ret));
   } else if (OB_FAIL(SERVER_EVENT_INSTANCE.init(sql_proxy, gctx_.self_addr()))) {
     FLOG_WARN("init server event history table failed", KR(ret));
   } else if (OB_FAIL(DEALOCK_EVENT_INSTANCE.init(sql_proxy))) {
@@ -349,6 +352,10 @@ int ObService::destroy()
     lease_state_mgr_.destroy();
     FLOG_INFO("lease state manager destroyed");
 
+    FLOG_INFO("begin to destroy cluster event instance");
+    CLUSTER_EVENT_INSTANCE.destroy();
+    FLOG_INFO("cluster event instance destroyed");
+
     FLOG_INFO("begin to destroy server event instance");
     SERVER_EVENT_INSTANCE.destroy();
     FLOG_INFO("server event instance destroyed");
@@ -1395,9 +1402,17 @@ int ObService::check_deployment_mode_match(
 int ObService::is_empty_server(const obrpc::ObCheckServerEmptyArg &arg, obrpc::Bool &is_empty)
 {
   int ret = OB_SUCCESS;
+  uint64_t sys_data_version = 0;
   if (!inited_) {
     ret = OB_NOT_INIT;
     LOG_WARN("not init", K(ret));
+  } else if (OB_FAIL(GET_MIN_DATA_VERSION(OB_SYS_TENANT_ID, sys_data_version))) {
+    LOG_WARN("fail to get sys data version", KR(ret));
+  } else if (arg.sys_data_version_ > 0
+             && sys_data_version > arg.sys_data_version_) {
+    ret = OB_NOT_SUPPORTED;
+    LOG_WARN("add server with larger sys data version is not supported",
+             KR(ret), K(arg), K(sys_data_version));
   } else {
     bool server_empty = false;
     // server dir must be created when 1) local mode, 2) OFS bootstrap this server
@@ -2281,5 +2296,30 @@ int ObService::estimate_tablet_block_count(const obrpc::ObEstBlockArg &arg,
   return ret;
 }
 
+int ObService::init_tenant_config(
+    const obrpc::ObInitTenantConfigArg &arg,
+    obrpc::ObInitTenantConfigRes &result)
+{
+  int ret = OB_SUCCESS;
+  if (!inited_) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("service is not inited", K(ret));
+  } else if (!arg.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("arg is invalid", KR(ret), K(arg));
+  } else {
+    for (int64_t i = 0; OB_SUCC(ret) && i < arg.get_tenant_configs().count(); i++) {
+      const ObTenantConfigArg &config = arg.get_tenant_configs().at(i);
+      if (OB_FAIL(OTC_MGR.init_tenant_config(config)))  {
+        LOG_WARN("fail to init tenant config", KR(ret), K(config));
+      }
+    } // end for
+  }
+  (void) result.set_ret(ret);
+  FLOG_INFO("init tenant config", KR(ret), K(arg));
+  // use result to pass ret
+  return OB_SUCCESS;
+}
+
 }// end namespace observer
 }// end namespace oceanbase

src/observer/ob_service.h

@@ -224,6 +224,9 @@ public:
   int submit_async_refresh_schema_task(const uint64_t tenant_id, const int64_t schema_version);
   int renew_in_zone_hb(const share::ObInZoneHbRequest &arg,
                        share::ObInZoneHbResponse &result);
+  int init_tenant_config(
+      const obrpc::ObInitTenantConfigArg &arg,
+      obrpc::ObInitTenantConfigRes &result);
 private:
   int inner_fill_tablet_info_(
       const int64_t tenant_id,

src/observer/ob_srv_xlator_primary.cpp

@@ -69,6 +69,7 @@ void oceanbase::observer::init_srv_xlator_for_sys(ObSrvRpcXlator *xlator) {
   RPC_PROCESSOR(ObCheckFrozenVersionP, gctx_);
   RPC_PROCESSOR(ObGetDiagnoseArgsP);
   RPC_PROCESSOR(ObGetMinSSTableSchemaVersionP, gctx_);
+  RPC_PROCESSOR(ObInitTenantConfigP, gctx_);
   RPC_PROCESSOR(ObBatchBroadcastSchemaP, gctx_);
 
   // interrupt

src/observer/omt/ob_tenant_config.cpp

@@ -357,65 +357,84 @@ int ObTenantConfig::update_local(int64_t expected_version, ObMySQLProxy::MySQLRe
   return ret;
 }
 
-int ObTenantConfig::add_extra_config(char *config_str,
+int ObTenantConfig::add_extra_config(const char *config_str,
                                      int64_t version /* = 0 */ ,
                                      bool check_name /* = false */)
 {
   int ret = OB_SUCCESS;
   const int64_t MAX_OPTS_LENGTH = sysconf(_SC_ARG_MAX);
+  int64_t config_str_length = 0;
+  char *buf = NULL;
   char *saveptr = NULL;
   char *token = NULL;
-  DRWLock::RDLockGuard lguard(ObConfigManager::get_serialize_lock());
-  DRWLock::WRLockGuard guard(lock_);
-  token = STRTOK_R(config_str, ",\n", &saveptr);
-  while (OB_SUCC(ret) && OB_LIKELY(NULL != token)) {
-    char *saveptr_one = NULL;
-    const char *name = NULL;
-    const char *value = NULL;
-    ObConfigItem *const *pp_item = NULL;
-    if (OB_ISNULL(name = STRTOK_R(token, "=", &saveptr_one))) {
-      ret = OB_INVALID_CONFIG;
-      LOG_ERROR("Invalid config string", K(token), K(ret));
-    } else if (OB_ISNULL(saveptr_one) || OB_UNLIKELY('\0' == *(value = saveptr_one))) {
-      LOG_INFO("Empty config string", K(token), K(name));
-      // ret = OB_INVALID_CONFIG;
-      name = "";
-    }
-    if (OB_SUCC(ret)) {
-      const int value_len = strlen(value);
-      // hex2cstring -> value_len / 2 + 1
-      // '\0' -> 1
-      const int external_info_val_len = value_len / 2 + 1 + 1;
-      char *external_info_val = (char*)ob_malloc(external_info_val_len, "temp");
-      DEFER(if (external_info_val != nullptr) ob_free(external_info_val););
-      if (OB_ISNULL(external_info_val)) {
-        ret = OB_ALLOCATE_MEMORY_FAILED;
-        LOG_WARN("failed to alloc", K(ret));
-      } else if (FALSE_IT(external_info_val[0] = '\0')) {
-      } else if (OB_ISNULL(pp_item = container_.get(ObConfigStringKey(name)))) {
-        /* make compatible with previous configuration */
-        ret = check_name ? OB_INVALID_CONFIG : OB_SUCCESS;
-        LOG_WARN("Invalid config string, no such config item", K(name), K(value), K(ret));
+  if (OB_ISNULL(config_str)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("config str is null", K(ret));
+  } else if ((config_str_length = static_cast<int64_t>(STRLEN(config_str))) >= MAX_OPTS_LENGTH) {
+    ret = OB_BUF_NOT_ENOUGH;
+    LOG_ERROR("Extra config is too long", K(ret));
+  } else if (OB_ISNULL(buf = new (std::nothrow) char[config_str_length + 1])) {
+    ret = OB_ALLOCATE_MEMORY_FAILED;
+    LOG_ERROR("ob tc malloc memory for buf fail", K(ret));
+  } else {
+    MEMCPY(buf, config_str, config_str_length);
+    buf[config_str_length] = '\0';
+    DRWLock::RDLockGuard lguard(ObConfigManager::get_serialize_lock());
+    DRWLock::WRLockGuard guard(lock_);
+    token = STRTOK_R(buf, ",\n", &saveptr);
+    while (OB_SUCC(ret) && OB_LIKELY(NULL != token)) {
+      char *saveptr_one = NULL;
+      const char *name = NULL;
+      const char *value = NULL;
+      ObConfigItem *const *pp_item = NULL;
+      if (OB_ISNULL(name = STRTOK_R(token, "=", &saveptr_one))) {
+        ret = OB_INVALID_CONFIG;
+        LOG_ERROR("Invalid config string", K(token), K(ret));
+      } else if (OB_ISNULL(saveptr_one) || OB_UNLIKELY('\0' == *(value = saveptr_one))) {
+        LOG_INFO("Empty config string", K(token), K(name));
+        // ret = OB_INVALID_CONFIG;
+        name = "";
       }
-      if (OB_FAIL(ret) || OB_ISNULL(pp_item)) {
-      } else if (!(*pp_item)->set_value(value)) {
-        ret = OB_INVALID_CONFIG;
-        LOG_WARN("Invalid config value", K(name), K(value), K(ret));
-      } else if (!(*pp_item)->check()) {
-        ret = OB_INVALID_CONFIG;
-        const char* range = (*pp_item)->range();
-        if (OB_ISNULL(range) || strlen(range) == 0) {
-          LOG_ERROR("Invalid config, value out of range", K(name), K(value), K(ret));
-        } else {
-          _LOG_ERROR("Invalid config, value out of %s (for reference only). name=%s, value=%s, ret=%d", range, name, value, ret);
+      if (OB_SUCC(ret)) {
+        const int value_len = strlen(value);
+        // hex2cstring -> value_len / 2 + 1
+        // '\0' -> 1
+        const int external_info_val_len = value_len / 2 + 1 + 1;
+        char *external_info_val = (char*)ob_malloc(external_info_val_len, "temp");
+        DEFER(if (external_info_val != nullptr) ob_free(external_info_val););
+        if (OB_ISNULL(external_info_val)) {
+          ret = OB_ALLOCATE_MEMORY_FAILED;
+          LOG_WARN("failed to alloc", K(ret));
+        } else if (FALSE_IT(external_info_val[0] = '\0')) {
+        } else if (OB_ISNULL(pp_item = container_.get(ObConfigStringKey(name)))) {
+          /* make compatible with previous configuration */
+          ret = check_name ? OB_INVALID_CONFIG : OB_SUCCESS;
+          LOG_WARN("Invalid config string, no such config item", K(name), K(value), K(ret));
         }
-      } else {
-        (*pp_item)->set_version(version);
-        LOG_INFO("Load tenant config succ", K(name), K(value));
+        if (OB_FAIL(ret) || OB_ISNULL(pp_item)) {
+        } else if (!(*pp_item)->set_value(value)) {
+          ret = OB_INVALID_CONFIG;
+          LOG_WARN("Invalid config value", K(name), K(value), K(ret));
+        } else if (!(*pp_item)->check()) {
+          ret = OB_INVALID_CONFIG;
+          const char* range = (*pp_item)->range();
+          if (OB_ISNULL(range) || strlen(range) == 0) {
+            LOG_ERROR("Invalid config, value out of range", K(name), K(value), K(ret));
+          } else {
+            _LOG_ERROR("Invalid config, value out of %s (for reference only). name=%s, value=%s, ret=%d", range, name, value, ret);
+          }
+        } else {
+          (*pp_item)->set_version(version);
+          LOG_INFO("Load tenant config succ", K(name), K(value));
+        }
+        token = STRTOK_R(NULL, ",\n", &saveptr);
       }
-      token = STRTOK_R(NULL, ",\n", &saveptr);
     }
   }
+  if (NULL != buf) {
+    delete [] buf;
+    buf = NULL;
+  }
   return ret;
 }
 

src/observer/omt/ob_tenant_config.h

@@ -89,9 +89,9 @@ public:
   int got_version(int64_t version, const bool remove_repeat);
   int update_local(int64_t expected_version, common::ObMySQLProxy::MySQLResult &result,
                    bool save2file = true);
-  int add_extra_config(char *config_str,
-                   int64_t version = 0 ,
-                   bool check_name = false);
+  int add_extra_config(const char *config_str,
+                       int64_t version = 0 ,
+                       bool check_name = false);
 
   OB_UNIS_VERSION(1);
 private:

src/observer/omt/ob_tenant_config_mgr.cpp

@@ -246,6 +246,21 @@ int ObTenantConfigMgr::refresh_tenants(const ObIArray<uint64_t> &tenants)
   return ret;
 }
 
+// This function will be called in the early stage in bootstrap/create tenant.
+// Meanwhile, related tenant's tables are not readable, so it's safe to call add_extra_config().
+int ObTenantConfigMgr::init_tenant_config(const obrpc::ObTenantConfigArg &arg)
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(add_tenant_config(arg.tenant_id_))) {
+    LOG_WARN("fail to add tenant config", KR(ret), K(arg));
+  } else if (OB_FAIL(add_extra_config(arg))) {
+    LOG_WARN("fail to add extra config", KR(ret), K(arg));
+  } else if (OB_FAIL(dump2file())) {
+    LOG_WARN("fail to dump config to file", KR(ret), K(arg));
+  }
+  return ret;
+}
+
 int ObTenantConfigMgr::add_tenant_config(uint64_t tenant_id)
 {
   int ret = OB_SUCCESS;
@@ -597,7 +612,7 @@ void ObTenantConfigMgr::notify_tenant_config_changed(uint64_t tenant_id)
   update_tenant_config_cb_(tenant_id);
 }
 
-int ObTenantConfigMgr::add_extra_config(obrpc::ObTenantConfigArg &arg)
+int ObTenantConfigMgr::add_extra_config(const obrpc::ObTenantConfigArg &arg)
 {
   int ret = OB_SUCCESS;
   ObTenantConfig *config = nullptr;
@@ -612,7 +627,7 @@ int ObTenantConfigMgr::add_extra_config(obrpc::ObTenantConfigArg &arg)
       ret = config->add_extra_config(arg.config_str_.ptr());
     }
   }
-  LOG_INFO("add tenant extra config", K(arg));
+  FLOG_INFO("add tenant extra config", K(arg));
   return ret;
 }
 

src/observer/omt/ob_tenant_config_mgr.h

@@ -113,6 +113,7 @@ public:
   int refresh_tenants(const common::ObIArray<uint64_t> &tenants);
   int add_tenant_config(uint64_t tenant_id);
   int del_tenant_config(uint64_t tenant_id);
+  int init_tenant_config(const obrpc::ObTenantConfigArg &arg);
 
   ObTenantConfig *get_tenant_config(uint64_t tenant_id) const;
   // lock to guarantee this will not be deleted by calling del_tenant_config.
@@ -133,6 +134,7 @@ public:
   int dump2file(const char *path = nullptr) const;
 
   void refresh_config_version_map(const common::ObIArray<uint64_t> &tenants);
+  void reset_version_has_refreshed() { version_has_refreshed_ = false; }
   int set_tenant_config_version(uint64_t tenant_id, int64_t version);
   int64_t get_tenant_config_version(uint64_t tenant_id);
   void get_lease_request(share::ObLeaseRequest &lease_request);
@@ -142,7 +144,7 @@ public:
   int got_version(uint64_t tenant_id, int64_t version, const bool remove_repeat = true);
   int update_local(uint64_t tenant_id, int64_t expected_version);
   void notify_tenant_config_changed(uint64_t tenatn_id);
-  int add_extra_config(obrpc::ObTenantConfigArg &arg);
+  int add_extra_config(const obrpc::ObTenantConfigArg &arg);
   int schedule(ObTenantConfig::TenantConfigUpdateTask &task, const int64_t delay);
   int cancel(const ObTenantConfig::TenantConfigUpdateTask &task);
   int wait(const ObTenantConfig::TenantConfigUpdateTask &task);

src/observer/virtual_table/ob_agent_table_base.cpp

@@ -18,6 +18,7 @@
 #include "observer/ob_inner_sql_result.h"
 #include "lib/string/ob_sql_string.h"
 #include "share/schema/ob_schema_utils.h"
+#include "common/object/ob_object.h"
 
 namespace oceanbase
 {
@@ -184,13 +185,54 @@ int ObAgentTableBase::build_base_table_mapping()
   return ret;
 }
 
-int ObAgentTableBase::construct_sql(common::ObSqlString &sql)
+int ObAgentTableBase::construct_sql(
+    const uint64_t exec_tenant_id,
+    common::ObSqlString &sql)
 {
   int ret = OB_SUCCESS;
-  if (OB_ISNULL(table_schema_) || OB_ISNULL(base_table_)) {
+  if (OB_ISNULL(base_table_)) {
     ret = OB_NOT_INIT;
-    LOG_WARN("table is NULL", K(ret));
-  } else {
+    LOG_WARN("table is NULL", KR(ret), KP(base_table_));
+  } else if (OB_FAIL(construct_columns(exec_tenant_id, sql))) {
+    LOG_WARN("fail to construct columns", KR(ret), K(exec_tenant_id));
+  } else if (OB_FAIL(sql.append_fmt(" FROM `%s` WHERE 1=1", base_table_->get_table_name()))) {
+    LOG_WARN("append sql failed", KR(ret));
+  } else if (OB_FAIL(append_sql_condition(sql))) {
+    LOG_WARN("append condition failed", KR(ret));
+  } else if (OB_FAIL(add_extra_condition(sql))) {
+    LOG_WARN("append extra condition failed", KR(ret));
+  } else if (OB_FAIL(append_sql_orderby(sql))) {
+    LOG_WARN("append order by failed", KR(ret));
+  }
+  return ret;
+}
+
+int ObAgentTableBase::construct_columns(
+    const uint64_t exec_tenant_id,
+    common::ObSqlString &sql)
+{
+  int ret = OB_SUCCESS;
+  // For compatiblity, iterate like virtual table should cast missing column with default value.
+  bool cast_default_value = (base_tenant_id_ != exec_tenant_id);
+  const ObTableSchema *exec_table_schema = NULL;
+  if (OB_ISNULL(base_table_)
+      || OB_ISNULL(schema_guard_)
+      || OB_ISNULL(table_schema_)) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("table is NULL", KR(ret), KP(base_table_), K(table_schema_), KP(schema_guard_));
+  } else if (!cast_default_value) {
+    // skip
+  } else if (OB_FAIL(schema_guard_->get_table_schema(
+             exec_tenant_id, base_table_id_, exec_table_schema))) {
+    LOG_WARN("fail to get table schema", KR(ret), K(exec_tenant_id), K(base_table_id_));
+  } else if (OB_ISNULL(exec_table_schema)) {
+    ret = OB_TABLE_NOT_EXIST;
+    LOG_WARN("exec table not exist", KR(ret), K(exec_tenant_id), K(base_table_id_));
+  }
+
+  if (OB_SUCC(ret)) {
+    const int64_t buf_len = 100;
+    char *buf = NULL;
     // 对query timeout取10的指数次，防止系统租户计划缓存数过大
     int64_t rest_time = scan_param_->timeout_ - ObTimeUtility::current_time();
     int64_t query_timeout = 1;
@@ -198,30 +240,131 @@ int ObAgentTableBase::construct_sql(common::ObSqlString &sql)
       query_timeout = query_timeout * 10;
       rest_time = rest_time / 10;
     }
-
-    if (OB_FAIL(sql.assign_fmt("SELECT /*+ query_timeout(%ld) */ ",
-        query_timeout))) {
-      LOG_WARN("append sql failed", K(ret));
-    } else if (scan_param_->column_ids_.empty() && 
-               OB_FAIL(sql.append("1"))) {
-      LOG_WARN("failed to append dummy select expr", K(ret));
+    if (OB_ISNULL(buf = static_cast<char*>(allocator_->alloc(buf_len)))) {
+      ret = OB_ALLOCATE_MEMORY_FAILED;
+      LOG_WARN("fail to alloc buf", KR(ret), K(buf_len));
+    } else if (OB_FAIL(sql.assign_fmt("SELECT /*+ query_timeout(%ld) */ ", query_timeout))) {
+      LOG_WARN("append sql failed", KR(ret));
+    } else if (scan_param_->column_ids_.empty() && OB_FAIL(sql.append("1"))) {
+      LOG_WARN("failed to append dummy select expr", KR(ret));
     }
     FOREACH_CNT_X(c, scan_param_->column_ids_, OB_SUCC(ret)) {
-      const ObString &name = mapping_.at(*c).base_col_name_;
-      if (OB_FAIL(sql.append_fmt("%s%.*s",
-          0 == __INNER_I__(c) ? "" : ", ", name.length(), name.ptr()))) {
-        LOG_WARN("append sql failed", K(ret));
+      const uint64_t column_id = *c;
+      const ObString &name = mapping_.at(column_id).base_col_name_;
+      ObString col_name;
+      bool need_cast_default_value = false;
+      if (cast_default_value
+          && 0 != name.case_compare("effective_tenant_id()")) {
+        bool column_exist = false;
+        (void) table_schema_->get_column_name_by_column_id(column_id, col_name, column_exist);
+        if (!column_exist) {
+          ret = OB_ERR_UNEXPECTED;
+          LOG_WARN("column should be exist", KR(ret), K(column_id), K_(table_schema));
+        } else if (OB_ISNULL(exec_table_schema->get_column_schema(col_name))) {
+          // column not exist in target tenant's table schema
+          need_cast_default_value = true;
+        }
+      }
+      if (OB_FAIL(ret)) {
+      } else if (!need_cast_default_value) {
+        if (OB_FAIL(sql.append_fmt("%s%.*s",
+            0 == __INNER_I__(c) ? "" : ", ", name.length(), name.ptr()))) {
+          LOG_WARN("append sql failed", KR(ret));
+        }
+      } else if (OB_FAIL(cast_as_default_value(
+                 0 == __INNER_I__(c),
+                 buf, buf_len, name, col_name, sql))) {
+        LOG_WARN("fail to cast default value", KR(ret), K(col_name));
+      }
+    } // end FOREACH_CNT_X
+  }
+  return ret;
+}
+
+// Supported column types for system table:
+// 1. varchar、varbinary
+// 2. longtext: not support to cast default value to longtext.
+// 3. timestamp: default value can only be `current_timestamp(x)`, not support cast datetime to timestamp.
+// 4. double、float、number: can't have default value, cast may failed.
+// 5. bool
+// 6. bigint、int
+// 7. uint
+int ObAgentTableBase::cast_as_default_value(
+    const bool first_column,
+    char *buf,
+    const int64_t buf_len,
+    const common::ObString &name,
+    const common::ObString &col_name,
+    common::ObSqlString &sql)
+{
+  int ret = OB_SUCCESS;
+  if (OB_ISNULL(base_table_) || OB_ISNULL(buf) || buf_len <= 0) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("invalid arg", KR(ret), KP(base_table_), KP(buf), K(buf_len));
+  } else {
+    const ObColumnSchemaV2 *column = base_table_->get_column_schema(col_name);
+    if (OB_ISNULL(column)) {
+      ret = OB_ENTRY_NOT_EXIST;
+      LOG_WARN("column not found", KR(ret), K(base_tenant_id_),
+               K(base_table_id_), K(col_name));
+    } else if (column->is_nullable()
+               && !column->is_not_null_validate_column()) {
+      if (OB_FAIL(sql.append_fmt("%s NULL AS %.*s",
+          first_column ? "" : ", ",
+          name.length(), name.ptr()))) {
+        LOG_WARN("append sql failed", KR(ret));
+      }
+    } else {
+      // For new adding columns, column which is longtext/timestamp/double/float/number should be nullable.
+      ObObj default_value = column->get_cur_default_value();
+      default_value.set_scale(column->get_data_scale());
+      if (column->get_meta_type().is_varchar()
+          || column->get_meta_type().is_varbinary()) {
+        // 1. varchar, varbinary
+        if (OB_FAIL(sql.append_fmt("%s '%s' AS %.*s",
+            first_column ? "" : ", ",
+            to_cstring(ObHexEscapeSqlStr(default_value.get_string())),
+            name.length(), name.ptr()))) {
+          LOG_WARN("append sql failed", KR(ret));
+        }
+      } else if (column->get_meta_type().is_signed_integer()
+                 || column->get_meta_type().is_unsigned()) {
+        lib::CompatModeGuard g(lib::Worker::CompatMode::MYSQL);
+        MEMSET(buf, '\0', buf_len);
+        int64_t len = 0;
+        if (OB_FAIL(default_value.print_plain_str_literal(buf, buf_len, len))) {
+          LOG_WARN("fail to print varchar", KR(ret), K(default_value));
+        } else if (column->get_meta_type().is_signed_integer()) {
+          if (column->get_meta_type().is_tinyint()) {
+            // 2. bool
+            if (OB_FAIL(sql.append_fmt("%s %s AS %.*s",
+                       first_column ? "" : ", ",
+                       default_value.get_tinyint() ? "True" : "False",
+                       name.length(), name.ptr()))) {
+              LOG_WARN("append sql failed", KR(ret), K(default_value));
+            }
+          } else {
+            // 3. bigint、int
+            if (OB_FAIL(sql.append_fmt("%s %.*s AS %.*s",
+                       first_column ? "" : ", ",
+                       static_cast<int>(len), buf,
+                       name.length(), name.ptr()))) {
+              LOG_WARN("append sql failed", KR(ret), K(default_value));
+            }
+          }
+        } else {
+          // 4. uint
+          if (OB_FAIL(sql.append_fmt("%s CAST(%.*s AS UNSIGNED) AS %.*s",
+                     first_column ? "" : ", ",
+                     static_cast<int>(len), buf,
+                     name.length(), name.ptr()))) {
+            LOG_WARN("append sql failed", KR(ret), K(default_value));
+          }
+        }
+      } else {
+        ret = OB_NOT_SUPPORTED;
+        LOG_WARN("not supported column type", KR(ret), KPC(column));
       }
-    }
-    if (OB_FAIL(ret)) {
-    } else if (OB_FAIL(sql.append_fmt(" FROM `%s` WHERE 1=1", base_table_->get_table_name()))) {
-      LOG_WARN("append sql failed", K(ret));
-    } else if (OB_FAIL(append_sql_condition(sql))) {
-      LOG_WARN("append condition failed", K(ret));
-    } else if (OB_FAIL(add_extra_condition(sql))) {
-      LOG_WARN("append extra condition failed", K(ret));
-    } else if (OB_FAIL(append_sql_orderby(sql))) {
-      LOG_WARN("append order by failed", K(ret));
     }
   }
   return ret;

src/observer/virtual_table/ob_agent_table_base.h

@@ -90,7 +90,17 @@ protected:
   virtual int setup_inital_rowkey_condition(
       common::ObSqlString &cols, common::ObSqlString &vals);
 
-  int construct_sql(common::ObSqlString &sql);
+  int construct_sql(const uint64_t exec_tenant_id, common::ObSqlString &sql);
+  int construct_columns(
+      const uint64_t exec_tenant_id,
+      common::ObSqlString &sql);
+  int cast_as_default_value(
+      const bool first_column,
+      char *buf,
+      const int64_t buf_len,
+      const common::ObString &name,
+      const common::ObString &col_name,
+      common::ObSqlString &sql);
   // convert scan_param_->key_ranges_ to sql conditions
   int append_sql_condition(common::ObSqlString &sql);
   int rowkey2condition(common::ObSqlString &cols,  common::ObSqlString &vals,

src/observer/virtual_table/ob_agent_virtual_table.cpp

@@ -146,23 +146,20 @@ int ObAgentVirtualTable::init(
 int ObAgentVirtualTable::do_open()
 {
   int ret = OB_SUCCESS;
+  ObSqlString sql;
   if (OB_FAIL(ObAgentTableBase::do_open())) {
-    LOG_WARN("base agent table open failed", K(ret));
+    LOG_WARN("base agent table open failed", KR(ret));
+  } else if (OB_FAIL(construct_sql(base_tenant_id_, sql))) {
+    LOG_WARN("construct sql failed", KR(ret), K(base_tenant_id_));
+  } else if (OB_FAIL(GCTX.sql_proxy_->read(*sql_res_, base_tenant_id_, sql.ptr()))) {
+    LOG_WARN("execute sql failed", KR(ret), K(base_tenant_id_), K(sql));
+  } else if (OB_ISNULL(sql_res_->get_result())) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("NULL sql executing result", KR(ret), K(base_tenant_id_), K(sql));
   } else {
-    ObSqlString sql;
-    if (OB_FAIL(ret)) {
-    } else if (OB_FAIL(construct_sql(sql))) {
-      LOG_WARN("construct sql failed", K(ret));
-    } else if (OB_FAIL(GCTX.sql_proxy_->read(*sql_res_, base_tenant_id_, sql.ptr()))) {
-      LOG_WARN("execute sql failed", K(ret), K(base_tenant_id_), K(sql));
-    } else if (OB_ISNULL(sql_res_->get_result())) {
-      ret = OB_ERR_UNEXPECTED;
-      LOG_WARN("NULL sql executing result", K(ret));
-    } else {
-      inner_sql_res_ = static_cast<ObInnerSQLResult *>(sql_res_->get_result());
-      if (general_tenant_id_ != OB_INVALID_TENANT_ID) {
-        inner_sql_res_->result_set().get_session().switch_tenant(general_tenant_id_);
-      }
+    inner_sql_res_ = static_cast<ObInnerSQLResult *>(sql_res_->get_result());
+    if (general_tenant_id_ != OB_INVALID_TENANT_ID) {
+      inner_sql_res_->result_set().get_session().switch_tenant(general_tenant_id_);
     }
   }
 

src/observer/virtual_table/ob_all_virtual_tenant_parameter_stat.cpp

@@ -268,7 +268,14 @@ int ObAllVirtualTenantParameterStat::fill_row_(common::ObNewRow *&row,
             break;
           }
           case VALUE: {
-            cells[i].set_varchar(iter->second->str());
+            if (0 == ObString("compatible").case_compare(iter->first.str())
+                && !iter->second->value_updated()) {
+              // `compatible` is used for tenant compatibility,
+              // default value should not be used when `compatible` is not loaded yet.
+              cells[i].set_varchar("0.0.0.0");
+            } else {
+              cells[i].set_varchar(iter->second->str());
+            }
             cells[i].set_collation_type(
                 ObCharset::get_default_collation(ObCharset::get_default_charset()));
             break;

src/observer/virtual_table/ob_iterate_private_virtual_table.cpp

@@ -97,12 +97,23 @@ int ObIteratePrivateVirtualTable::do_open()
             && (is_sys_tenant(effective_tenant_id_)
                 || *id == effective_tenant_id_)) {
           const ObSimpleTenantSchema *simple_tenant_schema = NULL;
-          if (OB_FAIL(schema_guard_->get_tenant_info(get_exec_tenant_id_(*id), simple_tenant_schema))) {
-            LOG_WARN("fail to get tenant schema", KR(ret), K(*id), "exec_tenant_id", get_exec_tenant_id_(*id));
+          const uint64_t exec_tenant_id = get_exec_tenant_id_(*id);
+          bool exist = false;
+          if (OB_FAIL(schema_guard_->get_tenant_info(exec_tenant_id, simple_tenant_schema))) {
+            LOG_WARN("fail to get tenant schema", KR(ret), K(*id), K(exec_tenant_id));
           } else if (OB_ISNULL(simple_tenant_schema)) {
             ret = OB_ERR_UNEXPECTED;
-            LOG_WARN("tenant schema is nullptr", KR(ret), K(*id), "exec_tenant_id", get_exec_tenant_id_(*id));
-          } else if (simple_tenant_schema->is_normal() && OB_FAIL(tenants_.push_back(*id))) {
+            LOG_WARN("tenant schema is nullptr", KR(ret), K(*id), K(exec_tenant_id));
+          } else if (!simple_tenant_schema->is_normal()) {
+            LOG_TRACE("tenant status is not normal, just skip", K(exec_tenant_id));
+          } else if (OB_FAIL(schema_guard_->check_table_exist(
+                     exec_tenant_id, base_table_id_, exist))) {
+            LOG_WARN("fail to check table exist",
+                     KR(ret), K(exec_tenant_id), K(base_table_id_));
+          } else if (!exist) {
+            LOG_TRACE("table not exist in tenant, maybe in upgrade process, just skip",
+                      K(exec_tenant_id), K(base_table_id_));
+          } else if (OB_FAIL(tenants_.push_back(*id))) {
             LOG_WARN("array push back failed", KR(ret));
           }
         }
@@ -232,10 +243,11 @@ int ObIteratePrivateVirtualTable::next_tenant_()
       LOG_WARN("sql or sql result no init", KR(ret), KP(sql_res_), K(sql_));
     } else if (OB_FAIL(sql_.set_length(0))) {
       LOG_WARN("reset sql failed", KR(ret));
-    } else if (OB_FAIL(construct_sql(sql_))) {
-      LOG_WARN("construct sql failed", KR(ret));
+    } else if (OB_FAIL(construct_sql(exec_tenant_id, sql_))) {
+      LOG_WARN("construct sql failed", KR(ret), K(exec_tenant_id));
     } else {
-      LOG_TRACE("construct iterate private virtual table sql", K(exec_tenant_id), K_(cur_tenant_id), K_(sql));
+      LOG_TRACE("construct iterate private virtual table sql",
+                K(exec_tenant_id), K_(cur_tenant_id), K_(sql));
       sql_res_->~ReadResult();
       inner_sql_res_ = NULL;
       new (sql_res_) ObMySQLProxy::MySQLResult();

src/observer/virtual_table/ob_iterate_virtual_table.cpp

@@ -98,7 +98,16 @@ int ObIterateVirtualTable::do_open()
         if (in_range
             && (is_sys_tenant(effective_tenant_id_)
                 || *id == effective_tenant_id_)) {
-          if (OB_FAIL(tenants_.push_back(*id))) {
+          bool exist = false;
+          if (OB_FAIL(schema_guard_->check_table_exist(*id, base_table_id_, exist))) {
+            LOG_WARN("fail to check table exist",
+                     KR(ret), "tenant_id", *id, K(base_table_id_));
+          } else if (!exist) {
+            // 1. in upgrade process
+            // 2. restore tenant from low data version
+            LOG_TRACE("table not exist in tenant, maybe inner schemas are old, just skip",
+                      "tenant_id", *id, K(base_table_id_));
+          } else if (OB_FAIL(tenants_.push_back(*id))) {
             LOG_WARN("array push back failed", KR(ret));
           }
         }
@@ -234,21 +243,21 @@ int ObIterateVirtualTable::next_tenant()
     ObSQLClientRetryWeak sql_client_retry_weak(GCTX.sql_proxy_, exec_tenant_id, base_table_id_);
     if (OB_ISNULL(sql_res_)) {
       ret = OB_ERR_UNEXPECTED;
-      LOG_WARN("sql or sql result no init", K(ret), KP(sql_res_), K(sql_));
+      LOG_WARN("sql or sql result no init", KR(ret), K(exec_tenant_id), KP(sql_res_), K(sql_));
     } else if (OB_FAIL(sql_.set_length(0))) {
-      LOG_WARN("reset sql failed", K(ret));
-    } else if (OB_FAIL(construct_sql(sql_))) {
-      LOG_WARN("construct sql failed", K(ret));
+      LOG_WARN("reset sql failed", KR(ret), K(exec_tenant_id), K(sql_));
+    } else if (OB_FAIL(construct_sql(exec_tenant_id, sql_))) {
+      LOG_WARN("construct sql failed", KR(ret), K(exec_tenant_id), K(sql_));
     } else {
       sql_res_->~ReadResult();
       inner_sql_res_ = NULL;
       new (sql_res_) ObMySQLProxy::MySQLResult();
-      LOG_DEBUG("execute sql", K(cur_tenant_id_), K(sql_));
+      LOG_TRACE("execute sql", K(exec_tenant_id), K(sql_));
       if (OB_FAIL(sql_client_retry_weak.read(*sql_res_, exec_tenant_id, sql_.ptr()))) {
-        LOG_WARN("execute sql failed", K(ret), K(sql_));
+        LOG_WARN("execute sql failed", KR(ret), K(exec_tenant_id), K(sql_));
       } else if (OB_ISNULL(sql_res_->get_result())) {
         ret = OB_ERR_UNEXPECTED;
-        LOG_WARN("NULL sql executing result", K(ret));
+        LOG_WARN("NULL sql executing result", KR(ret), K(exec_tenant_id), K(sql_));
       } else {
         inner_sql_res_ = static_cast<ObInnerSQLResult *>(sql_res_->get_result());
       }

src/rootserver/freeze/ob_freeze_info_manager.cpp

@@ -299,6 +299,7 @@ int ObFreezeInfoManager::set_freeze_info()
   ObSimpleFrozenStatus frozen_status;
   if (OB_SUCC(ret)) {
     int64_t schema_version_in_frozen_ts = 0;
+    uint64_t data_version = 0;
 
     // 3. generate new frozen_scn
     if (OB_FAIL(generate_frozen_scn(freeze_info_, remote_snapshot_gc_scn, new_frozen_scn))) {
@@ -306,10 +307,12 @@ int ObFreezeInfoManager::set_freeze_info()
     // 4. get schema_version at frozen_scn
     } else if (OB_FAIL(get_schema_version(new_frozen_scn, schema_version_in_frozen_ts))) {
       LOG_WARN("fail to get schema version", KR(ret), K(new_frozen_scn));
+    } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id_, data_version))) {
+      LOG_WARN("fail to get min data version", KR(ret), K_(tenant_id));
     } else {
       frozen_status.frozen_scn_ = new_frozen_scn;
       frozen_status.schema_version_ = schema_version_in_frozen_ts;
-      frozen_status.cluster_version_ = GET_MIN_CLUSTER_VERSION();
+      frozen_status.data_version_ = data_version;
 
       // 5. insert freeze info
       if (OB_FAIL(freeze_info_proxy.set_freeze_info(trans, frozen_status))) {

src/rootserver/ob_bootstrap.cpp

@@ -236,25 +236,27 @@ int ObPreBootstrap::prepare_bootstrap(ObAddr &master_rs)
   bool match = false;
   begin_ts_ = ObTimeUtility::current_time();
   if (OB_FAIL(check_inner_stat())) {
-    LOG_WARN("check_inner_stat failed", K(ret));
+    LOG_WARN("check_inner_stat failed", KR(ret));
   } else if (OB_FAIL(check_bootstrap_rs_list(rs_list_))) {
-    LOG_WARN("failed to check_bootstrap_rs_list", K_(rs_list), K(ret));
+    LOG_WARN("failed to check_bootstrap_rs_list", KR(ret), K_(rs_list));
   } else if (OB_FAIL(check_all_server_bootstrap_mode_match(match))) {
-    LOG_WARN("fail to check all server bootstrap mode match", K(ret));
+    LOG_WARN("fail to check all server bootstrap mode match", KR(ret));
   } else if (!match) {
     ret = OB_NOT_SUPPORTED;
-    LOG_WARN("cannot do bootstrap with different bootstrap mode on servers", K(ret));
+    LOG_WARN("cannot do bootstrap with different bootstrap mode on servers", KR(ret));
   } else if (OB_FAIL(check_is_all_server_empty(is_empty))) {
-    LOG_WARN("failed to check bootstrap stat", K(ret));
+    LOG_WARN("failed to check bootstrap stat", KR(ret));
   } else if (!is_empty) {
     ret = OB_INIT_TWICE;
-    LOG_WARN("cannot do bootstrap on not empty server", K(ret));
+    LOG_WARN("cannot do bootstrap on not empty server", KR(ret));
   } else if (OB_FAIL(notify_sys_tenant_server_unit_resource())) {
-    LOG_WARN("fail to notify sys tenant server unit resource", K(ret));
+    LOG_WARN("fail to notify sys tenant server unit resource", KR(ret));
+  } else if (OB_FAIL(notify_sys_tenant_config_())) {
+    LOG_WARN("fail to notify sys tenant config", KR(ret));
   } else if (OB_FAIL(create_ls())) {
-    LOG_WARN("failed to create core table partition", K(ret));
+    LOG_WARN("failed to create core table partition", KR(ret));
   } else if (OB_FAIL(wait_elect_ls(master_rs))) {
-    LOG_WARN("failed to wait elect master partition", K(ret));
+    LOG_WARN("failed to wait elect master partition", KR(ret));
   }
   BOOTSTRAP_CHECK_SUCCESS();
   return ret;
@@ -311,6 +313,33 @@ int ObPreBootstrap::notify_sys_tenant_server_unit_resource()
   return ret;
 }
 
+int ObPreBootstrap::notify_sys_tenant_config_()
+{
+  int ret = OB_SUCCESS;
+  common::ObConfigPairs config;
+  common::ObSEArray<common::ObConfigPairs, 1> init_configs;
+  ObArray<ObAddr> addrs;
+  if (OB_FAIL(ObDDLService::gen_tenant_init_config(
+      OB_SYS_TENANT_ID, DATA_CURRENT_VERSION, config))) {
+  } else if (OB_FAIL(init_configs.push_back(config))) {
+    LOG_WARN("fail to push back config", KR(ret), K(config));
+  } else if (OB_FAIL(addrs.reserve(rs_list_.count()))) {
+    LOG_WARN("fail to reserve array", KR(ret));
+  }
+  for (int64_t i = 0; OB_SUCC(ret) && i < rs_list_.count(); i++) {
+    if (OB_FAIL(addrs.push_back(rs_list_[i].server_))) {
+      LOG_WARN("fail to push back server", KR(ret));
+    }
+  } // end for
+  if (FAILEDx(ObDDLService::notify_init_tenant_config(
+              rpc_proxy_, init_configs, addrs))) {
+    LOG_WARN("fail to notify init tenant config", KR(ret), K(init_configs), K(addrs));
+  }
+
+  BOOTSTRAP_CHECK_SUCCESS();
+  return ret;
+}
+
 int ObPreBootstrap::create_ls()
 {
   int ret = OB_SUCCESS;
@@ -396,8 +425,8 @@ int ObPreBootstrap::check_is_all_server_empty(bool &is_empty)
   if (OB_FAIL(check_inner_stat())) {
     LOG_WARN("check_inner_stat failed", K(ret));
   } else {
-    ObCheckServerEmptyArg arg;
-    arg.mode_ = ObCheckServerEmptyArg::BOOTSTRAP;
+    ObCheckServerEmptyArg arg(ObCheckServerEmptyArg::BOOTSTRAP,
+                              DATA_CURRENT_VERSION);
     for (int64_t i = 0; OB_SUCC(ret) && is_empty && i < rs_list_.count(); ++i) {
       int64_t rpc_timeout = obrpc::ObRpcProxy::MAX_RPC_TIMEOUT;
       if (INT64_MAX != THIS_WORKER.get_timeout_ts()) {
@@ -1153,7 +1182,7 @@ int ObBootstrap::init_global_stat()
   ObMySQLProxy &sql_proxy = ddl_service_.get_sql_proxy();
   ObMySQLTransaction trans;
   if (OB_FAIL(check_inner_stat())) {
-    LOG_WARN("check_inner_stat failed", K(ret));
+    LOG_WARN("check_inner_stat failed", KR(ret));
   } else {
     const int64_t baseline_schema_version = -1;
     const int64_t rootservice_epoch = 0;
@@ -1162,21 +1191,21 @@ int ObBootstrap::init_global_stat()
     ObGlobalStatProxy global_stat_proxy(trans, OB_SYS_TENANT_ID);
     ObSchemaStatusProxy *schema_status_proxy = GCTX.schema_status_proxy_;
     if (OB_FAIL(trans.start(&sql_proxy, OB_SYS_TENANT_ID))) {
-      LOG_WARN("trans start failed", K(ret));
+      LOG_WARN("trans start failed", KR(ret));
     } else if (OB_ISNULL(schema_status_proxy)) {
       ret = OB_ERR_UNEXPECTED;
-      LOG_WARN("schema_status_proxy is null", K(ret));
+      LOG_WARN("schema_status_proxy is null", KR(ret));
     } else if (OB_FAIL(global_stat_proxy.set_init_value(
                OB_CORE_SCHEMA_VERSION, baseline_schema_version,
-               rootservice_epoch, snapshot_gc_scn, snapshot_gc_timestamp))) {
-      LOG_WARN("set_init_value failed", K(ret),
-               "schema_version", OB_CORE_SCHEMA_VERSION, K(baseline_schema_version),
-               K(rootservice_epoch));
+               rootservice_epoch, snapshot_gc_scn, snapshot_gc_timestamp,
+               DATA_CURRENT_VERSION, DATA_CURRENT_VERSION))) {
+      LOG_WARN("set_init_value failed", KR(ret), "schema_version", OB_CORE_SCHEMA_VERSION,
+               K(baseline_schema_version), K(rootservice_epoch), "data_version", DATA_CURRENT_VERSION);
     }
 
     int temp_ret = OB_SUCCESS;
     if (OB_SUCCESS != (temp_ret = trans.end(OB_SUCCESS == ret))) {
-      LOG_WARN("trans end failed", "commit", OB_SUCCESS == ret, K(temp_ret));
+      LOG_WARN("trans end failed", "commit", OB_SUCCESS == ret, KR(temp_ret));
       ret = (OB_SUCCESS == ret) ? temp_ret : ret;
     }
 
@@ -1187,7 +1216,7 @@ int ObBootstrap::init_global_stat()
       if (OB_FAIL(schema_status_proxy->set_tenant_schema_status(tenant_status))) {
         LOG_WARN("fail to init create partition status", KR(ret), K(tenant_status));
       } else if (OB_FAIL(init_sequence_id())) {
-        LOG_WARN("failed to init_sequence_id", K(ret));
+        LOG_WARN("failed to init_sequence_id", KR(ret));
       } else {}
     }
   }

src/rootserver/ob_bootstrap.h

@@ -105,6 +105,7 @@ private:
   virtual int create_ls();
   virtual int wait_elect_ls(common::ObAddr &master_rs);
 
+  int notify_sys_tenant_config_();
 private:
   volatile bool stop_;
   share::ObLSLeaderElectionWaiter ls_leader_waiter_;

src/rootserver/ob_ddl_operator.cpp

@@ -5005,10 +5005,12 @@ int ObDDLOperator::fetch_expire_recycle_objects(
   return ret;
 }
 
-int ObDDLOperator::init_tenant_env(const ObTenantSchema &tenant_schema,
-                                   const ObSysVariableSchema &sys_variable,
-                                   const share::ObTenantRole &tenant_role,
-                                   ObMySQLTransaction &trans)
+int ObDDLOperator::init_tenant_env(
+    const ObTenantSchema &tenant_schema,
+    const ObSysVariableSchema &sys_variable,
+    const share::ObTenantRole &tenant_role,
+    const common::ObIArray<common::ObConfigPairs> &init_configs,
+    ObMySQLTransaction &trans)
 {
   int ret = OB_SUCCESS;
   const uint64_t tenant_id = tenant_schema.get_tenant_id();
@@ -5041,10 +5043,10 @@ int ObDDLOperator::init_tenant_env(const ObTenantSchema &tenant_schema,
   }
   if (OB_SUCC(ret) && !is_user_tenant(tenant_id)) {
     uint64_t user_tenant_id = gen_user_tenant_id(tenant_id);
-    if (OB_FAIL(init_tenant_config(tenant_id, trans))) {
+    if (OB_FAIL(init_tenant_config(tenant_id, init_configs, trans))) {
       LOG_WARN("insert tenant config failed", KR(ret), K(tenant_id));
     } else if (is_meta_tenant(tenant_id)
-               && OB_FAIL(init_tenant_config(user_tenant_id, trans))) {
+               && OB_FAIL(init_tenant_config(user_tenant_id, init_configs, trans))) {
       LOG_WARN("insert tenant config failed", KR(ret), K(user_tenant_id));
     }
   }
@@ -5548,8 +5550,91 @@ int ObDDLOperator::init_tenant_users(const ObTenantSchema &tenant_schema,
   return ret;
 }
 
-int ObDDLOperator::init_tenant_config(const uint64_t tenant_id,
-                                      ObMySQLTransaction &trans)
+int ObDDLOperator::init_tenant_config(
+    const uint64_t tenant_id,
+    const common::ObIArray<common::ObConfigPairs> &init_configs,
+    ObMySQLTransaction &trans)
+{
+  int ret = OB_SUCCESS;
+  int64_t tenant_idx = !is_user_tenant(tenant_id) ? 0 : 1;
+  if (OB_UNLIKELY(
+      init_configs.count() < tenant_idx + 1
+      || tenant_id != init_configs.at(tenant_idx).get_tenant_id())) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid init_configs", KR(ret), K(tenant_idx), K(tenant_id), K(init_configs));
+  } else if (OB_FAIL(init_tenant_config_(tenant_id, init_configs.at(tenant_idx), trans))) {
+    LOG_WARN("fail to init tenant config", KR(ret), K(tenant_id));
+  } else if (OB_FAIL(init_tenant_config_from_seed_(tenant_id, trans))) {
+    LOG_WARN("fail to init tenant config from seed", KR(ret), K(tenant_id));
+  }
+  return ret;
+}
+
+int ObDDLOperator::init_tenant_config_(
+    const uint64_t tenant_id,
+    const common::ObConfigPairs &tenant_config,
+    ObMySQLTransaction &trans)
+{
+  int ret = OB_SUCCESS;
+  omt::ObTenantConfigGuard hard_code_config(TENANT_CONF(OB_SYS_TENANT_ID));
+  int64_t config_cnt = tenant_config.get_configs().count();
+  if (OB_UNLIKELY(tenant_id != tenant_config.get_tenant_id() || config_cnt <= 0)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid tenant config", KR(ret), K(tenant_id), K(tenant_config));
+  } else if (!hard_code_config.is_valid()) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("failed to get hard code config", KR(ret), K(tenant_id));
+  } else {
+    ObDMLSqlSplicer dml;
+    ObConfigItem *item = NULL;
+    char svr_ip[OB_MAX_SERVER_ADDR_SIZE] = "ANY";
+    int64_t svr_port = 0;
+    int64_t config_version = 1; // hard code init version
+    FOREACH_X(config, tenant_config.get_configs(), OB_SUCC(ret)) {
+      const ObConfigStringKey key(config->key_.ptr());
+      if (OB_ISNULL(hard_code_config->get_container().get(key))
+          || OB_ISNULL(item = *(hard_code_config->get_container().get(key)))) {
+        ret = OB_ENTRY_NOT_EXIST;
+        LOG_WARN("config not exist", KR(ret), KPC(config));
+      } else if (OB_FAIL(dml.add_pk_column("tenant_id", tenant_id))
+                 || OB_FAIL(dml.add_pk_column("zone", ""))
+                 || OB_FAIL(dml.add_pk_column("svr_type", print_server_role(OB_SERVER)))
+                 || OB_FAIL(dml.add_pk_column(K(svr_ip)))
+                 || OB_FAIL(dml.add_pk_column(K(svr_port)))
+                 || OB_FAIL(dml.add_pk_column("name", config->key_.ptr()))
+                 || OB_FAIL(dml.add_column("data_type", "varchar"))
+                 || OB_FAIL(dml.add_column("value", config->value_.ptr()))
+                 || OB_FAIL(dml.add_column("info", ""))
+                 || OB_FAIL(dml.add_column("config_version", config_version))
+                 || OB_FAIL(dml.add_column("section", item->section()))
+                 || OB_FAIL(dml.add_column("scope", item->scope()))
+                 || OB_FAIL(dml.add_column("source", item->source()))
+                 || OB_FAIL(dml.add_column("edit_level", item->edit_level()))) {
+        LOG_WARN("fail to add column", KR(ret), K(tenant_id), KPC(config));
+      } else if (OB_FAIL(dml.finish_row())) {
+        LOG_WARN("fail to finish row", KR(ret), K(tenant_id), KPC(config));
+      }
+    } // end foreach
+    ObSqlString sql;
+    int64_t affected_rows = 0;
+    const uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id);
+    if (FAILEDx(dml.splice_batch_insert_sql(OB_TENANT_PARAMETER_TNAME, sql))) {
+      LOG_WARN("fail to generate sql", KR(ret), K(tenant_id));
+    } else if (OB_FAIL(trans.write(exec_tenant_id, sql.ptr(), affected_rows))) {
+      LOG_WARN("fail to execute sql", KR(ret), K(tenant_id), K(exec_tenant_id), K(sql));
+    } else if (config_cnt != affected_rows) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_WARN("affected_rows not match", KR(ret), K(tenant_id), K(config_cnt), K(affected_rows));
+    } else if (OB_FAIL(OTC_MGR.set_tenant_config_version(tenant_id, config_version))) {
+      LOG_WARN("failed to set tenant config version", KR(ret), K(tenant_id), K(config_version));
+    }
+  }
+  return ret;
+}
+
+int ObDDLOperator::init_tenant_config_from_seed_(
+    const uint64_t tenant_id,
+    ObMySQLTransaction &trans)
 {
   int ret = OB_SUCCESS;
   int64_t start = ObTimeUtility::current_time();
@@ -5649,7 +5734,7 @@ int ObDDLOperator::init_freeze_info(const uint64_t tenant_id,
   int64_t start = ObTimeUtility::current_time();
   ObFreezeInfoProxy freeze_info_proxy(tenant_id);
   ObSimpleFrozenStatus frozen_status;
-  frozen_status.set_initial_value(GET_MIN_CLUSTER_VERSION());
+  frozen_status.set_initial_value(DATA_CURRENT_VERSION);
   // init freeze_info in __all_freeze_info
   if (OB_FAIL(freeze_info_proxy.set_freeze_info(trans, frozen_status))) {
     LOG_WARN("fail to set freeze info", KR(ret), K(frozen_status), K(tenant_id));

src/rootserver/ob_ddl_operator.h

@@ -498,6 +498,7 @@ public:
   virtual int init_tenant_env(const share::schema::ObTenantSchema &tenant_schema,
                               const share::schema::ObSysVariableSchema &sys_variable,
                               const share::ObTenantRole &tenant_role,
+                              const common::ObIArray<common::ObConfigPairs> &init_configs,
                               common::ObMySQLTransaction &trans);
   virtual int rename_table(const share::schema::ObTableSchema &table_schema,
                            const common::ObString &new_table_name,
@@ -961,6 +962,7 @@ private:
   virtual int init_tenant_sys_stats(const uint64_t tenant_id,
                                     common::ObMySQLTransaction &trans);
   virtual int init_tenant_config(const uint64_t tenant_id,
+                                 const common::ObIArray<common::ObConfigPairs> &init_configs,
                                  common::ObMySQLTransaction &trans);
   virtual int init_freeze_info(const uint64_t tenant_id,
                                common::ObMySQLTransaction &trans);
@@ -1106,6 +1108,13 @@ private:
                                        uint64_t tenant_id,
                                        ObMySQLTransaction &trans);
   int init_tenant_spm_configure(uint64_t tenant_id, ObMySQLTransaction &trans);
+  int init_tenant_config_(
+      const uint64_t tenant_id,
+      const common::ObConfigPairs &tenant_config,
+      common::ObMySQLTransaction &trans);
+  int init_tenant_config_from_seed_(
+      const uint64_t tenant_id,
+      common::ObMySQLTransaction &trans);
 private:
   share::schema::ObMultiVersionSchemaService &schema_service_;
   common::ObMySQLProxy &sql_proxy_;

src/rootserver/ob_ddl_service.cpp

@@ -50,6 +50,7 @@
 #include "share/ob_global_stat_proxy.h"
 #include "share/ob_freeze_info_proxy.h"
 #include "share/ob_service_epoch_proxy.h"
+#include "share/ob_primary_standby_service.h" // ObPrimaryStandbyService
 #include "sql/resolver/ob_stmt_type.h"
 #include "sql/resolver/ddl/ob_ddl_resolver.h"
 #include "sql/resolver/expr/ob_raw_expr_modify_column_name.h"
@@ -1708,7 +1709,8 @@ int ObDDLService::create_tables_in_trans(const bool if_not_exist,
     ObDDLOperator ddl_operator(*schema_service_, *sql_proxy_);
     ObSchemaGetterGuard schema_guard;
     uint64_t tenant_id = table_schemas.at(0).get_tenant_id();
-    share::schema::ObTableSchema &first_table = table_schemas.at(0);
+    share::schema::ObTableSchema *first_table = &table_schemas.at(0);
+    uint64_t data_table_id = first_table->get_table_id();
     ObArray<ObObjPriv> orig_obj_privs_ora;
     const ObTableSchema *old_view_schema = NULL;
     bool is_oracle_mode = false;
@@ -1720,8 +1722,8 @@ int ObDDLService::create_tables_in_trans(const bool if_not_exist,
     } else if (OB_FAIL(trans.start(sql_proxy_, tenant_id, refreshed_schema_version))) {
       LOG_WARN("failed to start trans", KR(ret), K(tenant_id),
                K(tenant_id), K(refreshed_schema_version));
-    } else if (OB_FAIL(first_table.check_if_oracle_compat_mode(is_oracle_mode))) {
-      LOG_WARN("fail to check is oracle mode", KR(ret), K(first_table));
+    } else if (OB_FAIL(first_table->check_if_oracle_compat_mode(is_oracle_mode))) {
+      LOG_WARN("fail to check is oracle mode", KR(ret), KPC(first_table));
     } else {
     }
     if (OB_SUCC(ret)) {
@@ -1775,6 +1777,31 @@ int ObDDLService::create_tables_in_trans(const bool if_not_exist,
       }
       RS_TRACE(operator_create_table_begin);
 
+      // Fill index/lob table for system table
+      if (OB_SUCC(ret)
+          && ObSysTableChecker::is_sys_table_has_index(data_table_id)) {
+        if (OB_FAIL(ObSysTableChecker::append_sys_table_index_schemas(
+                    tenant_id, data_table_id, table_schemas))) {
+          LOG_WARN("fail to add sys table index", KR(ret), K(tenant_id), "table_id", data_table_id);
+        } else {
+          first_table = &table_schemas.at(0);
+        }
+      }
+
+      if (OB_SUCC(ret) && is_system_table(data_table_id)) {
+        HEAP_VARS_2((ObTableSchema, lob_meta_schema), (ObTableSchema, lob_piece_schema)) {
+          if (OB_FAIL(add_sys_table_lob_aux(tenant_id, data_table_id, lob_meta_schema, lob_piece_schema))) {
+            LOG_WARN("fail to get sys table lob aux schema", KR(ret), K(data_table_id));
+          } else if (OB_FAIL(table_schemas.push_back(lob_meta_schema))) {
+            LOG_WARN("fail to push back lob meta table", KR(ret), K(lob_meta_schema));
+          } else if (OB_FAIL(table_schemas.push_back(lob_piece_schema))) {
+            LOG_WARN("fail to push back lob piece table", KR(ret), K(lob_piece_schema));
+          } else {
+            first_table = &table_schemas.at(0);
+          }
+        }
+      }
+
       for (int64_t i = 0; OB_SUCC(ret) && i < table_schemas.count(); i++) {
         ObTableSchema &table_schema = table_schemas.at(i);
         if (OB_FAIL(ddl_operator.create_sequence_in_create_table(table_schema,
@@ -1803,7 +1830,6 @@ int ObDDLService::create_tables_in_trans(const bool if_not_exist,
             }
           }
         }
-
         if (OB_SUCC(ret) && (0 == i) && table_schema.is_view_table() &&
             OB_NOT_NULL(old_view_schema) && is_oracle_mode) {
           const uint64_t db_id = table_schema.get_database_id();
@@ -1826,7 +1852,7 @@ int ObDDLService::create_tables_in_trans(const bool if_not_exist,
       }
 
       // add error info for create force view
-      if (OB_SUCC(ret) && 1 == table_schemas.count() && first_table.is_user_view()) {
+      if (OB_SUCC(ret) && 1 == table_schemas.count() && first_table->is_user_view()) {
         if (OB_LIKELY(ERROR_STATUS_HAS_ERROR != error_info.get_error_status())) {
           /* do nothing */
         } else if (OB_UNLIKELY(!is_oracle_mode)) {
@@ -1837,11 +1863,11 @@ int ObDDLService::create_tables_in_trans(const bool if_not_exist,
           if (OB_FAIL(tmp_error_info.assign(error_info))) {
             LOG_WARN("failed to assign error info", K(ret));
           } else {
-            tmp_error_info.set_obj_id(first_table.get_table_id());
+            tmp_error_info.set_obj_id(first_table->get_table_id());
             tmp_error_info.set_obj_type(static_cast<uint64_t>(ObObjectType::VIEW));
-            tmp_error_info.set_database_id(first_table.get_database_id());
-            tmp_error_info.set_tenant_id(first_table.get_tenant_id());
-            tmp_error_info.set_schema_version(first_table.get_schema_version());
+            tmp_error_info.set_database_id(first_table->get_database_id());
+            tmp_error_info.set_tenant_id(first_table->get_tenant_id());
+            tmp_error_info.set_schema_version(first_table->get_schema_version());
             if (OB_FAIL(tmp_error_info.handle_error_info(trans, NULL))) {
               LOG_WARN("insert create error info failed.", K(ret));
             }
@@ -1851,42 +1877,6 @@ int ObDDLService::create_tables_in_trans(const bool if_not_exist,
       if (FAILEDx(ddl_operator.deal_with_mock_fk_parent_tables(trans, schema_guard, mock_fk_parent_table_schema_array))) {
         LOG_WARN("fail to deal_with_mock_fk_parent_tables", K(ret), K(tenant_id));
       }
-      // Create a new indexed system table needs to write a schema
-      if (OB_SUCC(ret)
-          && ObSysTableChecker::is_sys_table_has_index(first_table.get_table_id())) {
-        ObArray<ObTableSchema> schemas;
-        if (OB_FAIL(ObSysTableChecker::append_sys_table_index_schemas(
-                    tenant_id, first_table.get_table_id(), schemas))) {
-          LOG_WARN("fail to add sys table index", K(ret), K(tenant_id),
-                   "table_id", first_table.get_table_id());
-        } else if (OB_FAIL(ddl_operator.create_table(schemas.at(0),
-                                                     trans,
-                                                     NULL,
-                                                     true /*need_sync_schema_version*/))) {
-          LOG_WARN("failed to create table schema", K(ret), "schema", schemas.at(0));
-        }
-      }
-
-      if (OB_SUCC(ret) && is_system_table(first_table.get_table_id())) {
-        HEAP_VARS_2((ObTableSchema, lob_meta_schema), (ObTableSchema, lob_piece_schema)) {
-          if (OB_FAIL(add_sys_table_lob_aux(tenant_id, first_table.get_table_id(),
-                      lob_meta_schema, lob_piece_schema))) {
-            LOG_WARN("fail to get sys table lob aux schema", KR(ret), K(first_table.get_table_id()));
-          } else {
-            if (OB_FAIL(ddl_operator.create_table(lob_meta_schema,
-                                                  trans,
-                                                  NULL,
-                                                  true /*need_sync_schema_version*/))) {
-              LOG_WARN("failed to create lob aux meta table.", K(ret), "schema", lob_meta_schema);
-            } else if (OB_FAIL(ddl_operator.create_table(lob_piece_schema,
-                                                         trans,
-                                                         NULL,
-                                                         true /*need_sync_schema_version*/))) {
-              LOG_WARN("failed to create lob aux data table.", K(ret), "schema", lob_piece_schema);
-            }
-          }
-        }
-      }
 
       SCN frozen_scn;
       if (OB_FAIL(ret)) {
@@ -1950,7 +1940,7 @@ int ObDDLService::create_tables_in_trans(const bool if_not_exist,
         if (OB_FAIL(ret)) {
         } else if (schemas.count() <= 0) {
           // virtual tablet and view skip
-        } else if (OB_FAIL(new_table_tablet_allocator.prepare(first_table))) {
+        } else if (OB_FAIL(new_table_tablet_allocator.prepare(*first_table))) {
           LOG_WARN("fail to prepare ls for index schema tablets");
         } else if (OB_FAIL(new_table_tablet_allocator.get_ls_id_array(
                 ls_id_array))) {
@@ -1958,7 +1948,7 @@ int ObDDLService::create_tables_in_trans(const bool if_not_exist,
         } else if (OB_FAIL(table_creator.add_create_tablets_of_tables_arg(
                 schemas,
                 ls_id_array))) {
-          LOG_WARN("create table partitions failed", KR(ret), K(first_table),
+          LOG_WARN("create table partitions failed", KR(ret), KPC(first_table),
                K(last_schema_version));
         } else if (OB_FAIL(table_creator.execute())) {
           LOG_WARN("execute create partition failed", KR(ret));
@@ -12688,7 +12678,6 @@ int ObDDLService::truncate_table_in_trans(const obrpc::ObTruncateTableArg &arg,
       }
 
       SCN frozen_scn;
-      share::ObSimpleFrozenStatus frozen_status;
       if (OB_FAIL(ret)) {
       } else if (OB_ISNULL(GCTX.root_service_)) {
         ret = OB_ERR_UNEXPECTED;
@@ -18793,6 +18782,8 @@ int ObDDLService::add_table_schema(
   } else if (OB_FAIL(create_table_in_trans(table_schema, NULL, NULL, schema_guard))) {
     LOG_WARN("create_table_in_trans failed", KR(ret), K(table_schema));
   }
+  LOG_INFO("[UPGRADE] add inner table", KR(ret), "tenant_id", table_schema.get_tenant_id(),
+           "table_id", table_schema.get_table_id(), "table_name", table_schema.get_table_name());
   return ret;
 }
 
@@ -18827,6 +18818,8 @@ int ObDDLService::drop_inner_table(const share::schema::ObTableSchema &table_sch
                                          NULL, NULL, NULL))) {
     LOG_WARN("drop table in transaction failed", KR(ret), K(tenant_id), K(table_id));
   }
+  LOG_INFO("[UPGRADE] drop inner table", KR(ret), K(tenant_id), K(table_id),
+           "table_name", table_schema.get_table_name());
   return ret;
 }
 
@@ -18867,9 +18860,15 @@ int ObDDLService::create_sys_tenant(
       // The update of __all_core_table must be a single-partition transaction.
       // Failure to create a tenant will result in garbage data, but it will not affect
       int64_t refreshed_schema_version = 0; // won't lock
+      common::ObConfigPairs config;
+      common::ObSEArray<common::ObConfigPairs, 1> init_configs;
       if (OB_ISNULL(schema_status_proxy)) {
         ret = OB_ERR_UNEXPECTED;
         LOG_WARN("schema_status_proxy is null", K(ret));
+      } else if (OB_FAIL(ObDDLService::gen_tenant_init_config(
+                 OB_SYS_TENANT_ID, DATA_CURRENT_VERSION, config))) {
+      } else if (OB_FAIL(init_configs.push_back(config))) {
+        LOG_WARN("fail to push back config", KR(ret), K(config));
       } else if (OB_FAIL(schema_status_proxy->set_tenant_schema_status(tenant_status))) {
         LOG_WARN("init tenant create partition status failed", K(ret), K(tenant_status));
       } else if (OB_FAIL(trans.start(sql_proxy_, OB_SYS_TENANT_ID, refreshed_schema_version))) {
@@ -18881,7 +18880,8 @@ int ObDDLService::create_sys_tenant(
       } else if (OB_FAIL(ddl_operator.replace_sys_variable(
               sys_variable, tenant_schema.get_schema_version(), trans, operation_type))) {
         LOG_WARN("fail to replace sys variable", K(ret), K(sys_variable));
-      } else if (OB_FAIL(ddl_operator.init_tenant_env(tenant_schema, sys_variable, share::PRIMARY_TENANT_ROLE, trans))) {
+      } else if (OB_FAIL(ddl_operator.init_tenant_env(
+                 tenant_schema, sys_variable, share::PRIMARY_TENANT_ROLE, init_configs, trans))) {
         LOG_WARN("init tenant env failed", K(tenant_schema), K(ret));
       } else if (OB_FAIL(ddl_operator.insert_tenant_merge_info(OB_DDL_ADD_TENANT, tenant_schema, trans))) {
         LOG_WARN("fail to insert tenant merge info", KR(ret));
@@ -18976,7 +18976,8 @@ int ObDDLService::generate_tenant_schema(
     ObTenantSchema &user_tenant_schema,
     ObSysVariableSchema &user_sys_variable,
     ObTenantSchema &meta_tenant_schema,
-    ObSysVariableSchema &meta_sys_variable)
+    ObSysVariableSchema &meta_sys_variable,
+    common::ObIArray<common::ObConfigPairs> &init_configs)
 {
   int ret = OB_SUCCESS;
   uint64_t user_tenant_id = arg.tenant_schema_.get_tenant_id();
@@ -19036,6 +19037,47 @@ int ObDDLService::generate_tenant_schema(
         }
       }
     }
+    // init tenant configs
+    if (OB_SUCC(ret)) {
+      init_configs.reset();
+      common::ObConfigPairs config;
+      const uint64_t meta_tenant_id = gen_meta_tenant_id(user_tenant_id);
+      if (OB_FAIL(gen_tenant_init_config(meta_tenant_id, DATA_CURRENT_VERSION, config))) {
+        LOG_WARN("fail to gen tenant init config", KR(ret), K(meta_tenant_id));
+      } else if (OB_FAIL(init_configs.push_back(config))) {
+        LOG_WARN("fail to push back config", KR(ret), K(meta_tenant_id), K(config));
+      } else if (!is_sys_tenant(user_tenant_id)) {
+        uint64_t compatible_version = arg.is_restore_ ? arg.compatible_version_ : DATA_CURRENT_VERSION;
+        if (OB_FAIL(gen_tenant_init_config(user_tenant_id, compatible_version, config))) {
+          LOG_WARN("fail to gen tenant init config", KR(ret), K(user_tenant_id), K(compatible_version));
+        } else if (OB_FAIL(init_configs.push_back(config))) {
+          LOG_WARN("fail to push back config", KR(ret), K(user_tenant_id), K(config));
+        }
+      }
+    }
+  }
+  return ret;
+}
+
+int ObDDLService::gen_tenant_init_config(
+    const uint64_t tenant_id,
+    const uint64_t compatible_version,
+    common::ObConfigPairs &tenant_config)
+{
+  int ret = OB_SUCCESS;
+  ObString config_name("compatible");
+  ObString config_value;
+  char version[common::OB_CLUSTER_VERSION_LENGTH] = {'\0'};
+  int64_t len = ObClusterVersion::print_version_str(
+                version, common::OB_CLUSTER_VERSION_LENGTH, compatible_version);
+  tenant_config.reset();
+  (void) tenant_config.init(tenant_id);
+  if (len < 0) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid version", KR(ret), K(tenant_id), K(compatible_version));
+  } else if (FALSE_IT(config_value.assign_ptr(version, len))) {
+  } else if (OB_FAIL(tenant_config.add_config(config_name, config_value))) {
+    LOG_WARN("fail to add config", KR(ret), K(config_name), K(config_value));
   }
   return ret;
 }
@@ -19094,6 +19136,7 @@ int ObDDLService::create_tenant(
   palf::PalfBaseInfo user_palf_base_info;
   palf::PalfBaseInfo meta_palf_base_info;
   bool create_ls_with_palf = false;
+  ObSEArray<ObConfigPairs, 2> init_configs;
   HEAP_VARS_4((ObTenantSchema, user_tenant_schema),
               (ObTenantSchema, meta_tenant_schema),
               (ObSysVariableSchema, user_sys_variable),
@@ -19112,9 +19155,11 @@ int ObDDLService::create_tenant(
       tenant_role = share::PRIMARY_TENANT_ROLE;
     }
     if (OB_FAIL(ret)) {
-    } else if (OB_FAIL(generate_tenant_schema(arg, tenant_role, schema_guard,
-            user_tenant_schema, user_sys_variable,
-            meta_tenant_schema, meta_sys_variable))) {
+    } else if (OB_FAIL(generate_tenant_schema(
+               arg, tenant_role, schema_guard,
+               user_tenant_schema, user_sys_variable,
+               meta_tenant_schema, meta_sys_variable,
+               init_configs))) {
       LOG_WARN("fail to generate tenant schema", KR(ret), K(arg), K(tenant_role));
     } else if (FALSE_IT(user_tenant_id = user_tenant_schema.get_tenant_id())) {
     } else if (FALSE_IT(meta_tenant_id = meta_tenant_schema.get_tenant_id())) {
@@ -19122,8 +19167,8 @@ int ObDDLService::create_tenant(
             user_tenant_schema.get_tenant_id(), tenant_role))) {
       LOG_WARN("fail to init schema status", KR(ret), K(user_tenant_id));
     } else if (OB_FAIL(create_tenant_schema(
-            arg, schema_guard,
-            user_tenant_schema, meta_tenant_schema))) {
+               arg, schema_guard, user_tenant_schema,
+               meta_tenant_schema, init_configs))) {
       LOG_WARN("fail to create tenant schema", KR(ret), K(arg));
     } else {
       DEBUG_SYNC(BEFORE_CREATE_META_TENANT);
@@ -19131,16 +19176,20 @@ int ObDDLService::create_tenant(
       ObArray<ObResourcePoolName> pools;
       if (OB_FAIL(get_pools(arg.pool_list_, pools))) {
         LOG_WARN("get_pools failed", KR(ret), K(arg));
-      } else if (OB_FAIL(create_normal_tenant(meta_tenant_id, pools, meta_tenant_schema, tenant_role,
-              meta_sys_variable, false/*create_ls_with_palf*/, meta_palf_base_info))) {
-        LOG_WARN("fail to create meta tenant", KR(ret), K(meta_tenant_id), K(pools), K(meta_sys_variable),
-            K(tenant_role), K(meta_palf_base_info));
+      } else if (OB_FAIL(create_normal_tenant(
+                 meta_tenant_id, pools, meta_tenant_schema, tenant_role,
+                 meta_sys_variable, false/*create_ls_with_palf*/,
+                 meta_palf_base_info, init_configs))) {
+        LOG_WARN("fail to create meta tenant", KR(ret), K(meta_tenant_id), K(pools),
+                 K(meta_sys_variable), K(tenant_role), K(meta_palf_base_info), K(init_configs));
       } else {
         DEBUG_SYNC(BEFORE_CREATE_USER_TENANT);
-        if (OB_FAIL(create_normal_tenant(user_tenant_id, pools, user_tenant_schema, tenant_role,
-                user_sys_variable, create_ls_with_palf, user_palf_base_info))) {
-          LOG_WARN("fail to create user tenant", KR(ret), K(user_tenant_id), K(pools), K(user_sys_variable),
-              K(tenant_role), K(user_palf_base_info));
+        if (OB_FAIL(create_normal_tenant(
+            user_tenant_id, pools, user_tenant_schema, tenant_role,
+            user_sys_variable, create_ls_with_palf,
+            user_palf_base_info, init_configs))) {
+          LOG_WARN("fail to create user tenant", KR(ret), K(user_tenant_id), K(pools),
+                   K(user_sys_variable), K(tenant_role), K(user_palf_base_info));
         }
       }
       // drop tenant if create tenant failed.
@@ -19198,7 +19247,8 @@ int ObDDLService::create_tenant_schema(
     const ObCreateTenantArg &arg,
     share::schema::ObSchemaGetterGuard &schema_guard,
     ObTenantSchema &user_tenant_schema,
-    ObTenantSchema &meta_tenant_schema)
+    ObTenantSchema &meta_tenant_schema,
+    const common::ObIArray<common::ObConfigPairs> &init_configs)
 {
   const int64_t start_time = ObTimeUtility::fast_current_time();
   LOG_INFO("[CREATE_TENANT] STEP 1. start create tenant schema", K(arg));
@@ -19258,6 +19308,20 @@ int ObDDLService::create_tenant_schema(
                "cost", ObTimeUtility::fast_current_time() - tmp_start_time);
     }
 
+    // 3. persist initial tenant config
+    if (OB_SUCC(ret)) {
+      LOG_INFO("[CREATE_TENANT] STEP 1.3. start persist tenant config", K(user_tenant_id));
+      const int64_t tmp_start_time = ObTimeUtility::fast_current_time();
+      ObArray<ObAddr> addrs;
+      if (OB_FAIL(unit_mgr_->get_servers_by_pools(pools, addrs))) {
+        LOG_WARN("fail to get tenant's servers", KR(ret), K(user_tenant_id));
+      } else if (OB_FAIL(notify_init_tenant_config(*rpc_proxy_, init_configs, addrs))) {
+        LOG_WARN("fail to notify broadcast tenant config", KR(ret), K(addrs), K(init_configs));
+      }
+      LOG_INFO("[CREATE_TENANT] STEP 1.3. finish persist tenant config",
+               KR(ret), K(user_tenant_id), "cost", ObTimeUtility::fast_current_time() - tmp_start_time);
+    }
+
     if (trans.is_started()) {
       int temp_ret = OB_SUCCESS;
       bool commit = OB_SUCC(ret);
@@ -19272,13 +19336,13 @@ int ObDDLService::create_tenant_schema(
     // the transaction is considered to have failed, and the unit_mgr memory state is not modified at this time,
     // and the transaction 1 is subsequently rolled back through drop tenant.
     if (OB_SUCC(ret)) {
-      LOG_INFO("[CREATE_TENANT] STEP 1.3. start change pool owners", K(user_tenant_id));
+      LOG_INFO("[CREATE_TENANT] STEP 1.4. start change pool owners", K(user_tenant_id));
       const int64_t tmp_start_time = ObTimeUtility::fast_current_time();
       const bool grant = true;
       if (OB_FAIL(unit_mgr_->commit_change_pool_owner(new_ug_id_array, grant, pools, user_tenant_id))) {
         LOG_WARN("commit change pool owner failed", K(grant), K(pools), K(user_tenant_id), KR(ret));
       }
-      LOG_INFO("[CREATE_TENANT] STEP 1.3. finish change pool owners", KR(ret), K(user_tenant_id),
+      LOG_INFO("[CREATE_TENANT] STEP 1.4. finish change pool owners", KR(ret), K(user_tenant_id),
                "cost", ObTimeUtility::fast_current_time() - tmp_start_time);
     }
     if (OB_SUCC(ret)) {
@@ -19296,6 +19360,97 @@ int ObDDLService::create_tenant_schema(
   return ret;
 }
 
+int ObDDLService::notify_init_tenant_config(
+    obrpc::ObSrvRpcProxy &rpc_proxy,
+    const common::ObIArray<common::ObConfigPairs> &init_configs,
+    const common::ObIArray<common::ObAddr> &addrs)
+{
+  int ret = OB_SUCCESS;
+  ObTimeoutCtx ctx;
+  const int64_t DEFAULT_TIMEOUT = 10 * 1000 * 1000L; // 10s
+  if (OB_UNLIKELY(
+      init_configs.count() <= 0
+      || addrs.count() <= 0)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("init configs count is invalid", KR(ret), K(init_configs), K(addrs));
+  } else if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, DEFAULT_TIMEOUT))) {
+    LOG_WARN("fail to set default timeout", KR(ret));
+  } else {
+    ObArenaAllocator allocator("InitTenantConf");
+    int64_t server_cnt = addrs.count();
+    // 1. construct arg
+    obrpc::ObInitTenantConfigArg arg;
+    for (int64_t i = 0; OB_SUCC(ret) && i < init_configs.count(); i++) {
+     const common::ObConfigPairs &pairs = init_configs.at(i);
+     obrpc::ObTenantConfigArg config;
+     char *buf = NULL;
+     int64_t length = pairs.get_config_str_length();
+     if (OB_ISNULL(buf = static_cast<char *>(allocator.alloc(length)))) {
+       ret = OB_ALLOCATE_MEMORY_FAILED;
+       LOG_WARN("alloc memory failed", KR(ret), K(length));
+     } else {
+       MEMSET(buf, '\0', length);
+       if (OB_FAIL(pairs.get_config_str(buf, length))) {
+         LOG_WARN("fail to get config str", KR(ret), K(length), K(pairs));
+       } else {
+         config.tenant_id_ = pairs.get_tenant_id();
+         config.config_str_.assign_ptr(buf, strlen(buf));
+         if (OB_FAIL(arg.add_tenant_config(config))) {
+           LOG_WARN("fail to add config", KR(ret), K(config));
+         }
+       }
+     }
+    } // end for
+    // 2. send rpc
+    rootserver::ObInitTenantConfigProxy proxy(
+        rpc_proxy, &obrpc::ObSrvRpcProxy::init_tenant_config);
+    bool call_rs = false;
+    ObAddr rs_addr = GCONF.self_addr_;
+    int64_t timeout = ctx.get_timeout();
+    for (int64_t i = 0; OB_SUCC(ret) && i < addrs.count(); i++) {
+      const ObAddr &addr = addrs.at(i);
+      if (OB_FAIL(proxy.call(addr, timeout, arg))) {
+        LOG_WARN("send rpc failed", KR(ret), K(addr), K(timeout), K(arg));
+      } else if (rs_addr == addr) {
+        call_rs = true;
+      }
+    } // end for
+    if (OB_FAIL(ret) || call_rs) {
+    } else if (OB_FAIL(proxy.call(rs_addr, timeout, arg))) {
+      LOG_WARN("fail to call rs", KR(ret), K(rs_addr), K(timeout), K(arg));
+    } else {
+      server_cnt++;
+    }
+    // 3. check result
+    ObArray<int> return_ret_array;
+    int tmp_ret = OB_SUCCESS;
+    if (OB_SUCCESS != (tmp_ret = proxy.wait_all(return_ret_array))) { // ignore ret
+      LOG_WARN("wait batch result failed", KR(tmp_ret), KR(ret));
+      ret = OB_SUCC(ret) ? tmp_ret : ret;
+    } else if (return_ret_array.count() != server_cnt) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_WARN("result cnt not match", KR(ret), K(server_cnt), "res_cnt", return_ret_array.count());
+    }
+    for (int64_t i = 0; OB_SUCC(ret) && i < return_ret_array.count(); i++) {
+      int return_ret = return_ret_array.at(i);
+      const ObAddr &addr = proxy.get_dests().at(i);
+      const ObInitTenantConfigRes *result = proxy.get_results().at(i);
+      if (OB_SUCCESS != return_ret) {
+        ret = return_ret;
+        LOG_WARN("rpc return error", KR(ret), K(addr), K(timeout));
+      } else if (OB_ISNULL(result)) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_WARN("get empty result", KR(ret), K(addr), K(timeout));
+      } else if (OB_SUCCESS != result->get_ret()) {
+        ret = result->get_ret();
+        LOG_WARN("persist tenant config failed", KR(ret), K(addr), K(timeout));
+      }
+    } // end for
+  }
+  return ret;
+}
+
+
 // 1. create tenant's sys ls
 // 2. broadcast sys table schemas
 // 3. create tenant's sys tablets
@@ -19307,7 +19462,8 @@ int ObDDLService::create_normal_tenant(
     const share::ObTenantRole &tenant_role,
     ObSysVariableSchema &sys_variable,
     const bool create_ls_with_palf,
-    const palf::PalfBaseInfo &palf_base_info)
+    const palf::PalfBaseInfo &palf_base_info,
+    const common::ObIArray<common::ObConfigPairs> &init_configs)
 {
   const int64_t start_time = ObTimeUtility::fast_current_time();
   LOG_INFO("[CREATE_TENANT] STEP 2. start create tenant", K(tenant_id), K(tenant_schema));
@@ -19330,9 +19486,10 @@ int ObDDLService::create_normal_tenant(
     LOG_WARN("fail to broadcast sys table schemas", KR(ret), K(tenant_id));
   } else if (OB_FAIL(create_tenant_sys_tablets(tenant_id, tables))) {
     LOG_WARN("fail to create tenant partitions", KR(ret), K(tenant_id));
-  } else if (OB_FAIL(init_tenant_schema(tenant_id, tenant_schema, tenant_role, tables, sys_variable))) {
+  } else if (OB_FAIL(init_tenant_schema(tenant_id, tenant_schema,
+             tenant_role, tables, sys_variable, init_configs))) {
     LOG_WARN("fail to init tenant schema", KR(ret), K(tenant_role),
-             K(tenant_id), K(tenant_schema), K(sys_variable));
+             K(tenant_id), K(tenant_schema), K(sys_variable), K(init_configs));
   }
   LOG_INFO("[CREATE_TENANT] STEP 2. finish create tenant", KR(ret), K(tenant_id),
            "cost", ObTimeUtility::fast_current_time() - start_time);
@@ -19669,7 +19826,8 @@ int ObDDLService::init_tenant_schema(
     const ObTenantSchema &tenant_schema,
     const share::ObTenantRole &tenant_role,
     common::ObIArray<ObTableSchema> &tables,
-    ObSysVariableSchema &sys_variable)
+    ObSysVariableSchema &sys_variable,
+    const common::ObIArray<common::ObConfigPairs> &init_configs)
 {
   const int64_t start_time = ObTimeUtility::fast_current_time();
   LOG_INFO("[CREATE_TENANT] STEP 2.4. start init tenant schemas", K(tenant_id));
@@ -19684,16 +19842,53 @@ int ObDDLService::init_tenant_schema(
     LOG_WARN("ptr is null", KR(ret), KP_(sql_proxy), KP_(schema_service), KP(GCTX.lst_operator_));
   } else {
     ObSchemaService *schema_service_impl = schema_service_->get_schema_service();
-    ObGlobalStatProxy global_stat_proxy(*sql_proxy_, tenant_id);
     // 1. init tenant global stat
     if (OB_SUCC(ret)) {
       const int64_t core_schema_version = OB_CORE_SCHEMA_VERSION + 1;
       const int64_t baseline_schema_version = OB_INVALID_VERSION;
       const SCN snapshot_gc_scn  = SCN::min_scn();
-      if (OB_FAIL(global_stat_proxy.set_tenant_init_global_stat(
-                 core_schema_version, baseline_schema_version, snapshot_gc_scn))) {
-        LOG_WARN("fail to set tenant init global stat",
-                 K(core_schema_version), K(baseline_schema_version));
+      // find compatible version
+      uint64_t data_version = 0;
+      for (int64_t i = 0; OB_SUCC(ret) && i < init_configs.count(); i++) {
+        const ObConfigPairs &config = init_configs.at(i);
+        if (tenant_id == config.get_tenant_id()) {
+          for (int64_t j = 0; data_version == 0 && OB_SUCC(ret) && j < config.get_configs().count(); j++) {
+            const ObConfigPairs::ObConfigPair &pair = config.get_configs().at(j);
+            if (0 != pair.key_.case_compare("compatible")) {
+            } else if (OB_FAIL(ObClusterVersion::get_version(pair.value_.ptr(), data_version))) {
+              LOG_WARN("fail to get compatible version", KR(ret), K(tenant_id), K(pair));
+            }
+          }
+        }
+      }
+
+      common::ObMySQLTransaction trans;
+      if (OB_FAIL(ret)) {
+      } else if (OB_FAIL(trans.start(sql_proxy_, tenant_id))) {
+        LOG_WARN("failed to start trans", KR(ret), K(tenant_id));
+      } else {
+        ObGlobalStatProxy global_stat_proxy(trans, tenant_id);
+        if (OB_FAIL(ret)) {
+        } else if (0 == data_version) {
+          ret = OB_ENTRY_NOT_EXIST;
+          LOG_WARN("compatible version not defined", KR(ret), K(tenant_id), K(init_configs));
+        } else if (OB_FAIL(global_stat_proxy.set_tenant_init_global_stat(
+                  core_schema_version, baseline_schema_version,
+                  snapshot_gc_scn, data_version, data_version))) {
+          LOG_WARN("fail to set tenant init global stat", KR(ret), K(tenant_id),
+                  K(core_schema_version), K(baseline_schema_version),
+                  K(snapshot_gc_scn), K(data_version));
+        } else if (is_user_tenant(tenant_id) && OB_FAIL(OB_PRIMARY_STANDBY_SERVICE.write_upgrade_barrier_log(
+                                                        trans, tenant_id, data_version))) {
+          LOG_WARN("fail to write_upgrade_barrier_log", KR(ret), K(tenant_id), K(data_version));
+        }
+      }
+      if (trans.is_started()) {
+        int tmp_ret = OB_SUCCESS;
+        if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) {
+          LOG_WARN("failed to commit trans", KR(ret), KR(tmp_ret));
+          ret = OB_SUCC(ret) ? tmp_ret : ret;
+        }
       }
     }
 
@@ -19716,13 +19911,14 @@ int ObDDLService::init_tenant_schema(
       } else if (OB_FAIL(ddl_operator.replace_sys_variable(
                  sys_variable, new_schema_version, trans, OB_DDL_ALTER_SYS_VAR))) {
         LOG_WARN("fail to replace sys variable", KR(ret), K(sys_variable));
-      } else if (OB_FAIL(ddl_operator.init_tenant_env(tenant_schema, sys_variable, tenant_role, trans))) {
+      } else if (OB_FAIL(ddl_operator.init_tenant_env(
+                 tenant_schema, sys_variable, tenant_role, init_configs, trans))) {
         LOG_WARN("init tenant env failed", KR(ret), K(tenant_role), K(tenant_schema));
       } else if (OB_FAIL(ddl_operator.insert_tenant_merge_info(OB_DDL_ADD_TENANT_START, tenant_schema, trans))) {
         LOG_WARN("fail to insert tenant merge info", KR(ret), K(tenant_schema));
-      } else if (is_meta_tenant(tenant_id) && OB_FAIL(ObServiceEpochProxy::init_service_epoch(trans, tenant_id, 
-          0/*freeze_service_epoch*/))) {
-        LOG_WARN("fail to init service epoch", KR(ret));
+      } else if (is_meta_tenant(tenant_id) && OB_FAIL(ObServiceEpochProxy::init_service_epoch(
+                 trans, tenant_id, 0/*freeze_service_epoch*/))) {
+        LOG_WARN("fail to init service epoch", KR(ret), K(tenant_id));
       }
 
       if (trans.is_started()) {
@@ -19751,6 +19947,7 @@ int ObDDLService::init_tenant_schema(
 
     // 3. set baseline schema version
     if (OB_SUCC(ret)) {
+      ObGlobalStatProxy global_stat_proxy(*sql_proxy_, tenant_id);
       ObRefreshSchemaStatus schema_status;
       schema_status.tenant_id_ = tenant_id;
       int64_t baseline_schema_version = OB_INVALID_VERSION;

src/rootserver/ob_ddl_service.h

@@ -34,6 +34,7 @@
 #include "storage/tablet/ob_tablet_binding_helper.h"
 #include "storage/ddl/ob_ddl_clog.h"
 #include "share/ob_freeze_info_proxy.h"
+#include "share/config/ob_config.h" // ObConfigPairs
 
 namespace oceanbase
 {
@@ -1680,6 +1681,14 @@ public:
       share::schema::ObTenantSchema &tenant_schema,
       share::schema::ObSchemaGetterGuard &schema_guard);
 
+  static int gen_tenant_init_config(
+             const uint64_t tenant_id,
+             const uint64_t compatible_version,
+             common::ObConfigPairs &tenant_config);
+  static int notify_init_tenant_config(
+             obrpc::ObSrvRpcProxy &rpc_proxy,
+             const common::ObIArray<common::ObConfigPairs> &init_configs,
+             const common::ObIArray<common::ObAddr> &addrs);
 private:
   int handle_security_audit_for_stmt(const obrpc::ObSecurityAuditArg &arg,
                                      share::schema::ObSAuditSchema &audit_schema);
@@ -1715,7 +1724,8 @@ private:
       share::schema::ObTenantSchema &user_tenant_schema,
       share::schema::ObSysVariableSchema &user_sys_variable,
       share::schema::ObTenantSchema &meta_tenant_schema,
-      share::schema::ObSysVariableSchema &meta_sys_variable);
+      share::schema::ObSysVariableSchema &meta_sys_variable,
+      common::ObIArray<common::ObConfigPairs> &init_configs);
   int init_schema_status(
       const uint64_t tenant_id,
       const share::ObTenantRole &tenant_role);
@@ -1731,7 +1741,8 @@ private:
       const obrpc::ObCreateTenantArg &arg,
       share::schema::ObSchemaGetterGuard &schema_guard,
       share::schema::ObTenantSchema &user_tenant_schema,
-      share::schema::ObTenantSchema &meta_tenant_schema);
+      share::schema::ObTenantSchema &meta_tenant_schema,
+      const common::ObIArray<common::ObConfigPairs> &init_configs);
   int create_normal_tenant(
       const uint64_t tenant_id,
       const ObIArray<share::ObResourcePoolName> &pool_list,
@@ -1739,7 +1750,8 @@ private:
       const share::ObTenantRole &tenant_role,
       share::schema::ObSysVariableSchema &sys_variable,
       const bool create_ls_with_palf,
-      const palf::PalfBaseInfo &palf_base_info);
+      const palf::PalfBaseInfo &palf_base_info,
+      const common::ObIArray<common::ObConfigPairs> &init_configs);
   int set_sys_ls_status(const uint64_t tenant_id);
   int create_tenant_sys_ls(
       const share::schema::ObTenantSchema &tenant_schema,
@@ -1757,7 +1769,8 @@ private:
       const share::schema::ObTenantSchema &tenant_schema,
       const share::ObTenantRole &tenant_role,
       common::ObIArray<share::schema::ObTableSchema> &tables,
-      share::schema::ObSysVariableSchema &sys_variable);
+      share::schema::ObSysVariableSchema &sys_variable,
+      const common::ObIArray<common::ObConfigPairs> &init_configs);
   int insert_restore_tenant_job(
       const uint64_t tenant_id,
       const ObString &tenant_name,

src/rootserver/ob_recovery_ls_service.cpp

@@ -34,6 +34,10 @@
 #include "share/schema/ob_multi_version_schema_service.h"  //ObMultiSchemaService
 #include "share/restore/ob_physical_restore_info.h" //restore_status
 #include "share/restore/ob_physical_restore_table_operator.h"//ObPhysicalRestoreTableOperator
+#include "share/ob_primary_standby_service.h" // ObPrimaryStandbyService
+#include "share/ob_standby_upgrade.h"  // ObStandbyUpgrade
+#include "share/ob_upgrade_utils.h"  // ObUpgradeChecker
+#include "share/ob_global_stat_proxy.h" // ObGlobalStatProxy
 #include "storage/tx/ob_tx_log.h"                          //ObTxLogHeader
 #include "storage/tx_storage/ob_ls_service.h"              //ObLSService
 #include "storage/tx_storage/ob_ls_handle.h"  //ObLSHandle
@@ -132,6 +136,10 @@ void ObRecoveryLSService::do_work()
           start_scn.reset();
           LOG_WARN("failed to do wowrk", KR(ret), K(start_scn));
         }
+
+        if (REACH_TIME_INTERVAL(10 * 1000 * 1000)) { // every 10 second
+          (void)try_tenant_upgrade_end_();
+        }
       }
       LOG_INFO("[LS_RECOVERY] finish one round", KR(ret), KR(tmp_ret), K(start_scn), K(thread_idx));
       idle(idle_time_us);
@@ -276,7 +284,11 @@ int ObRecoveryLSService::process_ls_tx_log_(ObTxLogBlock &tx_log_block, const SC
             commit_log.get_multi_source_data();
         for (int64_t i = 0; OB_SUCC(ret) && i < source_data.count(); ++i) {
           const ObTxBufferNode &node = source_data.at(i);
-          if (ObTxDataSourceType::LS_TABLE != node.get_data_source_type()) {
+          if (ObTxDataSourceType::STANDBY_UPGRADE == node.get_data_source_type()) {
+            if (OB_FAIL(process_upgrade_log_(node))) {
+              LOG_WARN("failed to process_upgrade_log_", KR(ret), K(node));
+            }
+          } else if (ObTxDataSourceType::LS_TABLE != node.get_data_source_type()) {
             // nothing
           } else if (has_operation) {
             ret = OB_ERR_UNEXPECTED;
@@ -311,6 +323,145 @@ int ObRecoveryLSService::process_ls_tx_log_(ObTxLogBlock &tx_log_block, const SC
   return ret;
 }
 
+int ObRecoveryLSService::process_upgrade_log_(const ObTxBufferNode &node)
+{
+  int ret = OB_SUCCESS;
+  uint64_t standby_data_version = 0;
+
+  if (!node.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("node is invalid", KR(ret), K(node));
+  } else {
+    ObStandbyUpgrade primary_data_version;
+    int64_t pos = 0;
+    if (OB_FAIL(primary_data_version.deserialize(node.get_data_buf().ptr(), node.get_data_buf().length(), pos))) {
+      LOG_WARN("failed to deserialize", KR(ret), K(node), KPHEX(node.get_data_buf().ptr(), node.get_data_buf().length()));
+    } else if (OB_UNLIKELY(pos > node.get_data_buf().length())) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_WARN("failed to get primary_data_version", KR(ret), K(pos), K(node.get_data_buf().length()));
+    } else {
+      LOG_INFO("get primary_data_version", K(primary_data_version));
+      if (!primary_data_version.is_valid()) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_WARN("primary_data_version not valid", KR(ret), K(primary_data_version));
+      } else if (OB_FAIL(ObUpgradeChecker::get_data_version_by_cluster_version(
+                                           GET_MIN_CLUSTER_VERSION(),
+                                           standby_data_version))) {
+        LOG_WARN("failed to get_data_version_by_cluster_version", KR(ret), K(GET_MIN_CLUSTER_VERSION()));
+      } else if (primary_data_version.get_data_version() > standby_data_version) {
+        ret = OB_EAGAIN;
+        if (REACH_TIME_INTERVAL(30 * 60 * 1000 * 1000)) { // 30min
+          LOG_ERROR("standby version is not new enough to recover primary clog", KR(ret),
+                   K(primary_data_version), K(standby_data_version));
+        }
+      }
+    }
+  }
+  return ret;
+}
+
+int ObRecoveryLSService::get_min_data_version_(uint64_t &compatible)
+{
+  int ret = OB_SUCCESS;
+  compatible = 0;
+  if (OB_ISNULL(proxy_)) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("sql proxy is null", KR(ret));
+  } else {
+    SMART_VAR(ObMySQLProxy::MySQLResult, result) {
+      uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id_);
+      ObSqlString sql;
+      if (OB_FAIL(sql.assign_fmt("select value from %s where tenant_id = '%lu' and name = 'compatible' ",
+                                 OB_TENANT_PARAMETER_TNAME, tenant_id_))) {
+        LOG_WARN("fail to generate sql", KR(ret), K_(tenant_id));
+      } else if (OB_FAIL(proxy_->read(result, exec_tenant_id, sql.ptr()))) {
+        LOG_WARN("read config from __tenant_parameter failed",
+                KR(ret), K_(tenant_id), K(exec_tenant_id), K(sql));
+      } else if (OB_ISNULL(result.get_result())) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_WARN("config result is null", KR(ret), K_(tenant_id), K(exec_tenant_id), K(sql));
+      } else if (OB_FAIL(result.get_result()->next())) {
+        LOG_WARN("get result next failed", KR(ret), K_(tenant_id), K(exec_tenant_id), K(sql));
+      } else if (OB_ISNULL(result.get_result())) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_WARN("result is null", KR(ret), K_(tenant_id), K(exec_tenant_id), K(sql));
+      } else {
+        ObString compatible_str;
+        EXTRACT_VARCHAR_FIELD_MYSQL(*result.get_result(), "compatible", compatible_str);
+
+        if (OB_FAIL(ret)) {
+          LOG_WARN("failed to get result", KR(ret), K_(tenant_id), K(exec_tenant_id), K(sql));
+        } else if (OB_FAIL(ObClusterVersion::get_version(compatible_str, compatible))) {
+          LOG_WARN("parse version failed", KR(ret), K(compatible_str));
+        }
+      }
+    }
+  }
+  return ret;
+}
+
+void ObRecoveryLSService::try_tenant_upgrade_end_()
+{
+  int ret = OB_SUCCESS;
+  uint64_t min_data_version = 0;
+  obrpc::ObCommonRpcProxy *rs_rpc_proxy_ = GCTX.rs_rpc_proxy_;
+  if (OB_ISNULL(proxy_) || !is_user_tenant(tenant_id_) || OB_ISNULL(rs_rpc_proxy_)) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("invalid argument", KR(ret), KP(proxy_), K_(tenant_id), KP(rs_rpc_proxy_));
+  } else if (OB_FAIL(get_min_data_version_(min_data_version))) {
+    LOG_WARN("fail to get min data version", KR(ret), K_(tenant_id));
+  } else if (min_data_version == DATA_CURRENT_VERSION) {
+    // already upgrade end
+  } else {
+    ObGlobalStatProxy proxy(*proxy_, tenant_id_);
+    uint64_t target_data_version = 0;
+    uint64_t current_data_version = 0;
+    if (OB_FAIL(proxy.get_target_data_version(false /* for_update */, target_data_version))) {
+      LOG_WARN("fail to get target data version", KR(ret), K_(tenant_id));
+    } else if (OB_FAIL(proxy.get_current_data_version(current_data_version))) {
+      LOG_WARN("fail to get current data version", KR(ret), K_(tenant_id));
+    } else if (!(target_data_version == current_data_version
+              && target_data_version != min_data_version
+              && min_data_version <= DATA_CURRENT_VERSION)) {
+      ret = EAGAIN;
+      LOG_WARN("data_version not match, run upgrade end later",
+               KR(ret), K_(tenant_id), K(target_data_version), K(current_data_version),
+               K(DATA_CURRENT_VERSION));
+    } else {
+      HEAP_VAR(obrpc::ObAdminSetConfigItem, item) {
+      ObSchemaGetterGuard guard;
+      const ObSimpleTenantSchema *tenant = NULL;
+      obrpc::ObAdminSetConfigArg arg;
+      item.exec_tenant_id_ = OB_SYS_TENANT_ID;
+      const int64_t timeout = GCONF.internal_sql_execute_timeout;
+      int64_t pos = ObClusterVersion::print_version_str(
+                    item.value_.ptr(), item.value_.capacity(),
+                    current_data_version);
+      if (pos <= 0) {
+        ret = OB_INVALID_ARGUMENT;
+        LOG_WARN("current_data_version is invalid",
+                  KR(ret), K_(tenant_id), K(current_data_version));
+      } else if (OB_FAIL(GSCHEMASERVICE.get_tenant_schema_guard(OB_SYS_TENANT_ID, guard))) {
+        LOG_WARN("fail to get schema guard", KR(ret));
+      } else if (OB_FAIL(guard.get_tenant_info(tenant_id_, tenant))) {
+        LOG_WARN("fail to get tenant info", KR(ret), K_(tenant_id));
+      } else if (OB_ISNULL(tenant)) {
+        ret = OB_TENANT_NOT_EXIST;
+        LOG_WARN("tenant not exist", KR(ret), K_(tenant_id));
+      } else if (OB_FAIL(item.tenant_name_.assign(tenant->get_tenant_name()))) {
+        LOG_WARN("fail to assign tenant name", KR(ret), K_(tenant_id));
+      } else if (OB_FAIL(item.name_.assign("compatible"))) {
+        LOG_WARN("fail to assign config name", KR(ret), K_(tenant_id));
+      } else if (OB_FAIL(arg.items_.push_back(item))) {
+        LOG_WARN("fail to push back item", KR(ret), K(item));
+      } else if (OB_FAIL(rs_rpc_proxy_->timeout(timeout).admin_set_config(arg))) {
+        LOG_WARN("fail to set config", KR(ret), K(arg), K(timeout));
+      }
+      } // end HEAP_VAR
+    }
+  }
+}
+
 int ObRecoveryLSService::process_gc_log_(logservice::ObGCLSLog &gc_log, const SCN &sync_scn)
 {
   int ret = OB_SUCCESS;

src/rootserver/ob_recovery_ls_service.h

@@ -18,6 +18,7 @@
 #include "logservice/palf/palf_iterator.h"          //PalfBufferIterator
 #include "ob_primary_ls_service.h" //ObTenantThreadHelper
 #include "lib/lock/ob_spin_lock.h" //ObSpinLock
+#include "storage/tx/ob_multi_data_source.h" //ObTxBufferNode
 
 namespace oceanbase
 {
@@ -75,6 +76,7 @@ private:
  int seek_log_iterator_(const share::SCN &syn_scn,
                         palf::PalfBufferIterator &iterator);
  int process_ls_log_(const share::SCN &start_scn,palf::PalfBufferIterator &iterator);
+ int process_upgrade_log_(const transaction::ObTxBufferNode &node);
  int process_gc_log_(logservice::ObGCLSLog &gc_log,
                      const share::SCN &syn_scn);
  int process_ls_tx_log_(transaction::ObTxLogBlock &tx_log,
@@ -94,6 +96,9 @@ private:
  int update_sys_ls_restore_finish_();
  //readable scn need report
  int report_sys_ls_recovery_stat_(const share::SCN &sync_scn);
+ void try_tenant_upgrade_end_();
+ int get_min_data_version_(uint64_t &compatible);
+
 private:
   bool inited_;
   uint64_t tenant_id_;

src/rootserver/ob_root_inspection.cpp

@@ -36,6 +36,7 @@
 #include "share/rc/ob_context.h"
 #include "share/schema/ob_schema_mgr.h"
 #include "share/ob_schema_status_proxy.h"//ObSchemaStatusProxy
+#include "share/ob_global_stat_proxy.h"//ObGlobalStatProxy
 
 namespace oceanbase
 {
@@ -652,9 +653,10 @@ ObAsyncTask *ObPurgeRecyclebinTask::deep_copy(char *buf, const int64_t buf_size)
 ObRootInspection::ObRootInspection()
   : inited_(false), stopped_(false), zone_passed_(false),
     sys_param_passed_(false), sys_stat_passed_(false),
-    sys_table_schema_passed_(false), all_checked_(false),
-    all_passed_(false), can_retry_(false), sql_proxy_(NULL),
-    rpc_proxy_(NULL), schema_service_(NULL), zone_mgr_(NULL)
+    sys_table_schema_passed_(false), data_version_passed_(false),
+    all_checked_(false), all_passed_(false), can_retry_(false),
+    sql_proxy_(NULL), rpc_proxy_(NULL), schema_service_(NULL),
+    zone_mgr_(NULL)
 {
 }
 
@@ -680,6 +682,7 @@ int ObRootInspection::init(ObMultiVersionSchemaService &schema_service,
     sys_param_passed_ = false;
     sys_stat_passed_ = false;
     sys_table_schema_passed_ = false;
+    data_version_passed_ = false;
     all_checked_ = false;
     all_passed_ = false;
     can_retry_ = false;
@@ -716,7 +719,7 @@ int ObRootInspection::check_all()
 
     // check sys stat
     tmp = OB_SUCCESS;
-    if (OB_SUCCESS != (tmp = check_sys_stat())) {
+    if (OB_SUCCESS != (tmp = check_sys_stat_())) {
       LOG_WARN("check_sys_stat failed", K(tmp));
       ret = (OB_SUCCESS == ret) ? tmp : ret;
     }
@@ -724,7 +727,7 @@ int ObRootInspection::check_all()
 
     // check sys param
     tmp = OB_SUCCESS;
-    if (OB_SUCCESS != (tmp = check_sys_param())) {
+    if (OB_SUCCESS != (tmp = check_sys_param_())) {
       LOG_WARN("check_sys_param failed", K(tmp));
       ret = (OB_SUCCESS == ret) ? tmp : ret;
     }
@@ -738,6 +741,14 @@ int ObRootInspection::check_all()
     }
     sys_table_schema_passed_ = (OB_SUCCESS == tmp);
 
+    // check tenant's data version
+    tmp = OB_SUCCESS;
+    if (OB_SUCCESS != (tmp = check_data_version_())) {
+      LOG_WARN("check_data_version failed", K(tmp));
+      ret = (OB_SUCCESS == ret) ? tmp : ret;
+    }
+    data_version_passed_ = (OB_SUCCESS == tmp);
+
     // upgrade job may still running, in order to avoid the upgrade process error stuck,
     // ignore the 4674 error
     for (int64_t i = 0; i < UPGRADE_JOB_TYPE_COUNT; i++) {
@@ -762,6 +773,32 @@ int ObRootInspection::check_all()
   return ret;
 }
 
+int ObRootInspection::check_tenant(const uint64_t tenant_id)
+{
+  int ret = OB_SUCCESS;
+  if (!inited_) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not init", KR(ret));
+  } else if (OB_FAIL(check_cancel())) {
+    LOG_WARN("check_cancel failed", KR(ret));
+  } else {
+    int tmp_ret = OB_SUCCESS;
+    if (OB_TMP_FAIL(check_sys_stat_(tenant_id))) {
+      LOG_WARN("fail to check sys stat", KR(tmp_ret), K(tenant_id));
+      ret = OB_SUCC(ret) ? tmp_ret : ret;
+    }
+    if (OB_TMP_FAIL(check_sys_param_(tenant_id))) {
+      LOG_WARN("fail to check param", KR(tmp_ret), K(tenant_id));
+      ret = OB_SUCC(ret) ? tmp_ret : ret;
+    }
+    if (OB_TMP_FAIL(check_sys_table_schemas_(tenant_id))) {
+      LOG_WARN("fail to check sys table", KR(tmp_ret), K(tenant_id));
+      ret = OB_SUCC(ret) ? tmp_ret : ret;
+    }
+  }
+  return ret;
+}
+
 int ObRootInspection::inspect(bool &passed, const char* &warning_info)
 {
   int ret = OB_SUCCESS;
@@ -819,97 +856,131 @@ int ObRootInspection::check_zone()
   return ret;
 }
 
-int ObRootInspection::check_sys_stat()
+int ObRootInspection::check_sys_stat_()
 {
   int ret = OB_SUCCESS;
   ObArray<uint64_t> tenant_ids;
   if (!inited_) {
     ret = OB_NOT_INIT;
-    LOG_WARN("not init", K(ret));
+    LOG_WARN("not init", KR(ret));
   } else if (OB_ISNULL(schema_service_)) {
     ret = OB_ERR_UNEXPECTED;
-    LOG_WARN("schema_service is null", K(ret));
+    LOG_WARN("schema_service is null", KR(ret));
   } else if (OB_FAIL(check_cancel())) {
-    LOG_WARN("check_cancel failed", K(ret));
+    LOG_WARN("check_cancel failed", KR(ret));
   } else if (OB_FAIL(ObTenantUtils::get_tenant_ids(schema_service_, tenant_ids))) {
-    LOG_WARN("get_tenant_ids failed", K(ret));
+    LOG_WARN("get_tenant_ids failed", KR(ret));
   } else {
-    ObArray<const char *> sys_stat_names;
-    ObSqlString extra_cond;
-    FOREACH_CNT_X(tenant_id, tenant_ids, OB_SUCCESS == ret) {
-      sys_stat_names.reuse();
-      extra_cond.reset();
-      // ObSysStat should construct every time because items are linked in list in construct
-      // function
-      ObSysStat sys_stat;
-      const uint64_t exec_tenant_id = ObSchemaUtils::get_exec_tenant_id(*tenant_id);
+    int backup_ret = OB_SUCCESS;
+    int tmp_ret = OB_SUCCESS;
+    FOREACH_CNT_X(tenant_id, tenant_ids, OB_SUCC(ret)) {
       if (OB_FAIL(check_cancel())) {
-        LOG_WARN("check_cancel failed", K(ret));
-      } else if (!schema_service_->is_tenant_full_schema(*tenant_id)) {
-        ret = OB_EAGAIN;
-        LOG_WARN("schema is not ready, try again", K(ret), K(*tenant_id));
-      } else if (OB_FAIL(sys_stat.set_initial_values(*tenant_id))) {
-        LOG_WARN("set initial values failed", K(ret));
-      } else if (OB_FAIL(extra_cond.assign_fmt("tenant_id = %lu", ObSchemaUtils::get_extract_tenant_id(
-                                               exec_tenant_id, *tenant_id)))) {
-        LOG_WARN("extra_cond assign_fmt failed", K(ret));
-      } else if (OB_FAIL(get_names(sys_stat.item_list_, sys_stat_names))) {
-        LOG_WARN("get sys stat names failed", K(ret));
-      } else if (OB_FAIL(check_names(*tenant_id, OB_ALL_SYS_STAT_TNAME, sys_stat_names, extra_cond))) {
-        LOG_WARN("check all sys stat names failed", "tenant_id", *tenant_id,
-            "table_name", OB_ALL_SYS_STAT_TNAME, K(sys_stat_names), K(ret));
+        LOG_WARN("check_cancel failed", KR(ret));
+      } else if (OB_TMP_FAIL(check_sys_stat_(*tenant_id))) {
+        LOG_WARN("fail to check sys stat", KR(tmp_ret), K(*tenant_id));
+        backup_ret = OB_SUCCESS == backup_ret ? tmp_ret : backup_ret;
       }
-    }
+    } // end foreach
+    ret = OB_SUCC(ret) ? backup_ret : ret;
   }
   return ret;
 }
 
-int ObRootInspection::check_sys_param()
+int ObRootInspection::check_sys_stat_(const uint64_t tenant_id)
+{
+  int ret = OB_SUCCESS;
+  ObArray<const char *> sys_stat_names;
+  ObSqlString extra_cond;
+  ObSysStat sys_stat;
+  const uint64_t exec_tenant_id = ObSchemaUtils::get_exec_tenant_id(tenant_id);
+  if (!inited_) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not init", KR(ret));
+  } else if (OB_ISNULL(schema_service_)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("schema_service is null", KR(ret));
+  } else if (OB_FAIL(check_cancel())) {
+    LOG_WARN("check_cancel failed", KR(ret));
+  } else if (!schema_service_->is_tenant_full_schema(tenant_id)) {
+    ret = OB_EAGAIN;
+    LOG_WARN("schema is not ready, try again", KR(ret), K(tenant_id));
+  } else if (OB_FAIL(sys_stat.set_initial_values(tenant_id))) {
+    LOG_WARN("set initial values failed", KR(ret), K(tenant_id));
+  } else if (OB_FAIL(extra_cond.assign_fmt("tenant_id = %lu",
+             ObSchemaUtils::get_extract_tenant_id(exec_tenant_id, tenant_id)))) {
+    LOG_WARN("extra_cond assign_fmt failed", KR(ret), K(tenant_id));
+  } else if (OB_FAIL(get_names(sys_stat.item_list_, sys_stat_names))) {
+    LOG_WARN("get sys stat names failed", KR(ret), K(tenant_id));
+  } else if (OB_FAIL(check_names(tenant_id, OB_ALL_SYS_STAT_TNAME, sys_stat_names, extra_cond))) {
+    LOG_WARN("check all sys stat names failed", K(ret), K(tenant_id),
+             "table_name", OB_ALL_SYS_STAT_TNAME, K(sys_stat_names));
+  }
+  return ret;
+}
+
+int ObRootInspection::check_sys_param_()
 {
   int ret = OB_SUCCESS;
   ObArray<uint64_t> tenant_ids;
   if (!inited_) {
     ret = OB_NOT_INIT;
-    LOG_WARN("not init", K(ret));
+    LOG_WARN("not init", KR(ret));
   } else if (OB_ISNULL(schema_service_)) {
     ret = OB_ERR_UNEXPECTED;
-    LOG_WARN("schema_service is null", K(ret));
+    LOG_WARN("schema_service is null", KR(ret));
   } else if (OB_FAIL(check_cancel())) {
-    LOG_WARN("check_cancel failed", K(ret));
+    LOG_WARN("check_cancel failed", KR(ret));
   } else if (OB_FAIL(ObTenantUtils::get_tenant_ids(schema_service_, tenant_ids))) {
-    LOG_WARN("get_tenant_ids failed", K(ret));
+    LOG_WARN("get_tenant_ids failed", KR(ret));
   } else {
-    ObArray<const char *> sys_param_names;
-    ObSqlString extra_cond;
-    FOREACH_CNT_X(tenant_id, tenant_ids, OB_SUCCESS == ret) {
-      sys_param_names.reuse();
-      extra_cond.reset();
-      const uint64_t exec_tenant_id = ObSchemaUtils::get_exec_tenant_id(*tenant_id);
+    int backup_ret = OB_SUCCESS;
+    int tmp_ret = OB_SUCCESS;
+    FOREACH_CNT_X(tenant_id, tenant_ids, OB_SUCC(ret)) {
       if (OB_FAIL(check_cancel())) {
         LOG_WARN("check_cancel failed", K(ret));
-      } else if (!schema_service_->is_tenant_full_schema(*tenant_id)) {
-        ret = OB_EAGAIN;
-        LOG_WARN("schema is not ready, try again", K(ret), K(*tenant_id));
-      } else if (OB_FAIL(extra_cond.assign_fmt("tenant_id = %lu", ObSchemaUtils::get_extract_tenant_id(
-                                               exec_tenant_id, *tenant_id)))) {
-      } else if (OB_FAIL(get_sys_param_names(sys_param_names))) {
-        LOG_WARN("get sys param names failed", K(ret));
-      } else if (OB_FAIL(check_and_insert_sys_params(*tenant_id,
-          OB_ALL_SYS_VARIABLE_TNAME, sys_param_names, extra_cond))) {
-        LOG_WARN("check and insert sys params failed", "tenant_id", *tenant_id,
-            "table_name", OB_ALL_SYS_VARIABLE_TNAME, K(sys_param_names), K(extra_cond), K(ret));
-      } else if (OB_FAIL(check_names(*tenant_id, OB_ALL_SYS_VARIABLE_TNAME,
-          sys_param_names, extra_cond))) { // For robustness, read it out again
-        LOG_WARN("check all sys params names failed", "tenant_id", *tenant_id,
-            "table_name", OB_ALL_SYS_VARIABLE_TNAME, K(sys_param_names), K(extra_cond), K(ret));
+      } else if (OB_TMP_FAIL(check_sys_param_(*tenant_id))) {
+        LOG_WARN("fail to check sys param", KR(tmp_ret), K(*tenant_id));
+        backup_ret = OB_SUCCESS == backup_ret ? tmp_ret : backup_ret;
       }
-    }
+    } // end foreach
+    ret = OB_SUCC(ret) ? backup_ret : ret;
+  }
+  return ret;
+}
+
+int ObRootInspection::check_sys_param_(const uint64_t tenant_id)
+{
+  int ret = OB_SUCCESS;
+  ObArray<uint64_t> tenant_ids;
+  ObArray<const char *> sys_param_names;
+  ObSqlString extra_cond;
+  const uint64_t exec_tenant_id = ObSchemaUtils::get_exec_tenant_id(tenant_id);
+  if (!inited_) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not init", KR(ret));
+  } else if (OB_ISNULL(schema_service_)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("schema_service is null", KR(ret));
+  } else if (OB_FAIL(check_cancel())) {
+    LOG_WARN("check_cancel failed", KR(ret));
+  } else if (!schema_service_->is_tenant_full_schema(tenant_id)) {
+    ret = OB_EAGAIN;
+    LOG_WARN("schema is not ready, try again", KR(ret), K(tenant_id));
+  } else if (OB_FAIL(extra_cond.assign_fmt("tenant_id = %lu",
+             ObSchemaUtils::get_extract_tenant_id(exec_tenant_id, tenant_id)))) {
+    LOG_WARN("extra_cond assign_fmt failed", KR(ret), K(tenant_id));
+  } else if (OB_FAIL(get_sys_param_names(sys_param_names))) {
+    LOG_WARN("get sys param names failed", KR(ret), K(tenant_id));
+  } else if (OB_FAIL(check_names(tenant_id, OB_ALL_SYS_VARIABLE_TNAME,
+             sys_param_names, extra_cond))) {
+    LOG_WARN("check all sys params names failed", KR(ret), K(tenant_id),
+             "table_name", OB_ALL_SYS_VARIABLE_TNAME, K(sys_param_names), K(extra_cond));
   }
   if (OB_SCHEMA_ERROR != ret) {
   } else if (GCONF.in_upgrade_mode()) {
-    LOG_WARN("check sys_variable failed", K(ret));
+    LOG_WARN("check sys_variable failed", KR(ret));
   } else {
-    LOG_ERROR("check sys_variable failed", K(ret));
+    LOG_ERROR("check sys_variable failed", KR(ret));
   }
   return ret;
 }
@@ -954,38 +1025,6 @@ int ObRootInspection::get_sys_param_names(ObIArray<const char *> &names)
   return ret;
 }
 
-int ObRootInspection::check_and_insert_sys_params(uint64_t tenant_id,
-                                                  const char *table_name,
-                                                  const ObIArray<const char *> &names,
-                                                  const ObSqlString &extra_cond)
-{
-  int ret = OB_SUCCESS;
-  ObArray<Name> fetch_names; // Get the data of the internal table
-  ObArray<Name> extra_names; // data inner table more than hard code
-  ObArray<Name> miss_names; // data inner table less than hard code
-  if (!inited_) {
-    ret = OB_NOT_INIT;
-    LOG_WARN("not init", K(ret));
-  } else if (OB_FAIL(check_cancel())) {
-    LOG_WARN("check_cancel failed", K(ret));
-  } else if (OB_INVALID_ID == tenant_id || NULL == table_name || names.count() <= 0) {
-    // extra_cond can be empty, so don't check it here
-    ret = OB_INVALID_ARGUMENT;
-    LOG_WARN("invalid tenant_id or table_name is null or names is empty", K(tenant_id),
-        K(table_name), K(names), K(ret));
-  } else if (OB_FAIL(calc_diff_names(tenant_id, table_name, names,
-      extra_cond, fetch_names, extra_names, miss_names))) {
-    LOG_WARN("fail to calc diff names", K(ret), K(table_name), K(names), K(extra_cond));
-  } else {
-    if (miss_names.count() > 0) {
-      // don't need to set ret
-      LOG_WARN("some sys var exist in hard code, but does not exist in inner table, "
-          "they will be inserted into table", K(table_name), K(miss_names));
-    }
-  }
-  return ret;
-}
-
 int ObRootInspection::check_names(const uint64_t tenant_id,
                                   const char *table_name,
                                   const ObIArray<const char *> &names,
@@ -1012,17 +1051,17 @@ int ObRootInspection::check_names(const uint64_t tenant_id,
       if (fetch_names.count() <= 0) {
         // don't need to set ret
         LOG_WARN("maybe tenant or zone has been deleted, ignore it",
-            K(table_name), K(extra_cond));
+                 K(tenant_id), K(table_name), K(extra_cond));
       } else {
         if (extra_names.count() > 0) {
           // don't need to set ret
           LOG_WARN("some item exist in table, but not hard coded",
-              K(table_name), K(extra_names));
+                   K(tenant_id), K(table_name), K(extra_names));
         }
         if (miss_names.count() > 0) {
           ret = OB_ENTRY_NOT_EXIST;
           LOG_WARN("some item exist in hard code, but not exist in inner table",
-                   K(ret), K(table_name), K(miss_names));
+                   K(ret), K(tenant_id), K(table_name), K(miss_names));
         }
       }
     }
@@ -1166,19 +1205,17 @@ int ObRootInspection::check_sys_table_schemas_()
   } else if (OB_FAIL(ObTenantUtils::get_tenant_ids(schema_service_, tenant_ids))) {
     LOG_WARN("get_tenant_ids failed", KR(ret));
   } else {
-    FOREACH(tenant_id, tenant_ids) { // ignore ret
-      int tmp_ret = OB_SUCCESS;
-      int64_t schema_version = OB_INVALID_VERSION;
-      if (OB_SUCCESS != (tmp_ret = schema_service_->get_tenant_refreshed_schema_version(*tenant_id, schema_version))) {
-        LOG_WARN("fail to get tenant refreshed schema_version", KR(tmp_ret), K(*tenant_id));
-      } else if (!ObSchemaService::is_formal_version(schema_version)) {
-        tmp_ret = OB_EAGAIN;
-        LOG_WARN("schema is not ready, try again", KR(tmp_ret), K(*tenant_id), K(schema_version));
-      } else if (OB_SUCCESS != (tmp_ret = check_sys_table_schemas_(*tenant_id))) {
+    int backup_ret = OB_SUCCESS;
+    int tmp_ret = OB_SUCCESS;
+    FOREACH_X(tenant_id, tenant_ids, OB_SUCC(ret)) {
+      if (OB_FAIL(check_cancel())) {
+        LOG_WARN("check_cancel failed", KR(ret));
+      } else if (OB_TMP_FAIL(check_sys_table_schemas_(*tenant_id))) {
         LOG_WARN("fail to check sys table schemas by tenant", KR(tmp_ret), K(*tenant_id));
+        backup_ret = OB_SUCCESS == backup_ret ? tmp_ret : backup_ret;
       }
-      ret = OB_SUCC(ret) ? tmp_ret : ret;
-    }
+    } // end foreach
+    ret = OB_SUCC(ret) ? backup_ret : ret;
   }
   return ret;
 }
@@ -1187,6 +1224,7 @@ int ObRootInspection::check_sys_table_schemas_(
     const uint64_t tenant_id)
 {
   int ret = OB_SUCCESS;
+  int64_t schema_version = OB_INVALID_VERSION;
   if (OB_UNLIKELY(!inited_)) {
     ret = OB_NOT_INIT;
     LOG_WARN("not init", KR(ret));
@@ -1197,6 +1235,15 @@ int ObRootInspection::check_sys_table_schemas_(
              || OB_INVALID_TENANT_ID == tenant_id)) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("invalid tenant_id", KR(ret), K(tenant_id));
+  } else if (OB_ISNULL(schema_service_)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("schema_service is null", KR(ret));
+  } else if (OB_FAIL(schema_service_->get_tenant_refreshed_schema_version(
+                     tenant_id, schema_version))) {
+    LOG_WARN("fail to get tenant refreshed schema_version", KR(ret), K(tenant_id));
+  } else if (!ObSchemaService::is_formal_version(schema_version)) {
+    ret = OB_EAGAIN;
+    LOG_WARN("schema is not ready, try again", KR(ret), K(tenant_id), K(schema_version));
   } else {
     const schema_create_func *creator_ptr_array[] = {
       share::all_core_table_schema_creator,
@@ -1209,6 +1256,7 @@ int ObRootInspection::check_sys_table_schemas_(
       NULL };
 
     int back_ret = OB_SUCCESS;
+    int tmp_ret = OB_SUCCESS;
     ObTableSchema table_schema;
     bool exist = false;
     for (const schema_create_func **creator_ptr_ptr = creator_ptr_array;
@@ -1230,17 +1278,23 @@ int ObRootInspection::check_sys_table_schemas_(
                    KR(ret), K(tenant_id), K(table_schema));
         } else if (!exist) {
           // skip
-        } else if (OB_FAIL(check_table_schema(tenant_id, table_schema))) {
-          // don't print table_schema, otherwise log will be too much
-          LOG_WARN("check_table_schema failed", KR(ret), K(tenant_id));
-        } else if (OB_FAIL(check_sys_view_(tenant_id, table_schema))) {
-          LOG_WARN("fail to check sys view", KR(ret), K(tenant_id));
+        } else {
+          if (OB_TMP_FAIL(check_table_schema(tenant_id, table_schema))) {
+            // don't print table_schema, otherwise log will be too much
+            LOG_WARN("check table schema failed", KR(tmp_ret), K(tenant_id),
+                     "table_id", table_schema.get_table_id(), "table_name", table_schema.get_table_name());
+            back_ret = OB_SUCCESS == back_ret ? tmp_ret : back_ret;
+          }
+
+          if (OB_TMP_FAIL(check_sys_view_(tenant_id, table_schema))) {
+            LOG_WARN("check sys view failed", KR(tmp_ret), K(tenant_id),
+                     "table_id", table_schema.get_table_id(), "table_name", table_schema.get_table_name());
+            back_ret = OB_SUCCESS == back_ret ? tmp_ret : back_ret;
+          }
         }
-        back_ret = OB_SUCCESS == back_ret ? ret : back_ret;
-        ret = OB_SUCCESS;
-      }
-    }
-    ret = back_ret;
+      } // end for
+    } // end for
+    ret = OB_SUCC(ret) ? back_ret : ret;
   }
   if (OB_SCHEMA_ERROR != ret) {
   } else if (GCONF.in_upgrade_mode()) {
@@ -1752,6 +1806,69 @@ int ObRootInspection::check_column_schema_(const ObString &table_name,
   return ret;
 }
 
+int ObRootInspection::check_data_version_()
+{
+  int ret = OB_SUCCESS;
+  ObArray<uint64_t> tenant_ids;
+  if (OB_UNLIKELY(!inited_)) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not init", KR(ret));
+  } else if (OB_FAIL(check_cancel())) {
+    LOG_WARN("check_cancel failed", KR(ret));
+  } else if (OB_ISNULL(schema_service_)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("schema_service is null", KR(ret));
+  } else if (OB_FAIL(ObTenantUtils::get_tenant_ids(schema_service_, tenant_ids))) {
+    LOG_WARN("get_tenant_ids failed", KR(ret));
+  } else {
+    int backup_ret = OB_SUCCESS;
+    int tmp_ret = OB_SUCCESS;
+    FOREACH_X(tenant_id, tenant_ids, OB_SUCC(ret)) {
+      if (OB_FAIL(check_cancel())) {
+        LOG_WARN("check_cancel failed", KR(ret));
+      } else if (OB_TMP_FAIL(check_data_version_(*tenant_id))) {
+        LOG_WARN("fail to check data version by tenant", KR(tmp_ret), K(*tenant_id));
+        backup_ret = OB_SUCCESS == backup_ret ? tmp_ret : backup_ret;
+      }
+    } // end foreach
+    ret = OB_SUCC(ret) ? backup_ret : ret;
+  }
+  return ret;
+}
+
+int ObRootInspection::check_data_version_(
+    const uint64_t tenant_id)
+{
+  int ret = OB_SUCCESS;
+  if (OB_UNLIKELY(!inited_)) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not init", KR(ret));
+  } else if (OB_FAIL(check_cancel())) {
+    LOG_WARN("check_cancel failed", KR(ret));
+  } else {
+    share::ObGlobalStatProxy proxy(*sql_proxy_, tenant_id);
+    uint64_t target_data_version = 0;
+    uint64_t current_data_version = 0;
+    uint64_t compatible_version = 0;
+    bool for_update = false;
+    if (OB_FAIL(proxy.get_target_data_version(for_update, target_data_version))) {
+      LOG_WARN("fail to get target data version", KR(ret), K(tenant_id));
+    } else if (OB_FAIL(proxy.get_current_data_version(current_data_version))) {
+      LOG_WARN("fail to get current data version", KR(ret), K(tenant_id));
+    } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compatible_version))) {
+      LOG_WARN("fail to get min data version", KR(ret), K(tenant_id));
+    } else if (target_data_version != current_data_version
+               || target_data_version != compatible_version
+               || target_data_version != DATA_CURRENT_VERSION) {
+      ret = OB_STATE_NOT_MATCH;
+      LOG_WARN("data_version not match, upgrade process should be run",
+               KR(ret), K(tenant_id), K(target_data_version),
+               K(current_data_version), K(compatible_version));
+    }
+  }
+  return ret;
+}
+
 int ObRootInspection::check_cancel()
 {
   int ret = OB_SUCCESS;
@@ -1846,6 +1963,8 @@ int ObUpgradeInspection::inner_get_next_row(common::ObNewRow *&row)
           root_inspection_->is_sys_param_passed()));
       ADD_ROW("sys_table_schema_check", CHECK_RESULT(root_inspection_->is_all_checked(),
           root_inspection_->is_sys_table_schema_passed()));
+      ADD_ROW("data_version_check", CHECK_RESULT(root_inspection_->is_all_checked(),
+          root_inspection_->is_data_version_passed()));
 
       bool upgrade_job_passed = true;
       for (int64_t i = 0; i < UPGRADE_JOB_TYPE_COUNT; i++) {

src/rootserver/ob_root_inspection.h

@@ -156,11 +156,13 @@ public:
   void start() { stopped_ = false; }
   void stop() { stopped_ = true; }
   virtual int check_all();
+  int check_tenant(const uint64_t tenant_id);
 
   inline bool is_zone_passed() const { return zone_passed_; }
   inline bool is_sys_param_passed() const { return sys_param_passed_; }
   inline bool is_sys_stat_passed() const { return sys_stat_passed_; }
   inline bool is_sys_table_schema_passed() const { return sys_table_schema_passed_; }
+  inline bool is_data_version_passed() const { return data_version_passed_; }
   inline bool is_all_checked() const { return all_checked_; }
   inline bool is_all_passed() const { return all_passed_; }
   inline bool can_retry() const { return can_retry_ && !stopped_; }
@@ -186,16 +188,14 @@ private:
   static const int64_t NAME_BUF_LEN = 64;
   typedef common::ObFixedLengthString<NAME_BUF_LEN> Name;
   int check_zone();
-  int check_sys_stat();
-  int check_sys_param();
+  int check_sys_stat_();
+  int check_sys_stat_(const uint64_t tenant_id);
+  int check_sys_param_();
+  int check_sys_param_(const uint64_t tenant_id);
 
   template<typename Item>
   int get_names(const common::ObDList<Item> &list, common::ObIArray<const char*> &names);
   int get_sys_param_names(common::ObIArray<const char *> &names);
-  int check_and_insert_sys_params(uint64_t tenant_id,
-                                  const char *table_name,
-                                  const common::ObIArray<const char *> &names,
-                                  const common::ObSqlString &extra_cond);
   int check_names(const uint64_t tenant_id,
                   const char *table_name,
                   const common::ObIArray<const char *> &names,
@@ -217,6 +217,9 @@ private:
                                   const share::schema::ObColumnSchemaV2 &hard_code_column,
                                   const bool ignore_column_id);
 
+  int check_data_version_();
+  int check_data_version_(const uint64_t tenant_id);
+
   bool check_str_with_lower_case_(const ObString &str);
   int check_sys_view_(const uint64_t tenant_id,
                       const share::schema::ObTableSchema &hard_code_table);
@@ -229,6 +232,7 @@ private:
   bool sys_param_passed_;
   bool sys_stat_passed_;
   bool sys_table_schema_passed_;
+  bool data_version_passed_;
 
   bool all_checked_;
   bool all_passed_;

src/rootserver/ob_root_service.cpp

@@ -93,6 +93,7 @@
 #include "storage/ob_file_system_router.h"
 #include "rootserver/freeze/ob_major_freeze_helper.h"
 #include "share/restore/ob_physical_restore_table_operator.h"//ObPhysicalRestoreTableOperator
+#include "share/ob_cluster_event_history_table_operator.h"//CLUSTER_EVENT_INSTANCE
 #include "share/scn.h"
 namespace oceanbase
 {
@@ -880,7 +881,8 @@ int ObRootService::init(ObServerConfig &config,
     FLOG_WARN("init snapshot manager failed", KR(ret));
   } else if (OB_FAIL(root_inspection_.init(*schema_service_, zone_manager_, sql_proxy_, &common_proxy_))) {
     FLOG_WARN("init root inspection failed", KR(ret));
-  } else if (OB_FAIL(upgrade_executor_.init(*schema_service_, sql_proxy_, rpc_proxy_, common_proxy_))) {
+  } else if (OB_FAIL(upgrade_executor_.init(*schema_service_,
+             root_inspection_, sql_proxy_, rpc_proxy_, common_proxy_))) {
     FLOG_WARN("init upgrade_executor failed", KR(ret));
   } else if (OB_FAIL(upgrade_storage_format_executor_.init(*this, ddl_service_))) {
     FLOG_WARN("init upgrade storage format executor failed", KR(ret));
@@ -1080,6 +1082,7 @@ int ObRootService::start_service()
     ddl_service_.restart();
     server_manager_.reset();
     zone_manager_.reset();
+    OTC_MGR.reset_version_has_refreshed();
 
     if (OB_FAIL(hb_checker_.start())) {
       FLOG_WARN("hb checker start failed", KR(ret));
@@ -1432,21 +1435,22 @@ int ObRootService::submit_offline_server_task(const common::ObAddr &server)
 }
 
 int ObRootService::submit_upgrade_task(
-    const obrpc::ObUpgradeJobArg::Action action,
-    const int64_t version)
+    const obrpc::ObUpgradeJobArg &arg)
 {
   int ret = OB_SUCCESS;
-  ObUpgradeTask task(upgrade_executor_, action, version);
+  ObUpgradeTask task(upgrade_executor_);
   task.set_retry_times(0); //not repeat
   if (!inited_) {
     ret = OB_NOT_INIT;
     LOG_WARN("not init", KR(ret));
+  } else if (OB_FAIL(task.init(arg))) {
+    LOG_WARN("task init failed", KR(ret), K(arg));
   } else if (OB_FAIL(upgrade_executor_.can_execute())) {
-    LOG_WARN("can't run task now", KR(ret), K(action), K(version));
+    LOG_WARN("can't run task now", KR(ret), K(arg));
   } else if (OB_FAIL(task_queue_.add_async_task(task))) {
-    LOG_WARN("submit upgrade task fail", KR(ret), K(action), K(version));
+    LOG_WARN("submit upgrade task fail", KR(ret), K(arg));
   } else {
-    LOG_INFO("submit upgrade task success", KR(ret), K(action), K(version));
+    LOG_INFO("submit upgrade task success", KR(ret), K(arg));
   }
   return ret;
 }
@@ -1999,6 +2003,19 @@ int ObRootService::execute_bootstrap(const obrpc::ObBootstrapArg &arg)
       LOG_ERROR("failed to update cpu_quota_concurrency", K(ret));
     }
 
+    if (OB_SUCC(ret)) {
+      char ori_min_server_version[OB_SERVER_VERSION_LENGTH] = {'\0'};
+      uint64_t ori_cluster_version = GET_MIN_CLUSTER_VERSION();
+      if (OB_INVALID_INDEX == ObClusterVersion::print_version_str(
+          ori_min_server_version, OB_SERVER_VERSION_LENGTH, ori_cluster_version)) {
+         ret = OB_INVALID_ARGUMENT;
+         LOG_WARN("fail to print version str", KR(ret), K(ori_cluster_version));
+      } else {
+        CLUSTER_EVENT_SYNC_ADD("BOOTSTRAP", "BOOTSTRAP_SUCCESS",
+                               "cluster_version", ori_min_server_version);
+      }
+    }
+
     //clear bootstrap flag, regardless failure or success
     int tmp_ret = OB_SUCCESS;
     if (OB_SUCCESS != (tmp_ret = clear_special_cluster_schema_status())) {
@@ -6678,19 +6695,20 @@ int ObRootService::construct_rs_list_arg(
 int ObRootService::add_server(const obrpc::ObAdminServerArg &arg)
 {
   int ret = OB_SUCCESS;
+  uint64_t sys_data_version = 0;
   // argument
-  ObCheckServerEmptyArg new_arg;
-  new_arg.mode_ = ObCheckServerEmptyArg::ADD_SERVER;
   if (!inited_) {
     ret = OB_NOT_INIT;
     LOG_WARN("not init", K(ret));
   } else if (!arg.is_valid()) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("invalid arg", K(arg), K(ret));
+  } else if (OB_FAIL(GET_MIN_DATA_VERSION(OB_SYS_TENANT_ID, sys_data_version))) {
+    LOG_WARN("fail to get sys data version", KR(ret));
   } else {
     LOG_INFO("add_server", K(arg), "timeout_ts", THIS_WORKER.get_timeout_ts());
-    ObCheckServerEmptyArg new_arg;
-    new_arg.mode_ = ObCheckServerEmptyArg::ADD_SERVER;
+    ObCheckServerEmptyArg new_arg(ObCheckServerEmptyArg::ADD_SERVER,
+                                  sys_data_version);
     ObCheckDeploymentModeArg dp_arg;
     dp_arg.single_zone_deployment_on_ = OB_FILE_SYSTEM_ROUTER.is_single_zone_deployment_on();
 
@@ -7967,30 +7985,23 @@ int ObRootService::run_upgrade_job(const obrpc::ObUpgradeJobArg &arg)
   int64_t version = arg.version_;
   if (!inited_) {
     ret = OB_NOT_INIT;
-    LOG_WARN("not init", KR(ret));
+    LOG_WARN("not init", KR(ret), K(arg));
   } else if (!arg.is_valid()) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("invalid arg", K(arg), KR(ret));
   } else if (ObUpgradeJobArg::UPGRADE_POST_ACTION == arg.action_
-             || ObUpgradeJobArg::UPGRADE_SYSTEM_VARIABLE == arg.action_
-             || ObUpgradeJobArg::UPGRADE_SYSTEM_TABLE == arg.action_) {
-    if (ObUpgradeJobArg::UPGRADE_POST_ACTION == arg.action_
-        && !ObUpgradeChecker::check_data_version_exist(version)) {
-      ret = OB_NOT_SUPPORTED;
-      LOG_WARN("unsupported version to run upgrade job", KR(ret), K(version));
-      LOG_USER_ERROR(OB_NOT_SUPPORTED, "run upgrade job with such version is");
-    } else if (OB_FAIL(submit_upgrade_task(arg.action_, version))) {
-      LOG_WARN("fail to submit upgrade task", KR(ret), K(arg));
-    }
+             && !ObUpgradeChecker::check_data_version_exist(version)) {
+    ret = OB_NOT_SUPPORTED;
+    LOG_WARN("unsupported version to run upgrade job", KR(ret), K(version));
+    LOG_USER_ERROR(OB_NOT_SUPPORTED, "run upgrade job with such version is");
   } else if (ObUpgradeJobArg::STOP_UPGRADE_JOB == arg.action_) {
     if (OB_FAIL(upgrade_executor_.stop())) {
       LOG_WARN("fail to stop upgrade task", KR(ret));
     } else {
       upgrade_executor_.start();
     }
-  } else {
-    ret = OB_ERR_UNEXPECTED;
-    LOG_WARN("invalid action type", KR(ret), K(arg));
+  } else if (OB_FAIL(submit_upgrade_task(arg))) {
+    LOG_WARN("fail to submit upgrade task", KR(ret), K(arg));
   }
   ROOTSERVICE_EVENT_ADD("root_service", "admin_run_upgrade_job", KR(ret), K(arg));
   return ret;
@@ -7999,12 +8010,34 @@ int ObRootService::run_upgrade_job(const obrpc::ObUpgradeJobArg &arg)
 int ObRootService::upgrade_table_schema(const obrpc::ObUpgradeTableSchemaArg &arg)
 {
   int ret = OB_SUCCESS;
+  const int64_t start = ObTimeUtility::current_time();
+  FLOG_INFO("[UPGRADE] start to upgrade table", K(arg));
   if (!inited_) {
     ret = OB_NOT_INIT;
     LOG_WARN("not init", KR(ret));
-  } else if (OB_FAIL(ddl_service_.upgrade_table_schema(arg))) {
-    LOG_WARN("fail to upgrade table schema", KR(ret), K(arg));
+  } else if (!arg.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("arg is invalid", KR(ret), K(arg));
+  } else if (!arg.upgrade_virtual_schema()) {
+    // upgrade single system table
+    if (OB_FAIL(ddl_service_.upgrade_table_schema(arg))) {
+      LOG_WARN("fail to upgrade table schema", KR(ret), K(arg));
+    }
+  } else {
+    // upgrade all virtual table/sys view
+    ObSystemAdminCtx ctx;
+    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
+      LOG_WARN("init_sys_admin_ctx failed", K(ret));
+    } else {
+      ObAdminUpgradeVirtualSchema admin_util(ctx);
+      int64_t upgrade_cnt = 0;
+      if (OB_FAIL(admin_util.execute(arg.get_tenant_id(), upgrade_cnt))) {
+        LOG_WARN("upgrade virtual schema failed", KR(ret), K(arg));
+      }
+    }
   }
+  FLOG_INFO("[UPGRADE] finish upgrade table", KR(ret), K(arg),
+            "cost_us", ObTimeUtility::current_time() - start);
   return ret;
 }
 

src/rootserver/ob_root_service.h

@@ -696,7 +696,7 @@ public:
   int report_single_replica(const int64_t tenant_id, const share::ObLSID &ls_id);
   // @see RsListChangeCb
   int submit_update_rslist_task(const bool force_update = false);
-  int submit_upgrade_task(const obrpc::ObUpgradeJobArg::Action action, const int64_t version);
+  int submit_upgrade_task(const obrpc::ObUpgradeJobArg &arg);
   int submit_upgrade_storage_format_version_task();
   int submit_create_inner_schema_task();
   int submit_async_minor_freeze_task(const obrpc::ObRootMinorFreezeArg &arg);

src/rootserver/ob_rs_async_rpc_proxy.h

@@ -62,6 +62,8 @@ RPC_F(obrpc::OB_BATCH_GET_TABLET_AUTOINC_SEQ, obrpc::ObBatchGetTabletAutoincSeqA
     obrpc::ObBatchGetTabletAutoincSeqRes, ObBatchGetTabletAutoincSeqProxy);
 RPC_F(obrpc::OB_BATCH_SET_TABLET_AUTOINC_SEQ, obrpc::ObBatchSetTabletAutoincSeqArg,
     obrpc::ObBatchSetTabletAutoincSeqRes, ObBatchSetTabletAutoincSeqProxy);
+RPC_F(obrpc::OB_INIT_TENANT_CONFIG, obrpc::ObInitTenantConfigArg,
+    obrpc::ObInitTenantConfigRes, ObInitTenantConfigProxy);
 RPC_F(obrpc::OB_DDL_CHECK_TABLET_MERGE_STATUS, obrpc::ObDDLCheckTabletMergeStatusArg,
     obrpc::ObDDLCheckTabletMergeStatusResult, ObCheckTabletMergeStatusProxy);
 }//end namespace rootserver

src/rootserver/ob_rs_job_table_operator.cpp

@@ -65,7 +65,14 @@ static const char* job_type_str_array[JOB_TYPE_MAX] = {
   "CREATE_INNER_SCHEMA",
   "UPGRADE_POST_ACTION",
   "UPGRADE_SYSTEM_VARIABLE",
-  "UPGRADE_SYSTEM_TABLE"
+  "UPGRADE_SYSTEM_TABLE",
+  "UPGRADE_BEGIN",
+  "UPGRADE_VIRTUAL_SCHEMA",
+  "UPGRADE_SYSTEM_PACKAGE",
+  "UPGRADE_ALL_POST_ACTION",
+  "UPGRADE_INSPECTION",
+  "UPGRADE_END",
+  "UPGRADE_ALL"
 };
 
 const char* ObRsJobTableOperator::get_job_type_str(ObRsJobType job_type)

src/rootserver/ob_rs_job_table_operator.h

@@ -85,6 +85,13 @@ enum ObRsJobType
   JOB_TYPE_UPGRADE_POST_ACTION,
   JOB_TYPE_UPGRADE_SYSTEM_VARIABLE,
   JOB_TYPE_UPGRADE_SYSTEM_TABLE,
+  JOB_TYPE_UPGRADE_BEGIN,
+  JOB_TYPE_UPGRADE_VIRTUAL_SCHEMA,
+  JOB_TYPE_UPGRADE_SYSTEM_PACKAGE,
+  JOB_TYPE_UPGRADE_ALL_POST_ACTION,
+  JOB_TYPE_UPGRADE_INSPECTION,
+  JOB_TYPE_UPGRADE_END,
+  JOB_TYPE_UPGRADE_ALL,
   JOB_TYPE_MAX
 };
 

src/rootserver/ob_system_admin_util.cpp

@@ -50,6 +50,7 @@
 #include "storage/ob_file_system_router.h"
 #include "logservice/leader_coordinator/table_accessor.h"
 #include "rootserver/freeze/ob_major_freeze_helper.h"
+#include "share/ob_cluster_event_history_table_operator.h"//CLUSTER_EVENT_INSTANCE
 namespace oceanbase
 {
 using namespace common;
@@ -1160,7 +1161,7 @@ int ObAdminUpgradeVirtualSchema::execute()
   } else {
     FOREACH(tenant_id, tenant_ids) { // ignore ret
       int tmp_ret = OB_SUCCESS;
-      if (OB_SUCCESS != (tmp_ret = execute_(*tenant_id, upgrade_cnt))) {
+      if (OB_SUCCESS != (tmp_ret = execute(*tenant_id, upgrade_cnt))) {
         LOG_WARN("fail to execute upgrade virtual table by tenant", KR(tmp_ret), K(*tenant_id));
       }
       ret = OB_SUCC(ret) ? tmp_ret : ret;
@@ -1176,7 +1177,7 @@ int ObAdminUpgradeVirtualSchema::execute()
   return ret;
 }
 
-int ObAdminUpgradeVirtualSchema::execute_(
+int ObAdminUpgradeVirtualSchema::execute(
     const uint64_t tenant_id,
     int64_t &upgrade_cnt)
 {
@@ -1362,56 +1363,49 @@ int ObAdminUpgradeVirtualSchema::upgrade_(
 int ObAdminUpgradeCmd::execute(const Bool &upgrade)
 {
   int ret = OB_SUCCESS;
-  char min_server_version[OB_SERVER_VERSION_LENGTH];
-  uint64_t current_version = 0;
-  if (OB_FAIL(ctx_.server_mgr_->get_min_server_version(min_server_version))) {
-    LOG_WARN("failed to get the min server version", KR(ret));
-  } else if (OB_FAIL(ObClusterVersion::get_version(min_server_version, current_version))) {
-    LOG_WARN("fail to parse current version", KR(ret), K(min_server_version));
-  } else {
-    // set min_observer_version and enable_upgrade_mode
-    HEAP_VAR(ObAdminSetConfigItem, item) {
-      obrpc::ObAdminSetConfigArg set_config_arg;
-      set_config_arg.is_inner_ = true;
-      const char *min_obs_version_name = "min_observer_version";
-      const char *enable_upgrade_name = "enable_upgrade_mode";
-      ObAdminSetConfig admin_set_config(ctx_);
+  // set enable_upgrade_mode
+  HEAP_VAR(ObAdminSetConfigItem, item) {
+    obrpc::ObAdminSetConfigArg set_config_arg;
+    set_config_arg.is_inner_ = true;
+    const char *enable_upgrade_name = "enable_upgrade_mode";
+    ObAdminSetConfig admin_set_config(ctx_);
+    char min_server_version[OB_SERVER_VERSION_LENGTH] = {'\0'};
+    uint64_t cluster_version = GET_MIN_CLUSTER_VERSION();
 
-      if (OB_FAIL(item.name_.assign(min_obs_version_name))) {
-        LOG_WARN("assign min_observer_version config name failed", KR(ret));
-      } else if (OB_FAIL(item.value_.assign(min_server_version))) {
-        LOG_WARN("assign min_observer_version config value failed", KR(ret));
+    if (OB_INVALID_INDEX == ObClusterVersion::print_version_str(
+        min_server_version, OB_SERVER_VERSION_LENGTH, cluster_version)) {
+       ret = OB_INVALID_ARGUMENT;
+       LOG_WARN("fail to print version str", KR(ret), K(cluster_version));
+    } else if (OB_FAIL(item.name_.assign(enable_upgrade_name))) {
+      LOG_WARN("assign enable_upgrade_mode config name failed", KR(ret));
+    } else if (OB_FAIL(item.value_.assign((upgrade ? "true" : "false")))) {
+      LOG_WARN("assign enable_upgrade_mode config value failed", KR(ret));
+    } else if (OB_FAIL(set_config_arg.items_.push_back(item))) {
+      LOG_WARN("add enable_upgrade_mode config item failed", KR(ret));
+    } else {
+      const char *upgrade_stage_name = "_upgrade_stage";
+      obrpc::ObUpgradeStage stage = upgrade ?
+                                    obrpc::OB_UPGRADE_STAGE_PREUPGRADE :
+                                    obrpc::OB_UPGRADE_STAGE_NONE;
+      if (OB_FAIL(item.name_.assign(upgrade_stage_name))) {
+        LOG_WARN("assign _upgrade_stage config name failed", KR(ret), K(upgrade));
+      } else if (OB_FAIL(item.value_.assign(obrpc::get_upgrade_stage_str(stage)))) {
+        LOG_WARN("assign _upgrade_stage config value failed", KR(ret), K(stage), K(upgrade));
       } else if (OB_FAIL(set_config_arg.items_.push_back(item))) {
-        LOG_WARN("add min_observer_version config item failed", KR(ret));
-      } else if (OB_FAIL(item.name_.assign(enable_upgrade_name))) {
-        LOG_WARN("assign enable_upgrade_mode config name failed", KR(ret));
-      } else if (OB_FAIL(item.value_.assign((upgrade ? "true" : "false")))) {
-        LOG_WARN("assign enable_upgrade_mode config value failed", KR(ret));
-      } else if (OB_FAIL(set_config_arg.items_.push_back(item))) {
-        LOG_WARN("add enable_upgrade_mode config item failed", KR(ret));
-      } else {
-        const char *upgrade_stage_name = "_upgrade_stage";
-        obrpc::ObUpgradeStage stage = upgrade ?
-                                      obrpc::OB_UPGRADE_STAGE_PREUPGRADE :
-                                      obrpc::OB_UPGRADE_STAGE_NONE;
-        if (OB_FAIL(item.name_.assign(upgrade_stage_name))) {
-          LOG_WARN("assign _upgrade_stage config name failed", KR(ret), K(upgrade));
-        } else if (OB_FAIL(item.value_.assign(obrpc::get_upgrade_stage_str(stage)))) {
-          LOG_WARN("assign _upgrade_stage config value failed", KR(ret), K(stage), K(upgrade));
-        } else if (OB_FAIL(set_config_arg.items_.push_back(item))) {
-          LOG_WARN("add _upgrade_stage config item failed", KR(ret), K(stage), K(upgrade));
-        }
-      }
-      if (OB_FAIL(ret)) {
-      } else if (OB_FAIL(admin_set_config.execute(set_config_arg))) {
-        LOG_WARN("execute set config failed", KR(ret));
-      } else {
-        LOG_INFO("change upgrade parameters",
-                 "min_observer_version", min_server_version,
-                 "enable_upgrade_mode", upgrade,
-                 "in_major_version_upgrade_mode", GCONF.in_major_version_upgrade_mode());
+        LOG_WARN("add _upgrade_stage config item failed", KR(ret), K(stage), K(upgrade));
       }
     }
+    if (FAILEDx(admin_set_config.execute(set_config_arg))) {
+      LOG_WARN("execute set config failed", KR(ret));
+    } else {
+      CLUSTER_EVENT_SYNC_ADD("UPGRADE",
+                             upgrade ? "BEGIN_UPGRADE" : "END_UPGRADE",
+                             "cluster_version", min_server_version);
+      LOG_INFO("change upgrade parameters",
+               "enable_upgrade_mode", upgrade,
+               "in_major_version_upgrade_mode", GCONF.in_major_version_upgrade_mode());
+
+    }
   }
   return ret;
 }
@@ -1419,24 +1413,57 @@ int ObAdminUpgradeCmd::execute(const Bool &upgrade)
 int ObAdminRollingUpgradeCmd::execute(const obrpc::ObAdminRollingUpgradeArg &arg)
 {
   int ret = OB_SUCCESS;
-  HEAP_VAR(ObAdminSetConfigItem, upgrade_stage_item) {
+  HEAP_VAR(ObAdminSetConfigItem, item) {
     obrpc::ObAdminSetConfigArg set_config_arg;
     set_config_arg.is_inner_ = true;
     const char *upgrade_stage_name = "_upgrade_stage";
     ObAdminSetConfig admin_set_config(ctx_);
+    char ori_min_server_version[OB_SERVER_VERSION_LENGTH] = {'\0'};
+    char min_server_version[OB_SERVER_VERSION_LENGTH] = {'\0'};
+    uint64_t ori_cluster_version = GET_MIN_CLUSTER_VERSION();
 
     if (!arg.is_valid()) {
       ret = OB_INVALID_ARGUMENT;
       LOG_WARN("invalid arg", KR(ret), K(arg));
-    } else if (OB_FAIL(upgrade_stage_item.name_.assign(upgrade_stage_name))) {
+    } else if (OB_INVALID_INDEX == ObClusterVersion::print_version_str(
+               ori_min_server_version, OB_SERVER_VERSION_LENGTH, ori_cluster_version)) {
+       ret = OB_INVALID_ARGUMENT;
+       LOG_WARN("fail to print version str", KR(ret), K(ori_cluster_version));
+    } else if (OB_FAIL(item.name_.assign(upgrade_stage_name))) {
       LOG_WARN("assign _upgrade_stage config name failed", KR(ret), K(arg));
-    } else if (OB_FAIL(upgrade_stage_item.value_.assign(obrpc::get_upgrade_stage_str(arg.stage_)))) {
+    } else if (OB_FAIL(item.value_.assign(obrpc::get_upgrade_stage_str(arg.stage_)))) {
       LOG_WARN("assign _upgrade_stage config value failed", KR(ret), K(arg));
-    } else if (OB_FAIL(set_config_arg.items_.push_back(upgrade_stage_item))) {
+    } else if (OB_FAIL(set_config_arg.items_.push_back(item))) {
       LOG_WARN("add _upgrade_stage config item failed", KR(ret), K(arg));
-    } else if (OB_FAIL(admin_set_config.execute(set_config_arg))) {
+    } else if (obrpc::OB_UPGRADE_STAGE_POSTUPGRADE == arg.stage_) {
+      // end rolling upgrade, should raise min_observer_version
+      const char *min_obs_version_name = "min_observer_version";
+      if (OB_ISNULL(ctx_.server_mgr_)) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_WARN("server_mgr is null", KR(ret));
+      } else if (OB_FAIL(ctx_.server_mgr_->get_min_server_version(min_server_version))) {
+        LOG_WARN("failed to get the min server version", KR(ret));
+      } else if (OB_FAIL(item.name_.assign(min_obs_version_name))) {
+        LOG_WARN("assign min_observer_version config name failed",
+                 KR(ret), K(min_obs_version_name));
+      } else if (OB_FAIL(item.value_.assign(min_server_version))) {
+        LOG_WARN("assign min_observer_version config value failed",
+                 KR(ret), K(min_server_version));
+      } else if (OB_FAIL(set_config_arg.items_.push_back(item))) {
+        LOG_WARN("add min_observer_version config item failed", KR(ret), K(item));
+      }
+    }
+    if (FAILEDx(admin_set_config.execute(set_config_arg))) {
       LOG_WARN("execute set config failed", KR(ret));
     } else {
+      if (obrpc::OB_UPGRADE_STAGE_POSTUPGRADE != arg.stage_) {
+        CLUSTER_EVENT_SYNC_ADD("UPGRADE", "BEGIN_ROLLING_UPGRADE",
+                               "cluster_version", ori_min_server_version);
+      } else {
+        CLUSTER_EVENT_SYNC_ADD("UPGRADE", "END_ROLLING_UPGRADE",
+                               "cluster_version", min_server_version,
+                               "ori_cluster_version", ori_min_server_version);
+      }
       LOG_INFO("change upgrade parameters", KR(ret), "_upgrade_stage", arg.stage_);
     }
   }

src/rootserver/ob_system_admin_util.h

@@ -371,8 +371,8 @@ public:
   virtual ~ObAdminUpgradeVirtualSchema() {}
 
   int execute();
+  int execute(const uint64_t tenant_id, int64_t &upgrade_cnt);
 private:
-  int execute_(const uint64_t tenant_id, int64_t &upgrade_cnt);
   int upgrade_(const uint64_t tenant_id, share::schema::ObTableSchema &table);
 private:
   DISALLOW_COPY_AND_ASSIGN(ObAdminUpgradeVirtualSchema);

src/rootserver/ob_unit_manager.cpp

@@ -11466,6 +11466,25 @@ int ObUnitManager::get_unit_infos(const common::ObIArray<share::ObResourcePoolNa
   return ret;
 }
 
+int ObUnitManager::get_servers_by_pools(
+    const common::ObIArray<share::ObResourcePoolName> &pools,
+    common::ObIArray<ObAddr> &addrs)
+{
+  int ret = OB_SUCCESS;
+  addrs.reset();
+  ObArray<share::ObUnitInfo> unit_infos;
+  if (OB_FAIL(get_unit_infos(pools, unit_infos))) {
+    LOG_WARN("fail to get unit infos", KR(ret), K(pools));
+  }
+  for (int64_t i = 0; OB_SUCC(ret) && i < unit_infos.count(); i++) {
+    const share::ObUnitInfo &unit_info = unit_infos.at(i);
+    if (OB_FAIL(addrs.push_back(unit_info.unit_.server_))) {
+      LOG_WARN("fail to push back addr", KR(ret), K(unit_info));
+    }
+  } // end for
+  return ret;
+}
+
 int ObUnitManager::inner_get_active_unit_infos_of_tenant(const ObTenantSchema &tenant_schema,
                                                   ObIArray<ObUnitInfo> &unit_info)
 {

src/rootserver/ob_unit_manager.h

@@ -378,6 +378,8 @@ public:
                                                    common::ObIArray<share::ObUnitInfo> &unit_info) const;
   virtual int get_unit_infos(const common::ObIArray<share::ObResourcePoolName> &pools,
                              common::ObIArray<share::ObUnitInfo> &unit_infos);
+  virtual int get_servers_by_pools(const common::ObIArray<share::ObResourcePoolName> &pools,
+                                   common::ObIArray<ObAddr> &addrs);
 
   virtual int get_unit_ids(common::ObIArray<uint64_t> &unit_ids) const;
   virtual int get_logonly_unit_ids(common::ObIArray<uint64_t> &unit_ids) const;

src/rootserver/ob_upgrade_executor.h

@@ -19,6 +19,8 @@
 #include "share/schema/ob_multi_version_schema_service.h"
 #include "share/schema/ob_schema_getter_guard.h"
 #include "share/ob_rpc_struct.h"
+#include "rootserver/ob_rs_job_table_operator.h"
+#include "rootserver/ob_root_inspection.h"
 
 namespace oceanbase
 {
@@ -29,19 +31,17 @@ class ObUpgradeExecutor;
 class ObUpgradeTask: public share::ObAsyncTask
 {
 public:
-  explicit ObUpgradeTask(ObUpgradeExecutor &upgrade_executor,
-                         const obrpc::ObUpgradeJobArg::Action action,
-                         const int64_t version)
-           : upgrade_executor_(&upgrade_executor), action_(action), version_(version)
+  explicit ObUpgradeTask(ObUpgradeExecutor &upgrade_executor)
+           : upgrade_executor_(&upgrade_executor), arg_()
   {}
   virtual ~ObUpgradeTask() {}
+  int init(const obrpc::ObUpgradeJobArg &arg);
   virtual int64_t get_deep_copy_size() const;
   share::ObAsyncTask *deep_copy(char *buf, const int64_t buf_size) const;
   virtual int process();
 private:
   ObUpgradeExecutor *upgrade_executor_;
-  obrpc::ObUpgradeJobArg::Action action_;
-  int64_t version_;
+  obrpc::ObUpgradeJobArg arg_;
 };
 
 class ObUpgradeExecutor : public share::ObCheckStopProvider
@@ -50,12 +50,12 @@ public:
   ObUpgradeExecutor();
   ~ObUpgradeExecutor() {}
   int init(share::schema::ObMultiVersionSchemaService &schema_service,
+           rootserver::ObRootInspection &root_inspection,
            common::ObMySQLProxy &sql_proxy,
            obrpc::ObSrvRpcProxy &rpc_proxy,
            obrpc::ObCommonRpcProxy &common_proxy);
 
-  int execute(const obrpc::ObUpgradeJobArg::Action action,
-              const int64_t version);
+  int execute(const obrpc::ObUpgradeJobArg &arg);
   int can_execute();
   int check_stop() const;
   bool check_execute() const;
@@ -66,19 +66,41 @@ private:
   int check_inner_stat_() const;
   int set_execute_mark_();
 
-  int run_upgrade_post_job_(const int64_t version);
-  int run_upgrade_system_variable_job_();
-  int run_upgrade_system_table_job_();
+  int run_upgrade_post_job_(const common::ObIArray<uint64_t> &tenant_ids,
+                            const int64_t version);
 
+  /*-----upgrade all cmd----*/
+  int run_upgrade_begin_action_(const common::ObIArray<uint64_t> &tenant_ids);
+  int run_upgrade_system_variable_job_(const common::ObIArray<uint64_t> &tenant_ids);
+  int run_upgrade_system_table_job_(const common::ObIArray<uint64_t> &tenant_ids);
+  int run_upgrade_virtual_schema_job_(const common::ObIArray<uint64_t> &tenant_ids);
+  int run_upgrade_system_package_job_();
+  int run_upgrade_all_post_action_(const common::ObIArray<uint64_t> &tenant_ids);
+  int run_upgrade_inspection_job_(const common::ObIArray<uint64_t> &tenant_ids);
+  int run_upgrade_end_action_(const common::ObIArray<uint64_t> &tenant_ids);
+
+  int run_upgrade_all_(const common::ObIArray<uint64_t> &tenant_ids);
+  /*-----upgrade all cmd----*/
+
+  int run_upgrade_begin_action_(const uint64_t tenant_id);
   int upgrade_system_table_(const uint64_t tenant_id);
   int check_table_schema_(const uint64_t tenant_id,
                           const share::schema::ObTableSchema &hard_code_table);
+  int upgrade_mysql_system_package_job_();
+  int upgrade_oracle_system_package_job_();
+  int run_upgrade_all_post_action_(const uint64_t tenant_id);
+  int run_upgrade_end_action_(const uint64_t tenant_id);
 
-  int check_schema_sync_();
+  int check_schema_sync_(const uint64_t tenant_id);
   int check_schema_sync_(
       obrpc::ObTenantSchemaVersions &primary_schema_versions,
       obrpc::ObTenantSchemaVersions &standby_schema_versions,
       bool &schema_sync);
+  int construct_tenant_ids_(
+      const common::ObIArray<uint64_t> &src_tenant_ids,
+      common::ObIArray<uint64_t> &dst_tenant_ids);
+  rootserver::ObRsJobType convert_to_job_type_(
+      const obrpc::ObUpgradeJobArg::Action &action);
 private:
   bool inited_;
   bool stopped_;
@@ -88,6 +110,7 @@ private:
   obrpc::ObSrvRpcProxy *rpc_proxy_;
   obrpc::ObCommonRpcProxy *common_rpc_proxy_;
   share::schema::ObMultiVersionSchemaService *schema_service_;
+  rootserver::ObRootInspection *root_inspection_;
   share::ObUpgradeProcesserSet upgrade_processors_;
   DISALLOW_COPY_AND_ASSIGN(ObUpgradeExecutor);
 };

src/rootserver/ob_upgrade_storage_format_version_executor.cpp

@@ -303,7 +303,7 @@ int ObUpgradeStorageFormatVersionExecutor::check_schema_sync()
     while (OB_SUCC(ret)) {
       if (OB_FAIL(check_stop())) {
         LOG_WARN("executor is stop", K(ret));
-      } else if (OB_FAIL(ObUpgradeUtils::check_schema_sync(is_sync))) {
+      } else if (OB_FAIL(ObUpgradeUtils::check_schema_sync(OB_INVALID_TENANT_ID, is_sync))) {
         LOG_WARN("fail to check schema sync", K(ret));
       } else if (is_sync) {
         break;

src/rootserver/restore/ob_restore_scheduler.cpp

@@ -358,6 +358,8 @@ int ObRestoreService::fill_create_tenant_arg(
      arg.tenant_schema_.set_compatibility_mode(mode);
      arg.if_not_exist_ = false;
      arg.is_restore_ = true;
+     //TODO(chongrong.th): should change to tenant's data version
+     arg.compatible_version_ = job.get_source_cluster_version();
      if (OB_FAIL(assign_pool_list(pool_list.ptr(), arg.pool_list_))) {
        LOG_WARN("fail to get pool list", K(ret), K(pool_list));
      }

src/share/CMakeLists.txt

@@ -177,7 +177,9 @@ ob_set_subtarget(ob_share common
   ob_admin_dump_helper.cpp
   ob_inner_kv_table_operator.cpp
   ob_inner_table_operator.cpp
+  ob_standby_upgrade.cpp
   ob_primary_standby_service.cpp
+  ob_cluster_event_history_table_operator.cpp
   scn.cpp
 )
 

src/share/config/ob_common_config.cpp

@@ -223,7 +223,7 @@ OB_DEF_DESERIALIZE(ObCommonConfig)
     } else {
       MEMSET(copy_buf, '\0', data_len + 1);
       MEMCPY(copy_buf, buf + pos, data_len);
-      if (OB_FAIL(add_extra_config(copy_buf))) {
+      if (OB_FAIL(ObCommonConfig::add_extra_config(copy_buf))) {
         LOG_ERROR("Read server config failed", K(ret));
       }
 

src/share/config/ob_config.cpp

@@ -15,6 +15,8 @@
 #include <cstring>
 #include <ctype.h>
 #include "common/ob_smart_var.h"
+#include "share/ob_cluster_version.h"
+#include "share/ob_task_define.h"
 
 using namespace oceanbase::share;
 namespace oceanbase
@@ -965,5 +967,159 @@ int ObConfigLogArchiveOptionsItem::format_option_str(const char *src, int64_t sr
   return ret;
 }
 
+ObConfigVersionItem::ObConfigVersionItem(ObConfigContainer *container,
+                                         Scope::ScopeInfo scope_info,
+                                         const char *name,
+                                         const char *def,
+                                         const char *range,
+                                         const char *info,
+                                         const ObParameterAttr attr)
+{
+  if (OB_LIKELY(NULL != container)) {
+    container->set_refactored(ObConfigStringKey(name), this, 1);
+  }
+  init(scope_info, name, def, range, info, attr);
+}
+
+ObConfigVersionItem::ObConfigVersionItem(ObConfigContainer *container,
+                                         Scope::ScopeInfo scope_info,
+                                         const char *name,
+                                         const char *def,
+                                         const char *info,
+                                         const ObParameterAttr attr)
+{
+  if (OB_LIKELY(NULL != container)) {
+    container->set_refactored(ObConfigStringKey(name), this, 1);
+  }
+  init(scope_info, name, def, "", info, attr);
+}
+
+bool ObConfigVersionItem::set(const char *str)
+{
+  int64_t old_value = value_;
+  bool value_update = value_updated();
+  bool valid = ObConfigIntegralItem::set(str);
+  if (valid && value_update && old_value > value_) {
+    OB_LOG(ERROR, "Attention!!! data version is retrogressive", K(old_value), K_(value));
+  }
+  if (old_value != value_) {
+    ObTaskController::get().allow_next_syslog();
+    OB_LOG(INFO, "Config data version changed", K(old_value), K_(value), K(value_update), K(valid));
+  }
+  return valid;
+}
+
+int64_t ObConfigVersionItem::parse(const char *str, bool &valid) const
+{
+  int ret = OB_SUCCESS;
+  uint64_t version = 0;
+  if (OB_FAIL(ObClusterVersion::get_version(str, version))) {
+    OB_LOG(ERROR, "parse version failed", KR(ret), "name", name(), K(str));
+  }
+  valid = OB_SUCC(ret);
+  return static_cast<int64_t>(version);
+}
+
+ObConfigVersionItem &ObConfigVersionItem::operator = (int64_t value)
+{
+  char buf[64] = {0};
+  (void) snprintf(buf, sizeof(buf), "%ld", value);
+  if (!set_value(buf)) {
+    OB_LOG(WARN, "obconfig version item set value failed");
+  }
+  return *this;
+}
+
+void ObConfigPairs::reset()
+{
+  tenant_id_ = OB_INVALID_TENANT_ID;
+  config_array_.reset();
+  allocator_.clear();
+}
+
+int ObConfigPairs::assign(const ObConfigPairs &other)
+{
+  int ret = OB_SUCCESS;
+  if (this != &other) {
+    reset();
+    int64_t array_cnt = other.config_array_.count();
+    if (OB_FAIL(config_array_.reserve(array_cnt))) {
+      OB_LOG(WARN, "fail to reserve array", KR(ret), K(array_cnt));
+    } else {
+      tenant_id_ = other.tenant_id_;
+    }
+    ObConfigPair pair;
+    bool c_like_str = true;
+    for (int64_t i = 0; OB_SUCC(ret) && i < array_cnt; i++) {
+      const ObConfigPair &other_pair = other.config_array_.at(i);
+      if (OB_FAIL(ob_write_string(allocator_, other_pair.key_, pair.key_, c_like_str))) {
+        OB_LOG(WARN, "fail to write string", KR(ret), K(other));
+      } else if (OB_FAIL(ob_write_string(allocator_, other_pair.value_, pair.value_, c_like_str))) {
+        OB_LOG(WARN, "fail to write string", KR(ret), K(other));
+      } else if (OB_FAIL(config_array_.push_back(pair))) {
+        OB_LOG(WARN, "fail to push back array", KR(ret), K(pair));
+      }
+    } // end for
+  }
+  return ret;
+}
+
+int ObConfigPairs::add_config(const ObString &key, const ObString &value)
+{
+  int ret = OB_SUCCESS;
+  for (int64_t i = 0; OB_SUCC(ret) && i < config_array_.count(); i++) {
+    const ObConfigPair &pair = config_array_.at(i);
+    if (0 == key.case_compare(pair.key_)) {
+      ret = OB_ENTRY_EXIST;
+      OB_LOG(WARN, "config already exist", KR(ret), K(key), K(value), K(pair));
+    }
+  } // end for
+
+  ObConfigPair tmp_pair;
+  bool c_like_str = true;
+  if (FAILEDx(ob_write_string(allocator_, key, tmp_pair.key_, c_like_str))) {
+    OB_LOG(WARN, "fail to write string", KR(ret), K(key));
+  } else if (OB_FAIL(ob_write_string(allocator_, value, tmp_pair.value_, c_like_str))) {
+    OB_LOG(WARN, "fail to write string", KR(ret), K(value));
+  } else if (OB_FAIL(config_array_.push_back(tmp_pair))) {
+    OB_LOG(WARN, "fail to push back array", KR(ret), K(tmp_pair));
+  }
+  return ret;
+}
+
+int ObConfigPairs::get_config_str(char *buf, const int64_t length) const
+{
+  int ret = OB_SUCCESS;
+  int64_t array_cnt = config_array_.count();
+  if (OB_UNLIKELY(
+      array_cnt <= 0
+      || OB_ISNULL(buf)
+      || length <= 0)) {
+    ret = OB_INVALID_ARGUMENT;
+    OB_LOG(WARN, "invalid arg", KR(ret), K(array_cnt), KP(buf), K(length));
+  }
+  int64_t pos = 0;
+  for (int64_t i = 0; OB_SUCC(ret) && i < array_cnt; i++) {
+    const ObConfigPair &pair = config_array_.at(i);
+    if (OB_FAIL(databuff_printf(buf, length, pos, "%s%s=%s",
+                (0 == pos ? "" : ","),
+                pair.key_.ptr(), pair.value_.ptr()))) {
+      OB_LOG(WARN, "fail to print pair", KR(ret), K(pair));
+    }
+  } // end for
+  return ret;
+}
+
+int64_t ObConfigPairs::get_config_str_length() const
+{
+  int64_t length = config_array_.count() * 2; // reserved for some characters
+  for (int64_t i = 0; i < config_array_.count(); i++) {
+    length += config_array_.at(i).key_.length();
+    length += config_array_.at(i).value_.length();
+  } // end for
+  return length;
+
+}
+
 } // end of namespace common
 } // end of namespace oceanbase

src/share/config/ob_config.h

@@ -45,6 +45,7 @@ enum ObConfigItemType{
   OB_CONF_ITEM_TYPE_MOMENT = 8,
   OB_CONF_ITEM_TYPE_CAPACITY = 9,
   OB_CONF_ITEM_TYPE_LOGARCHIVEOPT = 10,
+  OB_CONF_ITEM_TYPE_VERSION = 11,
 };
 
 class ObConfigItem
@@ -395,7 +396,7 @@ public:
 
 protected:
   //use current value to do input operation
-  bool set(const char *str);
+  virtual bool set(const char *str);
   virtual int64_t parse(const char *str, bool &valid) const = 0;
 
 private:
@@ -816,6 +817,85 @@ private:
   DISALLOW_COPY_AND_ASSIGN(ObConfigLogArchiveOptionsItem);
 };
 
+class ObConfigVersionItem
+  : public ObConfigIntegralItem
+{
+public:
+  ObConfigVersionItem(ObConfigContainer *container,
+                       Scope::ScopeInfo scope_info,
+                       const char *name,
+                       const char *def,
+                       const char *range,
+                       const char *info,
+                       const ObParameterAttr attr = ObParameterAttr());
+  ObConfigVersionItem(ObConfigContainer *container,
+                      Scope::ScopeInfo scope_info,
+                      const char *name,
+                      const char *def,
+                      const char *info,
+                      const ObParameterAttr attr = ObParameterAttr());
+  virtual ~ObConfigVersionItem() {}
+
+  virtual ObConfigItemType get_config_item_type() const {
+    return ObConfigItemType::OB_CONF_ITEM_TYPE_VERSION;
+  }
+  ObConfigVersionItem &operator = (int64_t value);
+
+protected:
+  virtual bool set(const char *str) override;
+  virtual int64_t parse(const char *str, bool &valid) const override;
+
+private:
+  uint64_t value_;
+  DISALLOW_COPY_AND_ASSIGN(ObConfigVersionItem);
+};
+
+
+class ObConfigPairs
+{
+public:
+
+struct ObConfigPair {
+public:
+  ObConfigPair()
+    : key_(), value_()
+  {}
+  ~ObConfigPair() {}
+  TO_STRING_KV(K_(key), K_(value));
+public:
+  ObString key_;
+  ObString value_;
+};
+
+public:
+  ObConfigPairs()
+    : tenant_id_(common::OB_INVALID_TENANT_ID),
+      allocator_(),
+      config_array_()
+  {}
+  ~ObConfigPairs() {}
+  void init(const uint64_t tenant_id) { tenant_id_ = tenant_id; }
+  bool is_valid() const
+  {
+    return OB_INVALID_TENANT_ID != tenant_id_
+           && config_array_.count() > 0;
+  }
+  void reset();
+  int assign(const ObConfigPairs &other);
+
+  uint64_t get_tenant_id() const { return tenant_id_; }
+  const common::ObSArray<ObConfigPair> &get_configs() const { return config_array_; }
+  int get_config_str(char *buf, const int64_t length) const;
+  int64_t get_config_str_length() const;
+  int add_config(const ObString &key, const ObString &value);
+
+  TO_STRING_KV(K_(tenant_id), K_(config_array));
+private:
+  uint64_t tenant_id_;
+  ObArenaAllocator allocator_;
+  common::ObSArray<ObConfigPair> config_array_;
+};
+
 } // namespace common
 } // namespace oceanbase
 

src/share/config/ob_server_config.cpp

@@ -78,6 +78,11 @@ bool ObServerConfig::in_upgrade_mode() const
   return bret;
 }
 
+bool ObServerConfig::in_dbupgrade_stage() const
+{
+  return obrpc::OB_UPGRADE_STAGE_DBUPGRADE == GCTX.get_upgrade_stage();
+}
+
 int ObServerConfig::read_config()
 {
   int ret = OB_SUCCESS;

src/share/config/ob_server_config.h

@@ -106,6 +106,7 @@ public:
   bool in_major_version_upgrade_mode() const { return in_upgrade_mode() && is_major_version_upgrade(); }
   bool enable_new_major() const {  return true; }
   bool in_upgrade_mode() const;
+  bool in_dbupgrade_stage() const;
   bool is_valid() const { return  system_config_!= NULL; };
   int64_t get_current_version() { return system_config_->get_version(); }
 

src/share/inner_table/ob_inner_table_schema.12351_12400.cpp

@@ -0,0 +1,180 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX SHARE_SCHEMA
+#include "ob_inner_table_schema.h"
+
+#include "share/schema/ob_schema_macro_define.h"
+#include "share/schema/ob_schema_service_sql_impl.h"
+#include "share/schema/ob_table_schema.h"
+
+namespace oceanbase
+{
+using namespace share::schema;
+using namespace common;
+namespace share
+{
+
+int ObInnerTableSchema::all_virtual_core_table_schema(ObTableSchema &table_schema)
+{
+  int ret = OB_SUCCESS;
+  uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1;
+
+  //generated fields:
+  table_schema.set_tenant_id(OB_SYS_TENANT_ID);
+  table_schema.set_tablegroup_id(OB_INVALID_ID);
+  table_schema.set_database_id(OB_SYS_DATABASE_ID);
+  table_schema.set_table_id(OB_ALL_VIRTUAL_CORE_TABLE_TID);
+  table_schema.set_rowkey_split_pos(0);
+  table_schema.set_is_use_bloomfilter(false);
+  table_schema.set_progressive_merge_num(0);
+  table_schema.set_rowkey_column_num(4);
+  table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK);
+  table_schema.set_table_type(VIRTUAL_TABLE);
+  table_schema.set_index_type(INDEX_TYPE_IS_NOT);
+  table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL);
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_CORE_TABLE_TNAME))) {
+      LOG_ERROR("fail to set table_name", K(ret));
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) {
+      LOG_ERROR("fail to set compress_func_name", K(ret));
+    }
+  }
+  table_schema.set_part_level(PARTITION_LEVEL_ZERO);
+  table_schema.set_charset_type(ObCharset::get_default_charset());
+  table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("tenant_id", //column_name
+      ++column_id, //column_id
+      1, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("table_name", //column_name
+      ++column_id, //column_id
+      2, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      OB_MAX_CORE_TALBE_NAME_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("row_id", //column_name
+      ++column_id, //column_id
+      3, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("column_name", //column_name
+      ++column_id, //column_id
+      4, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      OB_MAX_COLUMN_NAME_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA_TS("gmt_create", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObTimestampType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(ObPreciseDateTime), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      false); //is_on_update_for_timestamp
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA_TS("gmt_modified", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObTimestampType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(ObPreciseDateTime), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      false); //is_on_update_for_timestamp
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("column_value", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      OB_OLD_MAX_VARCHAR_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      true, //is_nullable
+      false); //is_autoincrement
+  }
+  table_schema.set_index_using_type(USING_BTREE);
+  table_schema.set_row_store_type(ENCODING_ROW_STORE);
+  table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL);
+  table_schema.set_progressive_merge_round(1);
+  table_schema.set_storage_format_version(3);
+  table_schema.set_tablet_id(0);
+
+  table_schema.set_max_used_column_id(column_id);
+  return ret;
+}
+
+
+} // end namespace share
+} // end namespace oceanbase

src/share/inner_table/ob_inner_table_schema.21301_21350.cpp

@@ -1281,6 +1281,56 @@ int ObInnerTableSchema::cdb_ob_database_privilege_schema(ObTableSchema &table_sc
   return ret;
 }
 
+int ObInnerTableSchema::dba_ob_cluster_event_history_schema(ObTableSchema &table_schema)
+{
+  int ret = OB_SUCCESS;
+  uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1;
+
+  //generated fields:
+  table_schema.set_tenant_id(OB_SYS_TENANT_ID);
+  table_schema.set_tablegroup_id(OB_INVALID_ID);
+  table_schema.set_database_id(OB_SYS_DATABASE_ID);
+  table_schema.set_table_id(OB_DBA_OB_CLUSTER_EVENT_HISTORY_TID);
+  table_schema.set_rowkey_split_pos(0);
+  table_schema.set_is_use_bloomfilter(false);
+  table_schema.set_progressive_merge_num(0);
+  table_schema.set_rowkey_column_num(0);
+  table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK);
+  table_schema.set_table_type(SYSTEM_VIEW);
+  table_schema.set_index_type(INDEX_TYPE_IS_NOT);
+  table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL);
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_table_name(OB_DBA_OB_CLUSTER_EVENT_HISTORY_TNAME))) {
+      LOG_ERROR("fail to set table_name", K(ret));
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) {
+      LOG_ERROR("fail to set compress_func_name", K(ret));
+    }
+  }
+  table_schema.set_part_level(PARTITION_LEVEL_ZERO);
+  table_schema.set_charset_type(ObCharset::get_default_charset());
+  table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT   gmt_create AS `TIMESTAMP`,   CAST(MODULE AS CHAR(256)) MODULE,   CAST(EVENT AS CHAR(256)) EVENT,   CAST(NAME1 AS CHAR(256)) NAME1,   CAST(VALUE1 AS CHAR(4096)) VALUE1,   CAST(NAME2 AS CHAR(256)) NAME2,   CAST(VALUE2 AS CHAR(4096)) VALUE2,   CAST(NAME3 AS CHAR(256)) NAME3,   CAST(VALUE3 AS CHAR(4096)) VALUE3,   CAST(NAME4 AS CHAR(256)) NAME4,   CAST(VALUE4 AS CHAR(4096)) VALUE4,   CAST(NAME5 AS CHAR(256)) NAME5,   CAST(VALUE5 AS CHAR(4096)) VALUE5,   CAST(NAME6 AS CHAR(256)) NAME6,   CAST(VALUE6 AS CHAR(4096)) VALUE6,   CAST(EXTRA_INFO AS CHAR(4096)) EXTRA_INFO FROM oceanbase.__all_cluster_event_history )__"))) {
+      LOG_ERROR("fail to set view_definition", K(ret));
+    }
+  }
+  table_schema.set_index_using_type(USING_BTREE);
+  table_schema.set_row_store_type(ENCODING_ROW_STORE);
+  table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL);
+  table_schema.set_progressive_merge_round(1);
+  table_schema.set_storage_format_version(3);
+  table_schema.set_tablet_id(0);
+
+  table_schema.set_max_used_column_id(column_id);
+  return ret;
+}
+
 
 } // end namespace share
 } // end namespace oceanbase

src/share/inner_table/ob_inner_table_schema.401_450.cpp

@@ -2856,6 +2856,352 @@ int ObInnerTableSchema::all_column_checksum_error_info_schema(ObTableSchema &tab
   return ret;
 }
 
+int ObInnerTableSchema::all_cluster_event_history_schema(ObTableSchema &table_schema)
+{
+  int ret = OB_SUCCESS;
+  uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1;
+
+  //generated fields:
+  table_schema.set_tenant_id(OB_SYS_TENANT_ID);
+  table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID);
+  table_schema.set_database_id(OB_SYS_DATABASE_ID);
+  table_schema.set_table_id(OB_ALL_CLUSTER_EVENT_HISTORY_TID);
+  table_schema.set_rowkey_split_pos(0);
+  table_schema.set_is_use_bloomfilter(false);
+  table_schema.set_progressive_merge_num(0);
+  table_schema.set_rowkey_column_num(1);
+  table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK);
+  table_schema.set_table_type(SYSTEM_TABLE);
+  table_schema.set_index_type(INDEX_TYPE_IS_NOT);
+  table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL);
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_table_name(OB_ALL_CLUSTER_EVENT_HISTORY_TNAME))) {
+      LOG_ERROR("fail to set table_name", K(ret));
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) {
+      LOG_ERROR("fail to set compress_func_name", K(ret));
+    }
+  }
+  table_schema.set_part_level(PARTITION_LEVEL_ZERO);
+  table_schema.set_charset_type(ObCharset::get_default_charset());
+  table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+
+  if (OB_SUCC(ret)) {
+    ObObj gmt_default;
+    ObObj gmt_default_null;
+
+    gmt_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG);
+    gmt_default_null.set_null();
+    ADD_COLUMN_SCHEMA_TS_T("gmt_create", //column_name
+      ++column_id, //column_id
+      1, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObTimestampType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      0, //column_length
+      -1, //column_precision
+      6, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      false, //is_on_update_for_timestamp
+      gmt_default_null,
+      gmt_default)
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("module", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      MAX_CLUSTER_EVENT_NAME_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("event", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      MAX_CLUSTER_EVENT_NAME_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ObObj name1_default;
+    name1_default.set_varchar(ObString::make_string(""));
+    ADD_COLUMN_SCHEMA_T("name1", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      MAX_CLUSTER_EVENT_NAME_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      name1_default,
+      name1_default); //default_value
+  }
+
+  if (OB_SUCC(ret)) {
+    ObObj value1_default;
+    value1_default.set_varchar(ObString::make_string(""));
+    ADD_COLUMN_SCHEMA_T("value1", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      MAX_CLUSTER_EVENT_VALUE_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      value1_default,
+      value1_default); //default_value
+  }
+
+  if (OB_SUCC(ret)) {
+    ObObj name2_default;
+    name2_default.set_varchar(ObString::make_string(""));
+    ADD_COLUMN_SCHEMA_T("name2", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      MAX_CLUSTER_EVENT_NAME_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      name2_default,
+      name2_default); //default_value
+  }
+
+  if (OB_SUCC(ret)) {
+    ObObj value2_default;
+    value2_default.set_varchar(ObString::make_string(""));
+    ADD_COLUMN_SCHEMA_T("value2", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      MAX_CLUSTER_EVENT_VALUE_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      value2_default,
+      value2_default); //default_value
+  }
+
+  if (OB_SUCC(ret)) {
+    ObObj name3_default;
+    name3_default.set_varchar(ObString::make_string(""));
+    ADD_COLUMN_SCHEMA_T("name3", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      MAX_CLUSTER_EVENT_NAME_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      name3_default,
+      name3_default); //default_value
+  }
+
+  if (OB_SUCC(ret)) {
+    ObObj value3_default;
+    value3_default.set_varchar(ObString::make_string(""));
+    ADD_COLUMN_SCHEMA_T("value3", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      MAX_CLUSTER_EVENT_VALUE_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      value3_default,
+      value3_default); //default_value
+  }
+
+  if (OB_SUCC(ret)) {
+    ObObj name4_default;
+    name4_default.set_varchar(ObString::make_string(""));
+    ADD_COLUMN_SCHEMA_T("name4", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      MAX_CLUSTER_EVENT_NAME_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      name4_default,
+      name4_default); //default_value
+  }
+
+  if (OB_SUCC(ret)) {
+    ObObj value4_default;
+    value4_default.set_varchar(ObString::make_string(""));
+    ADD_COLUMN_SCHEMA_T("value4", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      MAX_CLUSTER_EVENT_VALUE_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      value4_default,
+      value4_default); //default_value
+  }
+
+  if (OB_SUCC(ret)) {
+    ObObj name5_default;
+    name5_default.set_varchar(ObString::make_string(""));
+    ADD_COLUMN_SCHEMA_T("name5", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      MAX_CLUSTER_EVENT_NAME_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      name5_default,
+      name5_default); //default_value
+  }
+
+  if (OB_SUCC(ret)) {
+    ObObj value5_default;
+    value5_default.set_varchar(ObString::make_string(""));
+    ADD_COLUMN_SCHEMA_T("value5", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      MAX_CLUSTER_EVENT_VALUE_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      value5_default,
+      value5_default); //default_value
+  }
+
+  if (OB_SUCC(ret)) {
+    ObObj name6_default;
+    name6_default.set_varchar(ObString::make_string(""));
+    ADD_COLUMN_SCHEMA_T("name6", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      MAX_CLUSTER_EVENT_NAME_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      name6_default,
+      name6_default); //default_value
+  }
+
+  if (OB_SUCC(ret)) {
+    ObObj value6_default;
+    value6_default.set_varchar(ObString::make_string(""));
+    ADD_COLUMN_SCHEMA_T("value6", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      MAX_CLUSTER_EVENT_VALUE_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      value6_default,
+      value6_default); //default_value
+  }
+
+  if (OB_SUCC(ret)) {
+    ObObj extra_info_default;
+    extra_info_default.set_varchar(ObString::make_string(""));
+    ADD_COLUMN_SCHEMA_T("extra_info", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      MAX_CLUSTER_EVENT_VALUE_LENGTH, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false, //is_autoincrement
+      extra_info_default,
+      extra_info_default); //default_value
+  }
+  table_schema.set_index_using_type(USING_BTREE);
+  table_schema.set_row_store_type(ENCODING_ROW_STORE);
+  table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL);
+  table_schema.set_progressive_merge_round(1);
+  table_schema.set_storage_format_version(3);
+  table_schema.set_tablet_id(OB_ALL_CLUSTER_EVENT_HISTORY_TID);
+  table_schema.set_aux_lob_meta_tid(OB_ALL_CLUSTER_EVENT_HISTORY_AUX_LOB_META_TID);
+  table_schema.set_aux_lob_piece_tid(OB_ALL_CLUSTER_EVENT_HISTORY_AUX_LOB_PIECE_TID);
+
+  table_schema.set_max_used_column_id(column_id);
+  return ret;
+}
+
 
 } // end namespace share
 } // end namespace oceanbase

src/share/inner_table/ob_inner_table_schema.50401_50450.cpp

@@ -1779,6 +1779,141 @@ int ObInnerTableSchema::all_column_checksum_error_info_aux_lob_meta_schema(ObTab
   return ret;
 }
 
+int ObInnerTableSchema::all_cluster_event_history_aux_lob_meta_schema(ObTableSchema &table_schema)
+{
+  int ret = OB_SUCCESS;
+  uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1;
+
+  //generated fields:
+  table_schema.set_tenant_id(OB_SYS_TENANT_ID);
+  table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID);
+  table_schema.set_database_id(OB_SYS_DATABASE_ID);
+  table_schema.set_table_id(OB_ALL_CLUSTER_EVENT_HISTORY_AUX_LOB_META_TID);
+  table_schema.set_rowkey_split_pos(0);
+  table_schema.set_is_use_bloomfilter(false);
+  table_schema.set_progressive_merge_num(0);
+  table_schema.set_rowkey_column_num(2);
+  table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK);
+  table_schema.set_table_type(AUX_LOB_META);
+  table_schema.set_index_type(INDEX_TYPE_IS_NOT);
+  table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL);
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_table_name(OB_ALL_CLUSTER_EVENT_HISTORY_AUX_LOB_META_TNAME))) {
+      LOG_ERROR("fail to set table_name", K(ret));
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) {
+      LOG_ERROR("fail to set compress_func_name", K(ret));
+    }
+  }
+  table_schema.set_part_level(PARTITION_LEVEL_ZERO);
+  table_schema.set_charset_type(ObCharset::get_default_charset());
+  table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("lob_id", //column_name
+      ++column_id, //column_id
+      1, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_BINARY, //column_collation_type
+      16, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("seq_id", //column_name
+      ++column_id, //column_id
+      2, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_BINARY, //column_collation_type
+      8192, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("binary_len", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObUInt32Type, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(uint32_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("char_len", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObUInt32Type, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(uint32_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("piece_id", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObUInt64Type, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(uint64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("lob_data", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_BINARY, //column_collation_type
+      262144, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+  table_schema.set_index_using_type(USING_BTREE);
+  table_schema.set_row_store_type(ENCODING_ROW_STORE);
+  table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL);
+  table_schema.set_progressive_merge_round(1);
+  table_schema.set_storage_format_version(3);
+  table_schema.set_tablet_id(OB_ALL_CLUSTER_EVENT_HISTORY_AUX_LOB_META_TID);
+  table_schema.set_data_table_id(OB_ALL_CLUSTER_EVENT_HISTORY_TID);
+
+  table_schema.set_max_used_column_id(column_id);
+  return ret;
+}
+
 
 } // end namespace share
 } // end namespace oceanbase

src/share/inner_table/ob_inner_table_schema.60401_60450.cpp

@@ -1194,6 +1194,96 @@ int ObInnerTableSchema::all_column_checksum_error_info_aux_lob_piece_schema(ObTa
   return ret;
 }
 
+int ObInnerTableSchema::all_cluster_event_history_aux_lob_piece_schema(ObTableSchema &table_schema)
+{
+  int ret = OB_SUCCESS;
+  uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1;
+
+  //generated fields:
+  table_schema.set_tenant_id(OB_SYS_TENANT_ID);
+  table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID);
+  table_schema.set_database_id(OB_SYS_DATABASE_ID);
+  table_schema.set_table_id(OB_ALL_CLUSTER_EVENT_HISTORY_AUX_LOB_PIECE_TID);
+  table_schema.set_rowkey_split_pos(0);
+  table_schema.set_is_use_bloomfilter(false);
+  table_schema.set_progressive_merge_num(0);
+  table_schema.set_rowkey_column_num(1);
+  table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK);
+  table_schema.set_table_type(AUX_LOB_PIECE);
+  table_schema.set_index_type(INDEX_TYPE_IS_NOT);
+  table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL);
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_table_name(OB_ALL_CLUSTER_EVENT_HISTORY_AUX_LOB_PIECE_TNAME))) {
+      LOG_ERROR("fail to set table_name", K(ret));
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) {
+      LOG_ERROR("fail to set compress_func_name", K(ret));
+    }
+  }
+  table_schema.set_part_level(PARTITION_LEVEL_ZERO);
+  table_schema.set_charset_type(ObCharset::get_default_charset());
+  table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("piece_id", //column_name
+      ++column_id, //column_id
+      1, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObUInt64Type, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(uint64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("data_len", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObUInt32Type, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(uint32_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("lob_data", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_BINARY, //column_collation_type
+      32, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+  table_schema.set_index_using_type(USING_BTREE);
+  table_schema.set_row_store_type(ENCODING_ROW_STORE);
+  table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL);
+  table_schema.set_progressive_merge_round(1);
+  table_schema.set_storage_format_version(3);
+  table_schema.set_tablet_id(OB_ALL_CLUSTER_EVENT_HISTORY_AUX_LOB_PIECE_TID);
+  table_schema.set_data_table_id(OB_ALL_CLUSTER_EVENT_HISTORY_TID);
+
+  table_schema.set_max_used_column_id(column_id);
+  return ret;
+}
+
 
 } // end namespace share
 } // end namespace oceanbase

src/share/inner_table/ob_inner_table_schema.h

@@ -496,6 +496,7 @@ public:
   static int all_service_epoch_schema(share::schema::ObTableSchema &table_schema);
   static int all_spatial_reference_systems_schema(share::schema::ObTableSchema &table_schema);
   static int all_column_checksum_error_info_schema(share::schema::ObTableSchema &table_schema);
+  static int all_cluster_event_history_schema(share::schema::ObTableSchema &table_schema);
   static int tenant_virtual_all_table_schema(share::schema::ObTableSchema &table_schema);
   static int tenant_virtual_table_column_schema(share::schema::ObTableSchema &table_schema);
   static int tenant_virtual_table_index_schema(share::schema::ObTableSchema &table_schema);
@@ -836,6 +837,7 @@ public:
   static int all_virtual_schema_slot_schema(share::schema::ObTableSchema &table_schema);
   static int all_virtual_minor_freeze_info_schema(share::schema::ObTableSchema &table_schema);
   static int all_virtual_ha_diagnose_schema(share::schema::ObTableSchema &table_schema);
+  static int all_virtual_core_table_schema(share::schema::ObTableSchema &table_schema);
   static int all_virtual_sql_audit_ora_schema(share::schema::ObTableSchema &table_schema);
   static int all_virtual_plan_stat_ora_schema(share::schema::ObTableSchema &table_schema);
   static int all_virtual_plan_cache_plan_explain_ora_schema(share::schema::ObTableSchema &table_schema);
@@ -1291,6 +1293,7 @@ public:
   static int cdb_ob_users_schema(share::schema::ObTableSchema &table_schema);
   static int dba_ob_database_privilege_schema(share::schema::ObTableSchema &table_schema);
   static int cdb_ob_database_privilege_schema(share::schema::ObTableSchema &table_schema);
+  static int dba_ob_cluster_event_history_schema(share::schema::ObTableSchema &table_schema);
   static int dba_synonyms_schema(share::schema::ObTableSchema &table_schema);
   static int dba_objects_ora_schema(share::schema::ObTableSchema &table_schema);
   static int all_objects_schema(share::schema::ObTableSchema &table_schema);
@@ -1850,6 +1853,7 @@ public:
   static int all_service_epoch_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema);
   static int all_spatial_reference_systems_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema);
   static int all_column_checksum_error_info_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema);
+  static int all_cluster_event_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema);
   static int all_table_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema);
   static int all_column_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema);
   static int all_ddl_operation_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema);
@@ -2066,6 +2070,7 @@ public:
   static int all_service_epoch_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema);
   static int all_spatial_reference_systems_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema);
   static int all_column_checksum_error_info_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema);
+  static int all_cluster_event_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema);
   static int all_virtual_plan_cache_stat_all_virtual_plan_cache_stat_i1_schema(share::schema::ObTableSchema &table_schema);
   static int all_virtual_session_event_all_virtual_session_event_i1_schema(share::schema::ObTableSchema &table_schema);
   static int all_virtual_session_wait_all_virtual_session_wait_i1_schema(share::schema::ObTableSchema &table_schema);
@@ -2401,6 +2406,7 @@ const schema_create_func sys_table_schema_creators [] = {
   ObInnerTableSchema::all_service_epoch_schema,
   ObInnerTableSchema::all_spatial_reference_systems_schema,
   ObInnerTableSchema::all_column_checksum_error_info_schema,
+  ObInnerTableSchema::all_cluster_event_history_schema,
   NULL,};
 
 const schema_create_func virtual_table_schema_creators [] = {
@@ -2744,6 +2750,7 @@ const schema_create_func virtual_table_schema_creators [] = {
   ObInnerTableSchema::all_virtual_schema_slot_schema,
   ObInnerTableSchema::all_virtual_minor_freeze_info_schema,
   ObInnerTableSchema::all_virtual_ha_diagnose_schema,
+  ObInnerTableSchema::all_virtual_core_table_schema,
   ObInnerTableSchema::all_virtual_sql_audit_ora_schema,
   ObInnerTableSchema::all_virtual_plan_stat_ora_schema,
   ObInnerTableSchema::all_virtual_plan_cache_plan_explain_ora_schema,
@@ -3219,6 +3226,7 @@ const schema_create_func sys_view_schema_creators [] = {
   ObInnerTableSchema::cdb_ob_users_schema,
   ObInnerTableSchema::dba_ob_database_privilege_schema,
   ObInnerTableSchema::cdb_ob_database_privilege_schema,
+  ObInnerTableSchema::dba_ob_cluster_event_history_schema,
   ObInnerTableSchema::dba_synonyms_schema,
   ObInnerTableSchema::dba_objects_ora_schema,
   ObInnerTableSchema::all_objects_schema,
@@ -9148,6 +9156,14 @@ LOBMapping const lob_aux_table_mappings [] = {
     ObInnerTableSchema::all_column_checksum_error_info_aux_lob_piece_schema
   },
 
+  {
+    OB_ALL_CLUSTER_EVENT_HISTORY_TID,
+    OB_ALL_CLUSTER_EVENT_HISTORY_AUX_LOB_META_TID,
+    OB_ALL_CLUSTER_EVENT_HISTORY_AUX_LOB_PIECE_TID,
+    ObInnerTableSchema::all_cluster_event_history_aux_lob_meta_schema,
+    ObInnerTableSchema::all_cluster_event_history_aux_lob_piece_schema
+  },
+
 };
 
 static inline bool get_sys_table_lob_aux_table_id(const uint64_t tid, uint64_t& meta_tid, uint64_t& piece_tid)
@@ -9185,12 +9201,12 @@ static inline int get_sys_table_lob_aux_schema(const uint64_t tid,
 }
 
 const int64_t OB_CORE_TABLE_COUNT = 4;
-const int64_t OB_SYS_TABLE_COUNT = 213;
-const int64_t OB_VIRTUAL_TABLE_COUNT = 552;
-const int64_t OB_SYS_VIEW_COUNT = 603;
-const int64_t OB_SYS_TENANT_TABLE_COUNT = 1373;
+const int64_t OB_SYS_TABLE_COUNT = 214;
+const int64_t OB_VIRTUAL_TABLE_COUNT = 553;
+const int64_t OB_SYS_VIEW_COUNT = 604;
+const int64_t OB_SYS_TENANT_TABLE_COUNT = 1376;
 const int64_t OB_CORE_SCHEMA_VERSION = 1;
-const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 1376;
+const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 1379;
 
 } // end namespace share
 } // end namespace oceanbase

src/share/inner_table/ob_inner_table_schema.lob.cpp

@@ -21,7 +21,7 @@ inner_lob_map_t inner_lob_map;
 bool lob_mapping_init()
 {
   int ret = OB_SUCCESS;
-  if (OB_FAIL(inner_lob_map.create(216, ObModIds::OB_INNER_LOB_HASH_SET))) {
+  if (OB_FAIL(inner_lob_map.create(217, ObModIds::OB_INNER_LOB_HASH_SET))) {
     SERVER_LOG(WARN, "fail to create inner lob map", K(ret));
   } else {
     for (int64_t i = 0; OB_SUCC(ret) && i < ARRAYSIZEOF(lob_aux_table_mappings); ++i) {

src/share/inner_table/ob_inner_table_schema_constants.h

@@ -244,6 +244,7 @@ const uint64_t OB_ALL_LOG_RESTORE_SOURCE_TID = 409; // "__all_log_restore_source
 const uint64_t OB_ALL_SERVICE_EPOCH_TID = 412; // "__all_service_epoch"
 const uint64_t OB_ALL_SPATIAL_REFERENCE_SYSTEMS_TID = 413; // "__all_spatial_reference_systems"
 const uint64_t OB_ALL_COLUMN_CHECKSUM_ERROR_INFO_TID = 416; // "__all_column_checksum_error_info"
+const uint64_t OB_ALL_CLUSTER_EVENT_HISTORY_TID = 445; // "__all_cluster_event_history"
 const uint64_t OB_TENANT_VIRTUAL_ALL_TABLE_TID = 10001; // "__tenant_virtual_all_table"
 const uint64_t OB_TENANT_VIRTUAL_TABLE_COLUMN_TID = 10002; // "__tenant_virtual_table_column"
 const uint64_t OB_TENANT_VIRTUAL_TABLE_INDEX_TID = 10003; // "__tenant_virtual_table_index"
@@ -584,6 +585,7 @@ const uint64_t OB_ALL_VIRTUAL_SCHEMA_MEMORY_TID = 12336; // "__all_virtual_schem
 const uint64_t OB_ALL_VIRTUAL_SCHEMA_SLOT_TID = 12337; // "__all_virtual_schema_slot"
 const uint64_t OB_ALL_VIRTUAL_MINOR_FREEZE_INFO_TID = 12338; // "__all_virtual_minor_freeze_info"
 const uint64_t OB_ALL_VIRTUAL_HA_DIAGNOSE_TID = 12340; // "__all_virtual_ha_diagnose"
+const uint64_t OB_ALL_VIRTUAL_CORE_TABLE_TID = 12362; // "__all_virtual_core_table"
 const uint64_t OB_ALL_VIRTUAL_SQL_AUDIT_ORA_TID = 15009; // "ALL_VIRTUAL_SQL_AUDIT_ORA"
 const uint64_t OB_ALL_VIRTUAL_PLAN_STAT_ORA_TID = 15010; // "ALL_VIRTUAL_PLAN_STAT_ORA"
 const uint64_t OB_ALL_VIRTUAL_PLAN_CACHE_PLAN_EXPLAIN_ORA_TID = 15012; // "ALL_VIRTUAL_PLAN_CACHE_PLAN_EXPLAIN_ORA"
@@ -1039,6 +1041,7 @@ const uint64_t OB_DBA_OB_USERS_TID = 21336; // "DBA_OB_USERS"
 const uint64_t OB_CDB_OB_USERS_TID = 21337; // "CDB_OB_USERS"
 const uint64_t OB_DBA_OB_DATABASE_PRIVILEGE_TID = 21338; // "DBA_OB_DATABASE_PRIVILEGE"
 const uint64_t OB_CDB_OB_DATABASE_PRIVILEGE_TID = 21339; // "CDB_OB_DATABASE_PRIVILEGE"
+const uint64_t OB_DBA_OB_CLUSTER_EVENT_HISTORY_TID = 21345; // "DBA_OB_CLUSTER_EVENT_HISTORY"
 const uint64_t OB_DBA_SYNONYMS_TID = 25001; // "DBA_SYNONYMS"
 const uint64_t OB_DBA_OBJECTS_ORA_TID = 25002; // "DBA_OBJECTS_ORA"
 const uint64_t OB_ALL_OBJECTS_TID = 25003; // "ALL_OBJECTS"
@@ -1598,6 +1601,7 @@ const uint64_t OB_ALL_LOG_RESTORE_SOURCE_AUX_LOB_META_TID = 50409; // "__all_log
 const uint64_t OB_ALL_SERVICE_EPOCH_AUX_LOB_META_TID = 50412; // "__all_service_epoch_aux_lob_meta"
 const uint64_t OB_ALL_SPATIAL_REFERENCE_SYSTEMS_AUX_LOB_META_TID = 50413; // "__all_spatial_reference_systems_aux_lob_meta"
 const uint64_t OB_ALL_COLUMN_CHECKSUM_ERROR_INFO_AUX_LOB_META_TID = 50416; // "__all_column_checksum_error_info_aux_lob_meta"
+const uint64_t OB_ALL_CLUSTER_EVENT_HISTORY_AUX_LOB_META_TID = 50445; // "__all_cluster_event_history_aux_lob_meta"
 const uint64_t OB_ALL_TABLE_AUX_LOB_PIECE_TID = 60003; // "__all_table_aux_lob_piece"
 const uint64_t OB_ALL_COLUMN_AUX_LOB_PIECE_TID = 60004; // "__all_column_aux_lob_piece"
 const uint64_t OB_ALL_DDL_OPERATION_AUX_LOB_PIECE_TID = 60005; // "__all_ddl_operation_aux_lob_piece"
@@ -1814,6 +1818,7 @@ const uint64_t OB_ALL_LOG_RESTORE_SOURCE_AUX_LOB_PIECE_TID = 60409; // "__all_lo
 const uint64_t OB_ALL_SERVICE_EPOCH_AUX_LOB_PIECE_TID = 60412; // "__all_service_epoch_aux_lob_piece"
 const uint64_t OB_ALL_SPATIAL_REFERENCE_SYSTEMS_AUX_LOB_PIECE_TID = 60413; // "__all_spatial_reference_systems_aux_lob_piece"
 const uint64_t OB_ALL_COLUMN_CHECKSUM_ERROR_INFO_AUX_LOB_PIECE_TID = 60416; // "__all_column_checksum_error_info_aux_lob_piece"
+const uint64_t OB_ALL_CLUSTER_EVENT_HISTORY_AUX_LOB_PIECE_TID = 60445; // "__all_cluster_event_history_aux_lob_piece"
 const uint64_t OB_ALL_VIRTUAL_PLAN_CACHE_STAT_ALL_VIRTUAL_PLAN_CACHE_STAT_I1_TID = 14999; // "__all_virtual_plan_cache_stat"
 const uint64_t OB_ALL_VIRTUAL_SESSION_EVENT_ALL_VIRTUAL_SESSION_EVENT_I1_TID = 14998; // "__all_virtual_session_event"
 const uint64_t OB_ALL_VIRTUAL_SESSION_WAIT_ALL_VIRTUAL_SESSION_WAIT_I1_TID = 14997; // "__all_virtual_session_wait"
@@ -2136,6 +2141,7 @@ const char *const OB_ALL_LOG_RESTORE_SOURCE_TNAME = "__all_log_restore_source";
 const char *const OB_ALL_SERVICE_EPOCH_TNAME = "__all_service_epoch";
 const char *const OB_ALL_SPATIAL_REFERENCE_SYSTEMS_TNAME = "__all_spatial_reference_systems";
 const char *const OB_ALL_COLUMN_CHECKSUM_ERROR_INFO_TNAME = "__all_column_checksum_error_info";
+const char *const OB_ALL_CLUSTER_EVENT_HISTORY_TNAME = "__all_cluster_event_history";
 const char *const OB_TENANT_VIRTUAL_ALL_TABLE_TNAME = "__tenant_virtual_all_table";
 const char *const OB_TENANT_VIRTUAL_TABLE_COLUMN_TNAME = "__tenant_virtual_table_column";
 const char *const OB_TENANT_VIRTUAL_TABLE_INDEX_TNAME = "__tenant_virtual_table_index";
@@ -2476,6 +2482,7 @@ const char *const OB_ALL_VIRTUAL_SCHEMA_MEMORY_TNAME = "__all_virtual_schema_mem
 const char *const OB_ALL_VIRTUAL_SCHEMA_SLOT_TNAME = "__all_virtual_schema_slot";
 const char *const OB_ALL_VIRTUAL_MINOR_FREEZE_INFO_TNAME = "__all_virtual_minor_freeze_info";
 const char *const OB_ALL_VIRTUAL_HA_DIAGNOSE_TNAME = "__all_virtual_ha_diagnose";
+const char *const OB_ALL_VIRTUAL_CORE_TABLE_TNAME = "__all_virtual_core_table";
 const char *const OB_ALL_VIRTUAL_SQL_AUDIT_ORA_TNAME = "ALL_VIRTUAL_SQL_AUDIT";
 const char *const OB_ALL_VIRTUAL_PLAN_STAT_ORA_TNAME = "ALL_VIRTUAL_PLAN_STAT";
 const char *const OB_ALL_VIRTUAL_PLAN_CACHE_PLAN_EXPLAIN_ORA_TNAME = "ALL_VIRTUAL_PLAN_CACHE_PLAN_EXPLAIN";
@@ -2931,6 +2938,7 @@ const char *const OB_DBA_OB_USERS_TNAME = "DBA_OB_USERS";
 const char *const OB_CDB_OB_USERS_TNAME = "CDB_OB_USERS";
 const char *const OB_DBA_OB_DATABASE_PRIVILEGE_TNAME = "DBA_OB_DATABASE_PRIVILEGE";
 const char *const OB_CDB_OB_DATABASE_PRIVILEGE_TNAME = "CDB_OB_DATABASE_PRIVILEGE";
+const char *const OB_DBA_OB_CLUSTER_EVENT_HISTORY_TNAME = "DBA_OB_CLUSTER_EVENT_HISTORY";
 const char *const OB_DBA_SYNONYMS_TNAME = "DBA_SYNONYMS";
 const char *const OB_DBA_OBJECTS_ORA_TNAME = "DBA_OBJECTS";
 const char *const OB_ALL_OBJECTS_TNAME = "ALL_OBJECTS";
@@ -3490,6 +3498,7 @@ const char *const OB_ALL_LOG_RESTORE_SOURCE_AUX_LOB_META_TNAME = "__all_log_rest
 const char *const OB_ALL_SERVICE_EPOCH_AUX_LOB_META_TNAME = "__all_service_epoch_aux_lob_meta";
 const char *const OB_ALL_SPATIAL_REFERENCE_SYSTEMS_AUX_LOB_META_TNAME = "__all_spatial_reference_systems_aux_lob_meta";
 const char *const OB_ALL_COLUMN_CHECKSUM_ERROR_INFO_AUX_LOB_META_TNAME = "__all_column_checksum_error_info_aux_lob_meta";
+const char *const OB_ALL_CLUSTER_EVENT_HISTORY_AUX_LOB_META_TNAME = "__all_cluster_event_history_aux_lob_meta";
 const char *const OB_ALL_TABLE_AUX_LOB_PIECE_TNAME = "__all_table_aux_lob_piece";
 const char *const OB_ALL_COLUMN_AUX_LOB_PIECE_TNAME = "__all_column_aux_lob_piece";
 const char *const OB_ALL_DDL_OPERATION_AUX_LOB_PIECE_TNAME = "__all_ddl_operation_aux_lob_piece";
@@ -3706,6 +3715,7 @@ const char *const OB_ALL_LOG_RESTORE_SOURCE_AUX_LOB_PIECE_TNAME = "__all_log_res
 const char *const OB_ALL_SERVICE_EPOCH_AUX_LOB_PIECE_TNAME = "__all_service_epoch_aux_lob_piece";
 const char *const OB_ALL_SPATIAL_REFERENCE_SYSTEMS_AUX_LOB_PIECE_TNAME = "__all_spatial_reference_systems_aux_lob_piece";
 const char *const OB_ALL_COLUMN_CHECKSUM_ERROR_INFO_AUX_LOB_PIECE_TNAME = "__all_column_checksum_error_info_aux_lob_piece";
+const char *const OB_ALL_CLUSTER_EVENT_HISTORY_AUX_LOB_PIECE_TNAME = "__all_cluster_event_history_aux_lob_piece";
 const char *const OB_ALL_VIRTUAL_PLAN_CACHE_STAT_ALL_VIRTUAL_PLAN_CACHE_STAT_I1_TNAME = "__idx_11003_all_virtual_plan_cache_stat_i1";
 const char *const OB_ALL_VIRTUAL_SESSION_EVENT_ALL_VIRTUAL_SESSION_EVENT_I1_TNAME = "__idx_11013_all_virtual_session_event_i1";
 const char *const OB_ALL_VIRTUAL_SESSION_WAIT_ALL_VIRTUAL_SESSION_WAIT_I1_TNAME = "__idx_11014_all_virtual_session_wait_i1";

src/share/inner_table/ob_inner_table_schema_def.py

@@ -29,6 +29,10 @@
 #     real_agent access tables belong to current tenant
 # 8. Virtual table system design summary:
 #    https://yuque.antfin.com/docs/share/876fb26e-7466-476f-bd67-b3c4860fbdb8?#
+# 9. For compatibility, when add new column for system table, new column's definition should be "not null + default value" or "nullable".
+#    Specially, when column types are as follows:
+#    1. double、number：default value is not supported, so new column definition should be "nullable".
+#    2. longtext、timestamp：mysql can't cast default value to specified column type, so new column definition should be "nullable".
 
 global fields
 fields = [
@@ -5158,7 +5162,33 @@ def_table_schema(
 # 443 : __all_tenant_rewrite_rules
 
 # 444 : __all_reserved_snapshot
-# 445 : __all_cluster_event_history
+def_table_schema(
+    owner = 'yanmu.ztl',
+    table_name = '__all_cluster_event_history',
+    table_id = '445',
+    table_type = 'SYSTEM_TABLE',
+    gm_columns = [],
+    rowkey_columns = [
+      ('gmt_create', 'timestamp:6', 'false')
+    ],
+    normal_columns = [
+      ('module', 'varchar:MAX_CLUSTER_EVENT_NAME_LENGTH', 'false'),
+      ('event', 'varchar:MAX_CLUSTER_EVENT_NAME_LENGTH', 'false'),
+      ('name1', 'varchar:MAX_CLUSTER_EVENT_NAME_LENGTH', 'false', ''),
+      ('value1', 'varchar:MAX_CLUSTER_EVENT_VALUE_LENGTH', 'false', ''),
+      ('name2', 'varchar:MAX_CLUSTER_EVENT_NAME_LENGTH', 'false', ''),
+      ('value2', 'varchar:MAX_CLUSTER_EVENT_VALUE_LENGTH', 'false', ''),
+      ('name3', 'varchar:MAX_CLUSTER_EVENT_NAME_LENGTH', 'false', ''),
+      ('value3', 'varchar:MAX_CLUSTER_EVENT_VALUE_LENGTH', 'false', ''),
+      ('name4', 'varchar:MAX_CLUSTER_EVENT_NAME_LENGTH', 'false', ''),
+      ('value4', 'varchar:MAX_CLUSTER_EVENT_VALUE_LENGTH', 'false', ''),
+      ('name5', 'varchar:MAX_CLUSTER_EVENT_NAME_LENGTH', 'false', ''),
+      ('value5', 'varchar:MAX_CLUSTER_EVENT_VALUE_LENGTH', 'false', ''),
+      ('name6', 'varchar:MAX_CLUSTER_EVENT_NAME_LENGTH', 'false', ''),
+      ('value6', 'varchar:MAX_CLUSTER_EVENT_VALUE_LENGTH', 'false', ''),
+      ('extra_info', 'varchar:MAX_CLUSTER_EVENT_VALUE_LENGTH', 'false', '')
+    ],
+)
 # 446 : __all_ls_transfer_member_list_lock_info
 # 447 : __all_ls_log_restore_stat
 ################################################################################
@@ -10911,7 +10941,10 @@ def_table_schema(
 # 12360: __all_virtual_plan_table
 # 12361: __all_virtual_plan_real_info
 
-# 12362: __all_virtual_core_table
+def_table_schema(**gen_iterate_virtual_table_def(
+  table_id = '12362',
+  table_name = '__all_virtual_core_table',
+  keywords = all_def_keywords['__all_core_table']))
 
 # 12363: __all_virtual_malloc_sample_info
 # 12364: __all_virtual_ls_arb_replica_task
@@ -23410,7 +23443,36 @@ def_table_schema(
 # 21343: GV$OB_PLAN_REAL_INFO
 # 21344: V$OB_PLAN_REAL_INFO
 
-# 21345: DBA_OB_CLUSTER_EVENT_HISTORY
+def_table_schema(
+    owner = 'yanmu.ztl',
+    table_name     = 'DBA_OB_CLUSTER_EVENT_HISTORY',
+    table_id       = '21345',
+    table_type = 'SYSTEM_VIEW',
+    rowkey_columns  = [],
+    normal_columns  = [],
+    gm_columns      = [],
+    in_tenant_space = False,
+    view_definition = """
+SELECT
+  gmt_create AS `TIMESTAMP`,
+  CAST(MODULE AS CHAR(256)) MODULE,
+  CAST(EVENT AS CHAR(256)) EVENT,
+  CAST(NAME1 AS CHAR(256)) NAME1,
+  CAST(VALUE1 AS CHAR(4096)) VALUE1,
+  CAST(NAME2 AS CHAR(256)) NAME2,
+  CAST(VALUE2 AS CHAR(4096)) VALUE2,
+  CAST(NAME3 AS CHAR(256)) NAME3,
+  CAST(VALUE3 AS CHAR(4096)) VALUE3,
+  CAST(NAME4 AS CHAR(256)) NAME4,
+  CAST(VALUE4 AS CHAR(4096)) VALUE4,
+  CAST(NAME5 AS CHAR(256)) NAME5,
+  CAST(VALUE5 AS CHAR(4096)) VALUE5,
+  CAST(NAME6 AS CHAR(256)) NAME6,
+  CAST(VALUE6 AS CHAR(4096)) VALUE6,
+  CAST(EXTRA_INFO AS CHAR(4096)) EXTRA_INFO
+FROM oceanbase.__all_cluster_event_history
+""".replace("\n", " ")
+)
 
 # 21346: PARAMETERS
 

src/share/inner_table/ob_inner_table_schema_misc.ipp

@@ -1156,6 +1156,7 @@ case OB_ALL_VIRTUAL_CONSTRAINT_TID:
 case OB_ALL_VIRTUAL_CONSTRAINT_COLUMN_TID:
 case OB_ALL_VIRTUAL_CONSTRAINT_COLUMN_HISTORY_TID:
 case OB_ALL_VIRTUAL_CONSTRAINT_HISTORY_TID:
+case OB_ALL_VIRTUAL_CORE_TABLE_TID:
 case OB_ALL_VIRTUAL_DAM_CLEANUP_JOBS_TID:
 case OB_ALL_VIRTUAL_DAM_LAST_ARCH_TS_TID:
 case OB_ALL_VIRTUAL_DATABASE_TID:
@@ -1461,6 +1462,21 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       break;
     }
 
+    case OB_ALL_VIRTUAL_CORE_TABLE_TID: {
+      ObIterateVirtualTable *iter = NULL;
+      if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
+        SERVER_LOG(WARN, "create virtual table iterator failed", K(ret));
+      } else if (OB_FAIL(iter->init(OB_ALL_CORE_TABLE_TID, index_schema, params))) {
+        SERVER_LOG(WARN, "virtual table iter init failed", K(ret));
+        iter->~ObIterateVirtualTable();
+        allocator.free(iter);
+        iter = NULL;
+      } else {
+       vt_iter = iter;
+      }
+      break;
+    }
+
     case OB_ALL_VIRTUAL_DAM_CLEANUP_JOBS_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -1565,7 +1581,9 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
+  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
+  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_DBLINK_HISTORY_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -1580,9 +1598,7 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
-  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
-  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_DDL_CHECKSUM_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -1867,7 +1883,9 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
+  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
+  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_LS_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -1882,9 +1900,7 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
-  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
-  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_MOCK_FK_PARENT_TABLE_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -2169,7 +2185,9 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
+  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
+  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_PENDING_TRANSACTION_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -2184,9 +2202,7 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
-  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
-  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_PLAN_BASELINE_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -2471,7 +2487,9 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
+  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
+  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_SYS_VARIABLE_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -2486,9 +2504,7 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
-  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
-  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_SYS_VARIABLE_HISTORY_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -2773,7 +2789,9 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
+  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
+  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_TENANT_KEYSTORE_HISTORY_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -2788,9 +2806,7 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
-  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
-  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_TENANT_OLS_COMPONENT_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -3075,7 +3091,9 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
+  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
+  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_TIME_ZONE_NAME_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {
@@ -3090,9 +3108,7 @@ case OB_ALL_VIRTUAL_USER_HISTORY_TID:
       }
       break;
     }
-  END_CREATE_VT_ITER_SWITCH_LAMBDA
 
-  BEGIN_CREATE_VT_ITER_SWITCH_LAMBDA
     case OB_ALL_VIRTUAL_TIME_ZONE_TRANSITION_TID: {
       ObIterateVirtualTable *iter = NULL;
       if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIterateVirtualTable, iter))) {

src/share/ob_cluster_event_history_table_operator.cpp

@@ -0,0 +1,44 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include "ob_cluster_event_history_table_operator.h"
+#include "share/config/ob_server_config.h"
+namespace oceanbase
+{
+namespace observer
+{
+using namespace common;
+using namespace share;
+
+int ObAllClusterEventHistoryTableOperator::init(common::ObMySQLProxy &proxy)
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(ObEventHistoryTableOperator::init(proxy))) {
+    SHARE_LOG(WARN, "fail to init event history table operator", KR(ret));
+  } else {
+    set_event_table(share::OB_ALL_CLUSTER_EVENT_HISTORY_TNAME);
+  }
+  return ret;
+}
+
+ObAllClusterEventHistoryTableOperator &ObAllClusterEventHistoryTableOperator::get_instance()
+{
+  static ObAllClusterEventHistoryTableOperator instance;
+  return instance;
+}
+
+int ObAllClusterEventHistoryTableOperator::async_delete()
+{
+  return OB_NOT_SUPPORTED;
+}
+} // end namespace observer
+} // end namespace oceanbase

src/share/ob_cluster_event_history_table_operator.h

@@ -0,0 +1,54 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef _OB_CLUSTER_EVENT_HISTORY_TABLE_OPERATOR_H
+#define _OB_CLUSTER_EVENT_HISTORY_TABLE_OPERATOR_H
+#include "share/ob_event_history_table_operator.h"
+#include "share/ob_cluster_version.h"
+
+namespace oceanbase
+{
+namespace observer
+{
+class ObAllClusterEventHistoryTableOperator: public share::ObEventHistoryTableOperator
+{
+public:
+  virtual ~ObAllClusterEventHistoryTableOperator() {}
+
+  int init(common::ObMySQLProxy &proxy);
+
+  virtual int async_delete() override;
+
+  static ObAllClusterEventHistoryTableOperator &get_instance();
+private:
+  ObAllClusterEventHistoryTableOperator() {}
+  DISALLOW_COPY_AND_ASSIGN(ObAllClusterEventHistoryTableOperator);
+};
+
+} // end namespace observer
+} // end namespace oceanbase
+
+#define CLUSTER_EVENT_INSTANCE (::oceanbase::observer::ObAllClusterEventHistoryTableOperator::get_instance())
+
+#define CLUSTER_EVENT_SYNC_ADD(args...) \
+  if (OB_SUCC(ret)) { \
+    uint64_t data_version = 0; \
+    if (OB_FAIL(GET_MIN_DATA_VERSION(OB_SYS_TENANT_ID, data_version))) { \
+      SHARE_LOG(WARN, "fail to get data version", KR(ret), "tenant_id", OB_SYS_TENANT_ID); \
+    } else if (data_version < DATA_VERSION_4_1_0_0) { \
+      /* inner table is not ready, just skip */ \
+    } else if (OB_FAIL(CLUSTER_EVENT_INSTANCE.sync_add_event(args))) { \
+      SHARE_LOG(WARN, "fail to sync add event", KR(ret)); \
+    } \
+  }
+
+#endif /* _OB_CLUSTER_EVENT_HISTORY_TABLE_OPERATOR_H */

src/share/ob_cluster_version.cpp

@@ -12,7 +12,10 @@
 
 #include "share/config/ob_server_config.h"
 #include "share/ob_cluster_version.h"
+#include "share/ob_task_define.h"
+#include "lib/ob_define.h"
 #include "lib/string/ob_string.h"
+#include "observer/omt/ob_tenant_config_mgr.h"
 namespace oceanbase
 {
 namespace common
@@ -88,7 +91,9 @@ static int parse_version(const char *str, uint64_t *versions, const int64_t size
   return ret;
 }
 
-ObClusterVersion::ObClusterVersion() : is_inited_(false), config_(NULL), cluster_version_(0)
+ObClusterVersion::ObClusterVersion()
+  : is_inited_(false), config_(NULL),
+    tenant_config_mgr_(NULL), cluster_version_(0)
 {
   cluster_version_ = cal_version(DEF_MAJOR_VERSION,
                                  DEF_MINOR_VERSION,
@@ -115,20 +120,23 @@ int ObClusterVersion::init(const uint64_t cluster_version)
   return ret;
 }
 
-int ObClusterVersion::init(const ObServerConfig *config)
+int ObClusterVersion::init(
+    const ObServerConfig *config,
+    const omt::ObTenantConfigMgr *tenant_config_mgr)
 {
   int ret = OB_SUCCESS;
 
   if (is_inited_) {
-    COMMON_LOG(ERROR, "cluster version init twice", KP(config));
     ret = OB_INIT_TWICE;
-  } else if (NULL == config) {
-    COMMON_LOG(WARN, "invalid argument", KP(config));
+    COMMON_LOG(ERROR, "cluster version init twice", KR(ret), KP(config));
+  } else if (OB_ISNULL(config) || OB_ISNULL(tenant_config_mgr)) {
     ret = OB_INVALID_ARGUMENT;
+    COMMON_LOG(WARN, "invalid argument", KR(ret), KP(config), KP(tenant_config_mgr));
   } else if (OB_FAIL(refresh_cluster_version(config->min_observer_version.str()))) {
-    COMMON_LOG(WARN, "refresh cluster version error", K(ret));
+    COMMON_LOG(WARN, "refresh cluster version error", KR(ret));
   } else {
     config_ = config;
+    tenant_config_mgr_ = tenant_config_mgr;
     COMMON_LOG(INFO, "cluster version inited success", K_(cluster_version));
     is_inited_ = true;
   }
@@ -153,40 +161,60 @@ int64_t ObClusterVersion::to_string(char *buf, const int64_t buf_len) const
 
 int64_t ObClusterVersion::print_vsn(char *buf, const int64_t buf_len, uint64_t version)
 {
+  int ret = OB_SUCCESS;
   int64_t pos = 0;
   const uint32_t major = OB_VSN_MAJOR(version);
   const uint16_t minor = OB_VSN_MINOR(version);
   const uint8_t major_patch = OB_VSN_MAJOR_PATCH(version);
   const uint8_t minor_patch = OB_VSN_MINOR_PATCH(version);
   if (OB_UNLIKELY(!check_version_valid_(version))) {
-    pos = OB_INVALID_INDEX;
-    COMMON_LOG(ERROR, "invalid cluster version", KR(version), K(lbt()));
+    ret = OB_INVALID_ARGUMENT;
+    COMMON_LOG(ERROR, "invalid cluster version", KR(ret), KR(version), K(lbt()));
   } else if (major < 3
              || (3 == major && minor < 2)
              || (3 == major && 2 == minor && 0 == major_patch && minor_patch < 3)) {
-    databuff_printf(buf, buf_len, pos, "%lu(%u, %u, %u)", version, major, minor, minor_patch);
+    if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%lu(%u, %u, %u)",
+                version, major, minor, minor_patch))) {
+      COMMON_LOG(WARN, "fail to print vsn", KR(ret), K(version));
+    }
   } else {
-    databuff_printf(buf, buf_len, pos, "%lu(%u, %u, %u, %u)", version, major, minor, major_patch, minor_patch);
+    if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%lu(%u, %u, %u, %u)",
+                version, major, minor, major_patch, minor_patch))) {
+      COMMON_LOG(WARN, "fail to print vsn", KR(ret), K(version));
+    }
+  }
+  if (OB_FAIL(ret)) {
+    pos = OB_INVALID_INDEX;
   }
   return pos;
 }
 
 int64_t ObClusterVersion::print_version_str(char *buf, const int64_t buf_len, uint64_t version)
 {
+  int ret = OB_SUCCESS;
   int64_t pos = 0;
   const uint32_t major = OB_VSN_MAJOR(version);
   const uint16_t minor = OB_VSN_MINOR(version);
   const uint8_t major_patch = OB_VSN_MAJOR_PATCH(version);
   const uint8_t minor_patch = OB_VSN_MINOR_PATCH(version);
   if (OB_UNLIKELY(!check_version_valid_(version))) {
-    pos = OB_INVALID_INDEX;
+    ret = OB_INVALID_ARGUMENT;
     COMMON_LOG(ERROR, "invalid cluster version", KR(version), K(lbt()));
   } else if (major < 3
              || (3 == major && minor < 2)
              || (3 == major && 2 == minor && 0 == major_patch && minor_patch < 3)) {
-    databuff_printf(buf, buf_len, pos, "%u.%u.%u", major, minor, minor_patch);
+    if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%u.%u.%u",
+                major, minor, minor_patch))) {
+      COMMON_LOG(WARN, "fail to print version str", KR(ret), K(version));
+    }
   } else {
-    databuff_printf(buf, buf_len, pos, "%u.%u.%u.%u", major, minor, major_patch, minor_patch);
+    if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%u.%u.%u.%u",
+                major, minor, major_patch, minor_patch))) {
+      COMMON_LOG(WARN, "fail to print version str", KR(ret), K(version));
+    }
+  }
+  if (OB_FAIL(ret)) {
+    pos = OB_INVALID_INDEX;
   }
   return pos;
 }
@@ -235,9 +263,32 @@ int ObClusterVersion::get_tenant_data_version(
     uint64_t &data_version)
 {
   int ret = OB_SUCCESS;
-  //TODO: mock data version with cluster version for now.
-  UNUSED(tenant_id);
-  data_version = ATOMIC_LOAD(&cluster_version_);
+  data_version = 0;
+  if (OB_ISNULL(tenant_config_mgr_)) {
+    ret = OB_NOT_INIT;
+    COMMON_LOG(WARN, "tenant_config is null", KR(ret), KP(tenant_config_mgr_));
+  } else {
+    // wont't fallback or retry
+    omt::ObTenantConfigGuard tenant_config(tenant_config_mgr_->get_tenant_config_with_lock(tenant_id));
+    if (tenant_config.is_valid() && tenant_config->compatible.value_updated()) {
+      data_version = tenant_config->compatible;
+    } else if (is_sys_tenant(tenant_id)
+               || is_meta_tenant(tenant_id)
+               || get_cluster_version() <= CLUSTER_VERSION_4_1_0_0) {
+      // 1. For sys/meta tenant, circular dependency problem may exist when load tenant config from inner tables.
+      //    For safety, data_version will fallback to last barrier data version until actual tenant config is loaded.
+      // 2. To compatible with upgrade path from 4.0 to 4.1
+      data_version = LAST_BARRIER_DATA_VERSION;
+      if (REACH_TIME_INTERVAL(60 * 1000 * 1000L)) {
+        share::ObTaskController::get().allow_next_syslog();
+        COMMON_LOG(INFO, "tenant data version fallback to last barrier version", K(tenant_id), K(data_version));
+      }
+    } else {
+      // For user tenant
+      ret = OB_ENTRY_NOT_EXIST;
+      COMMON_LOG(WARN, "tenant compatible version is not refreshed", KR(ret), K(tenant_id));
+    }
+  }
   return ret;
 }
 
@@ -246,9 +297,12 @@ int ObClusterVersion::tenant_need_upgrade(
     bool &need_upgrade)
 {
   int ret = OB_SUCCESS;
-  //TODO: mock data version with cluster version for now.
-  UNUSED(tenant_id);
-  need_upgrade = get_cluster_version() < CLUSTER_CURRENT_VERSION;
+  uint64_t data_version = 0;
+  if (OB_FAIL(get_tenant_data_version(tenant_id, data_version))) {
+    COMMON_LOG(WARN, "fail to get tenant data version", KR(ret), K(tenant_id));
+  } else {
+    need_upgrade = (data_version < DATA_CURRENT_VERSION);
+  }
   return ret;
 }
 

src/share/ob_cluster_version.h

@@ -18,6 +18,10 @@
 
 namespace oceanbase
 {
+namespace omt
+{
+class ObTenantConfigMgr;
+}
 namespace common
 {
 class ObServerConfig;
@@ -32,8 +36,10 @@ public:
   void destroy();
   int64_t to_string(char *buf, const int64_t buf_len) const;
 
+  int init(const common::ObServerConfig *config,
+           const omt::ObTenantConfigMgr *tenant_config_mgr);
+
   /* cluster version related */
-  int init(const common::ObServerConfig *config);
   int init(const uint64_t cluster_version);
   int refresh_cluster_version(const char *verstr);
   int reload_config();
@@ -52,17 +58,17 @@ public:
   static int get_version(const common::ObString &verstr, uint64_t &version);
   static int64_t print_vsn(char *buf, const int64_t buf_len, uint64_t version);
   static int64_t print_version_str(char *buf, const int64_t buf_len, uint64_t version);
+  static bool check_version_valid_(const uint64_t version);
 public:
   static const int64_t MAX_VERSION_ITEM = 16;
   static const int64_t MAJOR_POS       = 0;
   static const int64_t MINOR_POS       = 1;
   static const int64_t MAJOR_PATCH_POS = 2;
   static const int64_t MINOR_PATCH_POS = 3;
-private:
-  static bool check_version_valid_(const uint64_t version);
 private:
   bool is_inited_;
   const common::ObServerConfig *config_;
+  const omt::ObTenantConfigMgr *tenant_config_mgr_;
   uint64_t cluster_version_;
 };
 
@@ -170,6 +176,7 @@ cal_version(const uint64_t major, const uint64_t minor, const uint64_t major_pat
 #define DATA_VERSION_4_1_0_0 (oceanbase::common::cal_version(4, 1, 0, 0))
 
 // should check returned ret
+#define LAST_BARRIER_DATA_VERSION DATA_VERSION_4_0_0_0
 #define DATA_CURRENT_VERSION DATA_VERSION_4_1_0_0
 #define GET_MIN_DATA_VERSION(tenant_id, data_version) (oceanbase::common::ObClusterVersion::get_instance().get_tenant_data_version((tenant_id), (data_version)))
 #define TENANT_NEED_UPGRADE(tenant_id, need) (oceanbase::common::ObClusterVersion::get_instance().tenant_need_upgrade((tenant_id), (need)))

src/share/ob_freeze_info_proxy.cpp

@@ -37,7 +37,7 @@ namespace share
 {
 
 OB_SERIALIZE_MEMBER(ObSimpleFrozenStatus, frozen_scn_,
-                    schema_version_, cluster_version_);
+                    schema_version_, data_version_);
 
 int ObFreezeInfoProxy::get_freeze_info(
     ObISQLClient &sql_proxy,
@@ -181,7 +181,7 @@ int ObFreezeInfoProxy::set_freeze_info(
     ret = OB_INVALID_ARGUMENT;
     LOG_ERROR("invalid argument", KR(ret), K(frozen_status), K_(tenant_id));
   } else if (OB_FAIL(dml.add_uint64_pk_column("frozen_scn", frozen_status.frozen_scn_.get_val_for_inner_table_field()))
-            || OB_FAIL(dml.add_column("cluster_version", frozen_status.cluster_version_))
+            || OB_FAIL(dml.add_column("cluster_version", frozen_status.data_version_))
             || OB_FAIL(dml.add_column("schema_version", frozen_status.schema_version_))) {
     LOG_WARN("fail to add column", KR(ret), K(frozen_status), K_(tenant_id));
   } else if (OB_FAIL(exec.exec_insert(OB_ALL_FREEZE_INFO_TNAME, dml, affected_rows))) {
@@ -388,7 +388,7 @@ int ObFreezeInfoProxy::construct_frozen_status_(
   int ret = OB_SUCCESS;
   uint64_t frozen_scn_val = OB_INVALID_SCN_VAL;
   EXTRACT_UINT_FIELD_MYSQL(result, "frozen_scn", frozen_scn_val, uint64_t);
-  EXTRACT_INT_FIELD_MYSQL(result, "cluster_version", frozen_status.cluster_version_, int64_t);
+  EXTRACT_INT_FIELD_MYSQL(result, "cluster_version", frozen_status.data_version_, int64_t);
   EXTRACT_INT_FIELD_MYSQL(result, "schema_version", frozen_status.schema_version_, int64_t);
   if (FAILEDx(frozen_status.frozen_scn_.convert_for_inner_table_field(frozen_scn_val))) {
     LOG_WARN("fail to convert val to SCN", KR(ret), K(frozen_scn_val));

src/share/ob_freeze_info_proxy.h

@@ -42,7 +42,7 @@ namespace share
 {
 /*
  * the columns of __all_freeze_info are as follows:
- * | frozen_scn | schema_version | cluster_version |
+ * | frozen_scn | schema_version | data_version |
  * we make sure the row_id of __all_freeze_info equals to frozen_scn
  */
 struct ObSimpleFrozenStatus
@@ -50,13 +50,13 @@ struct ObSimpleFrozenStatus
   ObSimpleFrozenStatus()
     : frozen_scn_(),
       schema_version_(INVALID_SCHEMA_VERSION), 
-      cluster_version_(0)
+      data_version_(0)
   {}
   ObSimpleFrozenStatus(const SCN &frozen_scn,
                        const int64_t schema_version,
-                       const int64_t cluster_version)
+                       const int64_t data_version)
     : schema_version_(schema_version),
-      cluster_version_(cluster_version)
+      data_version_(data_version)
   {
     frozen_scn_ = frozen_scn;
   }
@@ -65,20 +65,20 @@ struct ObSimpleFrozenStatus
   {
     frozen_scn_ = other.frozen_scn_;
     schema_version_ = other.schema_version_;
-    cluster_version_ = other.cluster_version_;
+    data_version_ = other.data_version_;
   }
 
   void reset()
   {
     frozen_scn_.reset();
     schema_version_ = INVALID_SCHEMA_VERSION;
-    cluster_version_ = 0;
+    data_version_ = 0;
   }
 
-  void set_initial_value(const int64_t cluster_version)
+  void set_initial_value(const int64_t data_version)
   {
     schema_version_ = ORIGIN_SCHEMA_VERSION;
-    cluster_version_ = cluster_version;
+    data_version_ = data_version;
     frozen_scn_ = share::SCN::base_scn();
   }
 
@@ -93,11 +93,11 @@ struct ObSimpleFrozenStatus
     return ((this == &other)
             || ((this->frozen_scn_ == other.frozen_scn_)
             && (this->schema_version_ == other.schema_version_)
-            && (this->cluster_version_ == other.cluster_version_)));
+            && (this->data_version_ == other.data_version_)));
   }
 
   TO_STRING_KV(N_FROZEN_VERSION, frozen_scn_, K_(schema_version),
-               K_(cluster_version));
+               K_(data_version));
 
   static const int64_t INVALID_SCHEMA_VERSION = 0;
   static const int64_t ORIGIN_SCHEMA_VERSION = 1;
@@ -108,7 +108,7 @@ struct ObSimpleFrozenStatus
 
   SCN frozen_scn_;
   int64_t schema_version_;
-  int64_t cluster_version_;
+  int64_t data_version_;
 
   OB_UNIS_VERSION(1);
 };

src/share/ob_global_stat_proxy.cpp

@@ -28,18 +28,23 @@ using namespace common::sqlclient;
 namespace share
 {
 const char *ObGlobalStatProxy::TENANT_ID_CNAME = "tenant_id";
-int ObGlobalStatProxy::set_init_value(const int64_t core_schema_version,
-                                      const int64_t baseline_schema_version,
-                                      const int64_t rootservice_epoch,
-                                      const SCN &snapshot_gc_scn,
-                                      const int64_t gc_schema_version)
+int ObGlobalStatProxy::set_init_value(
+    const int64_t core_schema_version,
+    const int64_t baseline_schema_version,
+    const int64_t rootservice_epoch,
+    const SCN &snapshot_gc_scn,
+    const int64_t gc_schema_version,
+    const uint64_t target_data_version,
+    const uint64_t current_data_version)
 {
   int ret = OB_SUCCESS;
   if (!is_valid() || core_schema_version <= 0 || baseline_schema_version < -1
-      || !snapshot_gc_scn.is_valid() || OB_INVALID_ID == rootservice_epoch || gc_schema_version < 0) {
+      || !snapshot_gc_scn.is_valid() || OB_INVALID_ID == rootservice_epoch || gc_schema_version < 0
+      || target_data_version <= 0 || current_data_version <= 0) {
     ret = OB_INVALID_ARGUMENT;
-    LOG_WARN("invalid argument", K(ret), "self valid", is_valid(), K(rootservice_epoch),
-        K(core_schema_version), K(baseline_schema_version), K(snapshot_gc_scn), K(gc_schema_version));
+    LOG_WARN("invalid argument", KR(ret), "self valid", is_valid(), K(rootservice_epoch),
+             K(core_schema_version), K(baseline_schema_version), K(snapshot_gc_scn),
+             K(gc_schema_version), K(target_data_version), K(current_data_version));
   } else {
     ObGlobalStatItem::ItemList list;
     ObGlobalStatItem core_schema_version_item(list, "core_schema_version", core_schema_version);
@@ -47,9 +52,11 @@ int ObGlobalStatProxy::set_init_value(const int64_t core_schema_version,
     ObGlobalStatItem rootservice_epoch_item(list, "rootservice_epoch", rootservice_epoch);
     ObGlobalStatItem snapshot_gc_scn_item(list, "snapshot_gc_scn", snapshot_gc_scn.get_val_for_inner_table_field());
     ObGlobalStatItem gc_schema_version_item(list, "gc_schema_version", gc_schema_version);
+    ObGlobalStatItem target_data_version_item(list, "target_data_version", static_cast<int64_t>(target_data_version));
+    ObGlobalStatItem current_data_version_item(list, "current_data_version", static_cast<int64_t>(current_data_version));
 
     if (OB_FAIL(update(list))) {
-      LOG_WARN("update failed", K(list), K(ret));
+      LOG_WARN("update failed", KR(ret), K(list));
     }
   }
   return ret;
@@ -58,18 +65,27 @@ int ObGlobalStatProxy::set_init_value(const int64_t core_schema_version,
 int ObGlobalStatProxy::set_tenant_init_global_stat(
     const int64_t core_schema_version,
     const int64_t baseline_schema_version,
-    const SCN &snapshot_gc_scn)
+    const SCN &snapshot_gc_scn,
+    const uint64_t target_data_version,
+    const uint64_t current_data_version)
 {
   int ret = OB_SUCCESS;
-  if (!is_valid() || core_schema_version <= 0 || baseline_schema_version < OB_INVALID_VERSION
-      || (!snapshot_gc_scn.is_valid())) {
+  if (!is_valid() || core_schema_version <= 0
+      || baseline_schema_version < OB_INVALID_VERSION
+      || !snapshot_gc_scn.is_valid()
+      || target_data_version <= 0
+      || current_data_version <= 0) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("invalid argument", KR(ret), "self valid", is_valid(),
-             K(core_schema_version), K(baseline_schema_version), K(snapshot_gc_scn));
+             K(core_schema_version), K(baseline_schema_version),
+             K(snapshot_gc_scn), K(target_data_version),
+             K(current_data_version));
   } else {
     ObGlobalStatItem::ItemList list;
     ObGlobalStatItem core_schema_version_item(list, "core_schema_version", core_schema_version);
     ObGlobalStatItem baseline_schema_version_item(list, "baseline_schema_version", baseline_schema_version);
+    ObGlobalStatItem target_data_version_item(list, "target_data_version", static_cast<int64_t>(target_data_version));
+    ObGlobalStatItem current_data_version_item(list, "current_data_version", static_cast<int64_t>(current_data_version));
     // only Normal state tenant can refresh snapshot_gc_scn
     ObGlobalStatItem snapshot_gc_scn_item(list, "snapshot_gc_scn", snapshot_gc_scn.get_val_for_inner_table_field());
     if (OB_FAIL(update(list))) {
@@ -136,12 +152,9 @@ int ObGlobalStatProxy::get_snapshot_gc_scn(SCN &snapshot_gc_scn)
   ObGlobalStatItem::ItemList list;
   uint64_t snapshot_gc_scn_val = 0;
   ObGlobalStatItem snapshot_gc_scn_item(list, "snapshot_gc_scn", snapshot_gc_scn_val);
-  ObTimeoutCtx ctx;
   if (!is_valid()) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("invalid argument", K(ret), "self valid", is_valid());
-  } else if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) {
-    LOG_WARN("fail to get timeout ctx", K(ret), K(ctx));
   } else if (OB_FAIL(get(list))) {
     LOG_WARN("get failed", K(ret));
   } else {
@@ -153,6 +166,72 @@ int ObGlobalStatProxy::get_snapshot_gc_scn(SCN &snapshot_gc_scn)
   return ret;
 }
 
+int ObGlobalStatProxy::update_current_data_version(const uint64_t current_data_version)
+{
+  int ret = OB_SUCCESS;
+  ObGlobalStatItem::ItemList list;
+  ObGlobalStatItem item(list, "current_data_version", current_data_version);
+  bool is_incremental = true;
+  if (!is_valid() || current_data_version <= 0) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid arg", KR(ret), "valid", is_valid(), K(current_data_version));
+  } else if (OB_FAIL(update(list, is_incremental))) {
+    LOG_WARN("update failed", KR(ret), K(list));
+  }
+  return ret;
+}
+
+int ObGlobalStatProxy::get_current_data_version(uint64_t &current_data_version)
+{
+  int ret = OB_SUCCESS;
+  current_data_version = 0;
+  ObGlobalStatItem::ItemList list;
+  ObGlobalStatItem item(list, "current_data_version", OB_INVALID_VERSION);
+  if (!is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", KR(ret), "self valid", is_valid());
+  } else if (OB_FAIL(get(list))) {
+    LOG_WARN("get failed", KR(ret));
+  } else {
+    current_data_version = static_cast<uint64_t>(item.value_);
+  }
+  return ret;
+}
+
+int ObGlobalStatProxy::update_target_data_version(const uint64_t target_data_version)
+{
+  int ret = OB_SUCCESS;
+  ObGlobalStatItem::ItemList list;
+  ObGlobalStatItem item(list, "target_data_version", target_data_version);
+  bool is_incremental = true;
+  if (!is_valid() || target_data_version <= 0) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid arg", KR(ret), "valid", is_valid(), K(target_data_version));
+  } else if (OB_FAIL(update(list, is_incremental))) {
+    LOG_WARN("update failed", KR(ret), K(list));
+  }
+  return ret;
+}
+
+int ObGlobalStatProxy::get_target_data_version(
+    const bool for_update,
+    uint64_t &target_data_version)
+{
+  int ret = OB_SUCCESS;
+  target_data_version = 0;
+  ObGlobalStatItem::ItemList list;
+  ObGlobalStatItem item(list, "target_data_version", OB_INVALID_VERSION);
+  if (!is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid argument", KR(ret), "self valid", is_valid());
+  } else if (OB_FAIL(get(list, for_update))) {
+    LOG_WARN("get failed", KR(ret));
+  } else {
+    target_data_version = static_cast<uint64_t>(item.value_);
+  }
+  return ret;
+}
+
 int ObGlobalStatProxy::inc_rootservice_epoch()
 {
   int ret = OB_SUCCESS;
@@ -201,10 +280,13 @@ int ObGlobalStatProxy::update(const ObGlobalStatItem::ItemList &list,
   int64_t affected_rows = 0;
   ObDMLSqlSplicer dml(ObDMLSqlSplicer::NAKED_VALUE_MODE);
   ObArray<ObCoreTableProxy::UpdateCell> cells;
+  ObTimeoutCtx ctx;
   if (!is_valid() || list.is_empty()) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("invalid argument", K(ret), "self valid", is_valid(),
         "list size", list.get_size());
+  } else if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) {
+    LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx));
   } else if (OB_FAIL(core_table_.load_for_update())) {
     LOG_WARN("core_table_load_for_update failed", K(ret));
   } else {
@@ -291,40 +373,44 @@ int ObGlobalStatProxy::get_rootservice_epoch(int64_t &rootservice_epoch)
   return ret;
 }
 
-int ObGlobalStatProxy::get(ObGlobalStatItem::ItemList &list)
+int ObGlobalStatProxy::get(
+    ObGlobalStatItem::ItemList &list,
+    bool for_update /*= false*/)
 {
   int ret = OB_SUCCESS;
   ObTimeoutCtx ctx;
   if (!is_valid() || list.is_empty()) {
     ret = OB_INVALID_ARGUMENT;
-    LOG_WARN("invalid argument", K(ret), "self valid", is_valid(),
+    LOG_WARN("invalid argument", KR(ret), "self valid", is_valid(),
         "list size", list.get_size());
   } else if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) {
-    LOG_WARN("fail to get timeout ctx", K(ret), K(ctx));
-  } else if (OB_FAIL(core_table_.load())) {
-    LOG_WARN("core_table load failed", K(ret));
+    LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx));
+  } else if (!for_update && OB_FAIL(core_table_.load())) {
+    LOG_WARN("core_table load failed", KR(ret));
+  } else if (for_update && OB_FAIL(core_table_.load_for_update())) {
+    LOG_WARN("core_table load failed", KR(ret));
   } else {
     if (OB_FAIL(core_table_.next())) {
       if (OB_ITER_END == ret) {
         ret = OB_EMPTY_RESULT;
-        LOG_WARN("no row exist", K(ret));
+        LOG_WARN("no row exist", KR(ret));
       } else {
-        LOG_WARN("next failed", K(ret));
+        LOG_WARN("next failed", KR(ret));
       }
     } else {
       ObGlobalStatItem *it = list.get_first();
       if (NULL == it) {
         ret = OB_ERR_UNEXPECTED;
-        LOG_WARN("NULL iterator", K(ret));
+        LOG_WARN("NULL iterator", KR(ret));
       }
       while (OB_SUCCESS == ret && it != list.get_header()) {
         if (OB_FAIL(core_table_.get_int(it->name_, it->value_))) {
-          LOG_WARN("get int failed", "name", it->name_, K(ret));
+          LOG_WARN("get int failed", "name", it->name_, KR(ret));
         } else {
           it = it->get_next();
           if (NULL == it) {
             ret = OB_ERR_UNEXPECTED;
-            LOG_WARN("NULL iterator", K(ret));
+            LOG_WARN("NULL iterator", KR(ret));
           }
         }
       }
@@ -334,9 +420,9 @@ int ObGlobalStatProxy::get(ObGlobalStatItem::ItemList &list)
           ret = OB_SUCCESS;
         } else if (OB_SUCC(ret)) {
           ret = OB_ERR_UNEXPECTED;
-          LOG_WARN("__all_global_stat table more than one row", K(ret));
+          LOG_WARN("__all_global_stat table more than one row", KR(ret));
         } else {
-          LOG_WARN("next failed", K(ret));
+          LOG_WARN("next failed", KR(ret));
         }
       }
     }

src/share/ob_global_stat_proxy.h

@@ -51,11 +51,15 @@ public:
                              const int64_t baseline_schema_version,
                              const int64_t rootservice_epoch,
                              const SCN &snapshot_gc_scn,
-                             const int64_t gc_schema_version);
+                             const int64_t gc_schema_version,
+                             const uint64_t target_data_version,
+                             const uint64_t current_data_version);
 
   virtual int set_tenant_init_global_stat(const int64_t core_schema_version,
                                           const int64_t baseline_schema_version,
-                                          const SCN &snapshot_gc_scn);
+                                          const SCN &snapshot_gc_scn,
+                                          const uint64_t target_data_version,
+                                          const uint64_t current_data_version);
 
   virtual int set_core_schema_version(const int64_t core_schema_version);
   virtual int set_baseline_schema_version(const int64_t baseline_schema_version);
@@ -65,12 +69,18 @@ public:
   virtual int get_baseline_schema_version(int64_t &baseline_schema_version);
 
   virtual int get_rootservice_epoch(int64_t &rootservice_epoch);
+
+  int update_current_data_version(const uint64_t current_data_version);
+  int get_current_data_version(uint64_t &current_data_version);
+  int update_target_data_version(const uint64_t target_data_version);
+  int get_target_data_version(const bool for_update, uint64_t &target_data_version);
+
   virtual int get_snapshot_info(int64_t &snapshot_gc_scn,
                                 int64_t &gc_schema_version);
-  static int select_snapshot_gc_scn_for_update(common::ObISQLClient &sql_client, 
+  static int select_snapshot_gc_scn_for_update(common::ObISQLClient &sql_client,
                                                const uint64_t tenant_id,
                                                SCN &snapshot_gc_scn);
-  static int update_snapshot_gc_scn(common::ObISQLClient &sql_client, 
+  static int update_snapshot_gc_scn(common::ObISQLClient &sql_client,
                                     const uint64_t tenant_id,
                                     const SCN &snapshot_gc_scn,
                                     int64_t &affected_rows);
@@ -79,7 +89,7 @@ public:
   int set_snapshot_gc_scn(const SCN &snapshot_gc_scn);
 private:
   int update(const ObGlobalStatItem::ItemList &list, const bool is_incremental = false);
-  int get(ObGlobalStatItem::ItemList &list);
+  int get(ObGlobalStatItem::ItemList &list, bool for_update = false);
 
 private:
   static const char* OB_ALL_GC_SCHEMA_VERSION_TNAME;

src/share/ob_primary_standby_service.cpp

@@ -19,6 +19,9 @@
 #include "rootserver/ob_rs_event_history_table_operator.h" // ROOTSERVICE_EVENT_ADD
 #include "rootserver/ob_tenant_role_transition_service.h" // ObTenantRoleTransitionService
 #include "share/restore/ob_log_archive_source_mgr.h"  // ObLogArchiveSourceMgr
+#include "share/ob_standby_upgrade.h"  // ObStandbyUpgrade
+#include "observer/ob_inner_sql_connection.h"//ObInnerSQLConnection
+#include "storage/tx/ob_trans_service.h" //ObTransService
 
 namespace oceanbase
 {
@@ -183,5 +186,47 @@ int ObPrimaryStandbyService::failover_to_primary(const uint64_t tenant_id, const
   return ret;
 }
 
+int ObPrimaryStandbyService::write_upgrade_barrier_log(
+    ObMySQLTransaction &trans,
+    const uint64_t tenant_id,
+    const uint64_t data_version)
+{
+  int ret = OB_SUCCESS;
+  ObStandbyUpgrade primary_data_version(data_version);
+  observer::ObInnerSQLConnection *inner_conn = static_cast<observer::ObInnerSQLConnection *>(trans.get_connection());
+  if (OB_FAIL(check_inner_stat_())) {
+    LOG_WARN("inner stat error", KR(ret), K_(inited));
+  } else if (OB_ISNULL(inner_conn)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("connection or trans service is null", KR(ret), KP(inner_conn));
+  } else if (!is_user_tenant(tenant_id)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("not user tenant_id", KR(ret), K(tenant_id));
+  } else if (!ObClusterVersion::check_version_valid_(data_version)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid data_version", KR(ret), K(data_version));
+  } else {
+    const int64_t length = primary_data_version.get_serialize_size();
+    char *buf = NULL;
+    int64_t pos = 0;
+    ObArenaAllocator allocator("StandbyUpgrade");
+    if (OB_ISNULL(buf = static_cast<char *>(allocator.alloc(length)))) {
+      ret = OB_ALLOCATE_MEMORY_FAILED;
+      LOG_WARN("fail to alloc buf", KR(ret), K(length));
+    } else if (OB_FAIL(primary_data_version.serialize(buf, length, pos))) {
+      LOG_WARN("failed to serialize", KR(ret), K(primary_data_version), K(length), K(pos));
+    } else if (OB_UNLIKELY(pos > length)) {
+      ret = OB_SIZE_OVERFLOW;
+      LOG_WARN("serialize error", KR(ret), K(pos), K(length), K(primary_data_version));
+    } else if (OB_FAIL(inner_conn->register_multi_data_source(
+                       tenant_id, SYS_LS, transaction::ObTxDataSourceType::STANDBY_UPGRADE,
+                       buf, length))) {
+      LOG_WARN("failed to register tx data", KR(ret), K(tenant_id));
+    }
+    LOG_INFO("write_upgrade_barrier_log finished", KR(ret), K(tenant_id), K(primary_data_version), K(length), KPHEX(buf, length));
+  }
+  return ret;
+}
+
 }
 }

src/share/ob_primary_standby_service.h

@@ -56,6 +56,8 @@ public:
    */
   int switch_tenant(const obrpc::ObSwitchTenantArg &arg);
 
+  int write_upgrade_barrier_log(ObMySQLTransaction &trans, const uint64_t tenant_id, const uint64_t data_version);
+
 private:
   int check_inner_stat_();
 

src/share/ob_rpc_struct.cpp

@@ -520,7 +520,7 @@ OB_SERIALIZE_MEMBER(ObSysVarIdValue, sys_id_, value_);
 bool ObCreateTenantArg::is_valid() const
 {
   return !tenant_schema_.get_tenant_name_str().empty() && pool_list_.count() > 0
-         && (!is_restore_ || (is_restore_ && palf_base_info_.is_valid()));
+         && (!is_restore_ || (palf_base_info_.is_valid() && compatible_version_ > 0));
 }
 
 int ObCreateTenantArg::check_valid() const
@@ -552,17 +552,31 @@ int ObCreateTenantArg::assign(const ObCreateTenantArg &other)
     name_case_mode_ = other.name_case_mode_;
     is_restore_ = other.is_restore_;
     palf_base_info_ = other.palf_base_info_;
+    compatible_version_ = other.compatible_version_;
   }
   return ret;
 }
 
+void ObCreateTenantArg::reset()
+{
+  ObDDLArg::reset();
+  tenant_schema_.reset();
+  pool_list_.reset();
+  if_not_exist_ = false;
+  sys_var_list_.reset();
+  name_case_mode_ = common::OB_NAME_CASE_INVALID;
+  is_restore_ = false;
+  palf_base_info_.reset();
+  compatible_version_ = 0;
+}
+
 int ObCreateTenantArg::init(const share::schema::ObTenantSchema &tenant_schema,
                             const common::ObIArray<common::ObString> &pool_list,
                             const bool is_sync_from_primary,
                             const bool if_not_exist)
 {
   int ret = OB_SUCCESS;
-  ObDDLArg::reset();
+  reset();
   if (OB_UNLIKELY(!tenant_schema.is_valid() || 0 == pool_list.count())) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("invalid argument", KR(ret), K(tenant_schema), K(pool_list));
@@ -587,7 +601,8 @@ DEF_TO_STRING(ObCreateTenantArg)
        K_(sys_var_list),
        K_(name_case_mode),
        K_(is_restore),
-       K_(palf_base_info));
+       K_(palf_base_info),
+       K_(compatible_version));
   return pos;
 }
 
@@ -597,7 +612,9 @@ OB_SERIALIZE_MEMBER((ObCreateTenantArg, ObDDLArg),
                     if_not_exist_,
                     sys_var_list_,
                     name_case_mode_,
-                    is_restore_, palf_base_info_);
+                    is_restore_,
+                    palf_base_info_,
+                    compatible_version_);
 
 bool ObCreateTenantEndArg::is_valid() const
 {
@@ -3925,8 +3942,11 @@ OB_SERIALIZE_MEMBER((ObRunJobArg, ObServerZoneArg), job_);
 
 ObUpgradeJobArg::ObUpgradeJobArg()
   : action_(INVALID_ACTION),
-    version_(common::OB_INVALID_VERSION)
-{}
+    version_(common::OB_INVALID_VERSION),
+    tenant_ids_()
+{
+  tenant_ids_.set_label("UpgJobArr");
+}
 bool ObUpgradeJobArg::is_valid() const
 {
   return INVALID_ACTION != action_
@@ -3935,21 +3955,27 @@ bool ObUpgradeJobArg::is_valid() const
 int ObUpgradeJobArg::assign(const ObUpgradeJobArg &other)
 {
   int ret = OB_SUCCESS;
-  action_ = other.action_;
-  version_ = other.version_;
+  if (OB_FAIL(tenant_ids_.assign(other.tenant_ids_))) {
+    LOG_WARN("fail to assign tenant_ids", KR(ret));
+  } else {
+    action_ = other.action_;
+    version_ = other.version_;
+  }
   return ret;
 }
-OB_SERIALIZE_MEMBER(ObUpgradeJobArg, action_, version_);
+OB_SERIALIZE_MEMBER(ObUpgradeJobArg, action_, version_, tenant_ids_);
 
 int ObUpgradeTableSchemaArg::init(
     const uint64_t tenant_id,
-    const uint64_t table_id)
+    const uint64_t table_id,
+    const bool upgrade_virtual_schema)
 {
   int ret = OB_SUCCESS;
   ObDDLArg::reset();
   exec_tenant_id_ = tenant_id;
   tenant_id_ = tenant_id;
   table_id_ = table_id;
+  upgrade_virtual_schema_ = upgrade_virtual_schema;
   return ret;
 }
 
@@ -3957,8 +3983,7 @@ bool ObUpgradeTableSchemaArg::is_valid() const
 {
   return common::OB_INVALID_TENANT_ID != exec_tenant_id_
          && common::OB_INVALID_TENANT_ID != tenant_id_
-         /*index、lob table will be created with related system table*/
-         && is_system_table(table_id_);
+         && (upgrade_virtual_schema_ || is_system_table(table_id_));
 }
 
 int ObUpgradeTableSchemaArg::assign(const ObUpgradeTableSchemaArg &other)
@@ -3970,11 +3995,12 @@ int ObUpgradeTableSchemaArg::assign(const ObUpgradeTableSchemaArg &other)
   } else {
     tenant_id_ = other.tenant_id_;
     table_id_ = other.table_id_;
+    upgrade_virtual_schema_ = other.upgrade_virtual_schema_;
   }
   return ret;
 }
 
-OB_SERIALIZE_MEMBER((ObUpgradeTableSchemaArg, ObDDLArg), tenant_id_, table_id_);
+OB_SERIALIZE_MEMBER((ObUpgradeTableSchemaArg, ObDDLArg), tenant_id_, table_id_, upgrade_virtual_schema_);
 
 int ObAdminFlushCacheArg::assign(const ObAdminFlushCacheArg &other)
 {
@@ -5096,7 +5122,7 @@ OB_SERIALIZE_MEMBER((ObLabelSePolicyDDLArg, ObDDLArg), ddl_type_, schema_);
 OB_SERIALIZE_MEMBER((ObLabelSeComponentDDLArg, ObDDLArg), ddl_type_, schema_, policy_name_);
 OB_SERIALIZE_MEMBER((ObLabelSeLabelDDLArg, ObDDLArg), ddl_type_, schema_, policy_name_);
 OB_SERIALIZE_MEMBER((ObLabelSeUserLevelDDLArg, ObDDLArg), ddl_type_, level_schema_, policy_name_);
-OB_SERIALIZE_MEMBER(ObCheckServerEmptyArg, mode_);
+OB_SERIALIZE_MEMBER(ObCheckServerEmptyArg, mode_, sys_data_version_);
 OB_SERIALIZE_MEMBER(ObCheckDeploymentModeArg, single_zone_deployment_on_);
 
 OB_SERIALIZE_MEMBER(ObArchiveLogArg, enable_, tenant_id_, archive_tenant_ids_);
@@ -7201,5 +7227,37 @@ int ObFetchLocationResult::set_servers(
   return servers_.assign(servers);
 }
 
+OB_SERIALIZE_MEMBER(ObInitTenantConfigArg, tenant_configs_);
+
+int ObInitTenantConfigArg::assign(const ObInitTenantConfigArg &other)
+{
+  int ret = OB_SUCCESS;
+  if (this == &other) {
+  } else if (OB_FAIL(tenant_configs_.assign(other.tenant_configs_))) {
+    LOG_WARN("fail to assign tenant configs", KR(ret), K(other));
+  }
+  return ret;
+}
+
+int ObInitTenantConfigArg::add_tenant_config(const ObTenantConfigArg &arg)
+{
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(tenant_configs_.push_back(arg))) {
+    LOG_WARN("fail to push back config configs", KR(ret), K(arg));
+  }
+  return ret;
+}
+
+OB_SERIALIZE_MEMBER(ObInitTenantConfigRes, ret_);
+int ObInitTenantConfigRes::assign(const ObInitTenantConfigRes &other)
+{
+  int ret = OB_SUCCESS;
+  if (this != &other) {
+    ret_ = other.ret_;
+  }
+  return ret;
+}
+
+
 }//end namespace obrpc
 }//end namepsace oceanbase

src/share/ob_rpc_struct.h

@@ -64,6 +64,7 @@
 #include "logservice/palf/log_define.h"//INVALID_PROPOSAL_ID
 #include "share/scn.h"//SCN
 #include "share/location_cache/ob_vtable_location_service.h" // share::ObVtableLocationType
+#include "share/config/ob_config.h" // ObConfigArray
 #include "logservice/palf/log_meta_info.h"//LogConfigVersion
 
 namespace oceanbase
@@ -498,11 +499,12 @@ struct ObCreateTenantArg : public ObDDLArg
 public:
   ObCreateTenantArg()
     : ObDDLArg(), tenant_schema_(), pool_list_(), if_not_exist_(false),
-      sys_var_list_(), name_case_mode_(common::OB_NAME_CASE_INVALID), is_restore_(false),
-      palf_base_info_() {}
+      sys_var_list_(), name_case_mode_(common::OB_NAME_CASE_INVALID),
+      is_restore_(false), palf_base_info_(), compatible_version_(0) {}
   virtual ~ObCreateTenantArg() {};
   bool is_valid() const;
   int check_valid() const;
+  void reset();
   int assign(const ObCreateTenantArg &other);
 
   virtual bool is_allow_in_standby() const { return sync_from_primary_; }
@@ -521,6 +523,8 @@ public:
   bool is_restore_;
   //for restore tenant sys ls
   palf::PalfBaseInfo palf_base_info_;
+  //for restore tenant, from backuped meta file
+  uint64_t compatible_version_;
 };
 
 struct ObCreateTenantEndArg : public ObDDLArg
@@ -4523,15 +4527,23 @@ public:
     STOP_UPGRADE_JOB,
     UPGRADE_SYSTEM_VARIABLE,
     UPGRADE_SYSTEM_TABLE,
+    UPGRADE_BEGIN,
+    UPGRADE_VIRTUAL_SCHEMA,
+    UPGRADE_SYSTEM_PACKAGE,
+    UPGRADE_ALL_POST_ACTION,
+    UPGRADE_INSPECTION,
+    UPGRADE_END,
+    UPGRADE_ALL
   };
 public:
   ObUpgradeJobArg();
   bool is_valid() const;
   int assign(const ObUpgradeJobArg &other);
-  TO_STRING_KV(K_(action), K_(version));
+  TO_STRING_KV(K_(action), K_(version), K_(tenant_ids));
 public:
   Action action_;
   int64_t version_;
+  ObSArray<uint64_t> tenant_ids_; // empty means all tenant
 };
 
 struct ObUpgradeTableSchemaArg : public ObDDLArg
@@ -4541,17 +4553,24 @@ public:
   ObUpgradeTableSchemaArg()
     : ObDDLArg(),
       tenant_id_(common::OB_INVALID_TENANT_ID),
-      table_id_(common::OB_INVALID_ID) {}
+      table_id_(common::OB_INVALID_ID),
+      upgrade_virtual_schema_(false) {}
   ~ObUpgradeTableSchemaArg() {}
-  int init(const uint64_t tenant_id, const uint64_t table_id);
+  int init(const uint64_t tenant_id,
+           const uint64_t table_id,
+           const bool upgrade_virtual_schema);
   bool is_valid() const;
   int assign(const ObUpgradeTableSchemaArg &other);
   uint64_t get_tenant_id() const { return tenant_id_; }
   uint64_t get_table_id() const { return table_id_; }
-  TO_STRING_KV(K_(tenant_id), K_(table_id));
+  bool upgrade_virtual_schema() const { return upgrade_virtual_schema_; }
+  TO_STRING_KV(K_(tenant_id), K_(table_id), K_(upgrade_virtual_schema));
 private:
   uint64_t tenant_id_;
+  // 1. upgrade_virtual_schema_ = false, table_id_ should be sys table(except sys index/lob table).
+  // 2. upgrade_virtual_schema_ = true, table_id_ is invalid.
   uint64_t table_id_;
+  bool upgrade_virtual_schema_;
 };
 
 struct ObAdminMergeArg
@@ -6742,9 +6761,13 @@ public:
     ADD_SERVER
   };
 
-  ObCheckServerEmptyArg(): mode_(BOOTSTRAP) {}
-  TO_STRING_KV(K_(mode));
+  ObCheckServerEmptyArg(): mode_(BOOTSTRAP), sys_data_version_(0) {}
+  ObCheckServerEmptyArg(const Mode mode,
+                        const uint64_t sys_data_version)
+    : mode_(mode), sys_data_version_(sys_data_version) {}
+  TO_STRING_KV(K_(mode), K_(sys_data_version));
   Mode mode_;
+  uint64_t sys_data_version_;
 };
 
 struct ObArchiveLogArg
@@ -8068,6 +8091,35 @@ private:
   common::ObSArray<common::ObAddr> servers_;
 };
 
+struct ObInitTenantConfigArg
+{
+  OB_UNIS_VERSION(1);
+public:
+  ObInitTenantConfigArg() : tenant_configs_() {}
+  ~ObInitTenantConfigArg() {}
+  bool is_valid() const { return tenant_configs_.count() > 0; }
+  int assign(const ObInitTenantConfigArg &other);
+  int add_tenant_config(const ObTenantConfigArg &arg);
+  const common::ObSArray<ObTenantConfigArg> &get_tenant_configs() const { return tenant_configs_; }
+  TO_STRING_KV(K_(tenant_configs));
+private:
+  common::ObSArray<ObTenantConfigArg> tenant_configs_;
+};
+
+struct ObInitTenantConfigRes
+{
+  OB_UNIS_VERSION(1);
+public:
+  ObInitTenantConfigRes() : ret_(common::OB_ERROR) {}
+  ~ObInitTenantConfigRes() {}
+  int assign(const ObInitTenantConfigRes &other);
+  void set_ret(int ret) { ret_ = ret; }
+  int64_t get_ret() const { return ret_; }
+  TO_STRING_KV(K_(ret));
+private:
+  int ret_;
+};
+
 }//end namespace obrpc
 }//end namespace oceanbase
 #endif

src/share/ob_srv_rpc_proxy.h

@@ -50,6 +50,8 @@ public:
   RPC_AP(PR5 check_frozen_scn, OB_CHECK_FROZEN_SCN, (obrpc::ObCheckFrozenScnArg));
   RPC_AP(PR5 get_min_sstable_schema_version, OB_GET_MIN_SSTABLE_SCHEMA_VERSION,
       (obrpc::ObGetMinSSTableSchemaVersionArg), obrpc::ObGetMinSSTableSchemaVersionRes);
+  RPC_AP(PR5 init_tenant_config, OB_INIT_TENANT_CONFIG,
+      (obrpc::ObInitTenantConfigArg), obrpc::ObInitTenantConfigRes);
   RPC_S(PR5 fetch_sys_ls, OB_FETCH_SYS_LS,
         share::ObLSReplica);
   RPC_S(PR5 broadcast_rs_list, OB_BROADCAST_RS_LIST, (obrpc::ObRsListArg));

src/share/ob_standby_upgrade.cpp

@@ -0,0 +1,27 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#define USING_LOG_PREFIX SHARE
+
+#include "ob_standby_upgrade.h"
+
+using namespace oceanbase;
+using namespace oceanbase::common;
+namespace oceanbase
+{
+namespace share
+{
+
+OB_SERIALIZE_MEMBER(ObStandbyUpgrade, data_version_);
+
+}
+}

src/share/ob_standby_upgrade.h

@@ -0,0 +1,50 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#ifndef OCEANBASE_SHARE_OB_STANDBY_UPGRADE_H_
+#define OCEANBASE_SHARE_OB_STANDBY_UPGRADE_H_
+
+#include "lib/utility/ob_print_utils.h"       // Print*
+#include "lib/utility/ob_unify_serialize.h"       // OB_UNIS_VERSION
+#include "share/ob_cluster_version.h"
+
+namespace oceanbase
+{
+
+namespace share
+{
+
+struct ObStandbyUpgrade
+{
+  OB_UNIS_VERSION(1);
+ public:
+  ObStandbyUpgrade(): data_version_(0) {}
+  ObStandbyUpgrade(const uint64_t data_version): data_version_(data_version) {}
+  ~ObStandbyUpgrade() {}
+  bool is_valid() const
+  {
+    return ObClusterVersion::check_version_valid_(data_version_);
+  }
+  uint64_t get_data_version() const
+  {
+    return data_version_;
+  }
+
+  TO_STRING_KV(K_(data_version));
+private:
+  uint64_t data_version_;
+};
+
+}
+}
+
+#endif /* !OCEANBASE_SHARE_OB_STANDBY_UPGRADE_H_ */

src/share/ob_upgrade_utils.cpp

@@ -34,9 +34,32 @@ using namespace sql;
 namespace share
 {
 const uint64_t ObUpgradeChecker::UPGRADE_PATH[DATA_VERSION_NUM] = {
+  CALC_VERSION(4UL, 0UL, 0UL, 0UL),  // 4.0.0.0
   CALC_VERSION(4UL, 1UL, 0UL, 0UL)   // 4.1.0.0
 };
 
+int ObUpgradeChecker::get_data_version_by_cluster_version(
+    const uint64_t cluster_version,
+    uint64_t &data_version)
+{
+  int ret = OB_SUCCESS;
+  switch (cluster_version) {
+    case CLUSTER_VERSION_4_0_0_0: {
+      data_version = DATA_VERSION_4_0_0_0;
+      break;
+    }
+    case CLUSTER_VERSION_4_1_0_0: {
+      data_version = DATA_VERSION_4_1_0_0;
+      break;
+    }
+    default: {
+      ret = OB_INVALID_ARGUMENT;
+      LOG_WARN("invalid cluster_version", KR(ret), K(cluster_version));
+    }
+  }
+  return ret;
+}
+
 bool ObUpgradeChecker::check_data_version_exist(
      const uint64_t version)
 {
@@ -189,7 +212,10 @@ int ObUpgradeUtils::check_rs_job_success(ObRsJobType job_type, bool &success)
   return ret;
 }
 
-int ObUpgradeUtils::check_schema_sync(bool &is_sync)
+// tenant_id == OB_INVALID_TENANT_ID: check all tenants' schema statuses
+int ObUpgradeUtils::check_schema_sync(
+    const uint64_t tenant_id,
+    bool &is_sync)
 {
   int ret = OB_SUCCESS;
   ObSqlString sql;
@@ -199,26 +225,32 @@ int ObUpgradeUtils::check_schema_sync(bool &is_sync)
     is_sync = false;
     if (OB_ISNULL(GCTX.sql_proxy_)) {
       ret = OB_ERR_UNEXPECTED;
-      LOG_WARN("sql_proxy is null", K(ret));
-    } else if (sql.assign_fmt("SELECT floor(count(*)) as count FROM %s AS a "
-                              "JOIN %s AS b ON a.tenant_id = b.tenant_id "
-                              "WHERE a.refreshed_schema_version != b.refreshed_schema_version",
-                              OB_ALL_VIRTUAL_SERVER_SCHEMA_INFO_TNAME,
-                              OB_ALL_VIRTUAL_SERVER_SCHEMA_INFO_TNAME)) {
-      LOG_WARN("fail to assign sql", K(ret));
+      LOG_WARN("sql_proxy is null", KR(ret));
+    } else if (OB_FAIL(sql.assign_fmt(
+               "SELECT floor(count(*)) as count FROM %s AS a "
+               "JOIN %s AS b ON a.tenant_id = b.tenant_id "
+               "WHERE (a.refreshed_schema_version != b.refreshed_schema_version "
+               "       OR (a.refreshed_schema_version mod %ld) != 0) ",
+               OB_ALL_VIRTUAL_SERVER_SCHEMA_INFO_TNAME,
+               OB_ALL_VIRTUAL_SERVER_SCHEMA_INFO_TNAME,
+               schema::ObSchemaService::SCHEMA_VERSION_INC_STEP))) {
+      LOG_WARN("fail to assign sql", KR(ret));
+    } else if (OB_INVALID_TENANT_ID != tenant_id
+               && OB_FAIL(sql.append_fmt(" AND a.tenant_id = %ld", tenant_id))) {
+      LOG_WARN("fail to append sql", KR(ret), K(tenant_id));
     } else if (OB_FAIL(GCTX.sql_proxy_->read(res, sql.ptr()))) {
-      LOG_WARN("fail to execute sql", K(ret), K(sql));
+      LOG_WARN("fail to execute sql", KR(ret), K(sql));
     } else if (NULL == (result = res.get_result())) {
       ret = OB_ERR_UNEXPECTED;
       LOG_WARN("fail to get sql result", K(ret));
     } else if (OB_FAIL((result->next()))) {
-      LOG_WARN("fail to get result", K(ret));
+      LOG_WARN("fail to get result", KR(ret));
     } else {
       EXTRACT_INT_FIELD_MYSQL(*result, "count", count, int32_t);
       if (OB_FAIL(ret)) {
       } else if (count < 0) {
         ret = OB_ERR_UNEXPECTED;
-        LOG_WARN("invalid count", K(ret), K(count));
+        LOG_WARN("invalid count", KR(ret), K(count));
       } else {
         is_sync = (count == 0);
       }
@@ -562,6 +594,7 @@ int ObUpgradeProcesserSet::init(
     }
 
     // order by data version asc
+    INIT_PROCESSOR_BY_VERSION(4, 0, 0, 0);
     INIT_PROCESSOR_BY_VERSION(4, 1, 0, 0);
 #undef INIT_PROCESSOR_BY_VERSION
     inited_ = true;

src/share/ob_upgrade_utils.h

@@ -37,7 +37,7 @@ public:
   virtual ~ObUpgradeUtils() {}
   static int can_run_upgrade_job(rootserver::ObRsJobType job_type, bool &can);
   static int check_upgrade_job_passed(rootserver::ObRsJobType job_type);
-  static int check_schema_sync(bool &is_sync);
+  static int check_schema_sync(const uint64_t tenant_id, bool &is_sync);
   // upgrade_sys_variable()/upgrade_sys_stat() can be called when enable_ddl = false.
   static int upgrade_sys_variable(
              obrpc::ObCommonRpcProxy &rpc_proxy,
@@ -165,12 +165,16 @@ class ObUpgradeChecker
 {
 public:
   static bool check_data_version_exist(const uint64_t version);
+  static int get_data_version_by_cluster_version(
+             const uint64_t cluster_version,
+             uint64_t &data_version);
 public:
-  static const int64_t DATA_VERSION_NUM = 1;
+  static const int64_t DATA_VERSION_NUM = 2;
   static const uint64_t UPGRADE_PATH[DATA_VERSION_NUM];
 };
 
 /* =========== special upgrade processor start ============= */
+DEF_SIMPLE_UPGRARD_PROCESSER(4, 0, 0, 0)
 class ObUpgradeFor4100Processor : public ObBaseUpgradeProcessor
 {
 public:

src/share/parameter/ob_parameter_macro.h

@@ -122,6 +122,9 @@ access_specifier:
 #define DEF_STR(args...)                                                       \
   _DEF_PARAMETER_SCOPE_EASY(public, String, args)
 
+#define DEF_VERSION(args...)                                                   \
+  _DEF_PARAMETER_SCOPE_EASY(public, Version, args)
+
 #define DEF_STR_WITH_CHECKER(args...)                                          \
   _DEF_PARAMETER_SCOPE_CHECKER_EASY(public, String, args)
 

src/share/parameter/ob_parameter_seed.ipp

@@ -459,8 +459,8 @@ DEF_TIME(tablet_meta_table_check_interval, OB_CLUSTER_PARAMETER, "30m", "[1m,)",
          ObParameterAttr(Section::ROOT_SERVICE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
 DEF_STR(min_observer_version, OB_CLUSTER_PARAMETER, "4.1.0.0", "the min observer version",
         ObParameterAttr(Section::ROOT_SERVICE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
-DEF_STR(compatible, OB_TENANT_PARAMETER, "4.1.0.0", "compatible version for persisted data",
-        ObParameterAttr(Section::ROOT_SERVICE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
+DEF_VERSION(compatible, OB_TENANT_PARAMETER, "4.1.0.0", "compatible version for persisted data",
+            ObParameterAttr(Section::ROOT_SERVICE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
 DEF_BOOL(enable_ddl, OB_CLUSTER_PARAMETER, "True", "specifies whether DDL operation is turned on. "
          "Value:  True:turned on;  False: turned off",
          ObParameterAttr(Section::ROOT_SERVICE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));

src/share/restore/ob_physical_restore_info.h

@@ -160,6 +160,7 @@ public:
 
   Property_declare_int(int64_t, restore_start_ts)
   Property_declare_int(share::SCN, restore_scn)
+  // TODO(chongrong.th): should use tenant's data version
   Property_declare_int(uint64_t, post_cluster_version)
   Property_declare_int(uint64_t, source_cluster_version)
   //from cmd

src/share/schema/ob_error_info.cpp

@@ -155,12 +155,8 @@ int ObErrorInfo::collect_error_info(const IObErrorInfo *info)
     LOG_WARN("collect error info failed", K(ret));
   } else {
     const ObWarningBuffer *warnings_buf = common::ob_get_tsi_warning_buffer();
-    uint16_t wcnt = 0;
-    if (OB_ISNULL(warnings_buf)) {
-      ret = OB_ERR_SYS;
-      LOG_WARN("can not get thread warnings buffer");
-    } else {
-      wcnt = static_cast<uint16_t>(warnings_buf->get_readable_warning_count());
+    if (OB_NOT_NULL(warnings_buf)) {
+      uint16_t wcnt = static_cast<uint16_t>(warnings_buf->get_readable_warning_count());
       if (OB_FAIL(collect_error_info(info, warnings_buf, wcnt > 0))) {
         LOG_WARN("failed to fill error info", K(ret), K(*this));
       } else {

src/sql/parser/sql_parser_mysql_mode.y

@@ -13982,9 +13982,9 @@ ALTER SYSTEM upgrade_action UPGRADE
   malloc_non_terminal_node($$, result->malloc_pool_, T_ADMIN_UPGRADE_CMD, 1, $3);
 }
 |
-ALTER SYSTEM RUN UPGRADE JOB STRING_VALUE
+ALTER SYSTEM RUN UPGRADE JOB STRING_VALUE opt_tenant_list_v2
 {
-  malloc_non_terminal_node($$, result->malloc_pool_, T_ADMIN_RUN_UPGRADE_JOB, 1, $6);
+  malloc_non_terminal_node($$, result->malloc_pool_, T_ADMIN_RUN_UPGRADE_JOB, 2, $6, $7);
 }
 |
 ALTER SYSTEM STOP UPGRADE JOB

src/sql/resolver/cmd/ob_alter_system_resolver.cpp

@@ -2587,26 +2587,51 @@ int ObRunUpgradeJobResolver::resolve(const ParseNode &parse_tree)
     ret = OB_ERR_UNEXPECTED;
     LOG_WARN("type is not T_ADMIN_RUN_UPGRADE_JOB",
              KR(ret), "type", get_type_name(parse_tree.type_));
-  } else if (OB_UNLIKELY(NULL == parse_tree.children_)) {
+  } else if (OB_ISNULL(parse_tree.children_)) {
     ret = OB_ERR_UNEXPECTED;
     LOG_WARN("children should not be null", KR(ret));
+  } else if (OB_ISNULL(session_info_)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("session info is null", KR(ret));
   } else {
     ObRunUpgradeJobStmt *stmt = create_stmt<ObRunUpgradeJobStmt>();
-    if (NULL == stmt) {
+    if (OB_ISNULL(stmt)) {
       ret = OB_ALLOCATE_MEMORY_FAILED;
       LOG_ERROR("create ObRunUpgradeJobStmt failed", KR(ret));
     } else {
       stmt_ = stmt;
+      // 1. parse action
       ObString str;
       uint64_t version = OB_INVALID_VERSION;
       if (OB_FAIL(Util::resolve_string(parse_tree.children_[0], str))) {
         LOG_WARN("resolve string failed", KR(ret));
+      } else if (0 == str.case_compare(rootserver::ObRsJobTableOperator::get_job_type_str(
+                 rootserver::JOB_TYPE_UPGRADE_BEGIN))) {
+        stmt->get_rpc_arg().action_ = obrpc::ObUpgradeJobArg::UPGRADE_BEGIN;
       } else if (0 == str.case_compare(rootserver::ObRsJobTableOperator::get_job_type_str(
                  rootserver::JOB_TYPE_UPGRADE_SYSTEM_VARIABLE))) {
         stmt->get_rpc_arg().action_ = obrpc::ObUpgradeJobArg::UPGRADE_SYSTEM_VARIABLE;
       } else if (0 == str.case_compare(rootserver::ObRsJobTableOperator::get_job_type_str(
                  rootserver::JOB_TYPE_UPGRADE_SYSTEM_TABLE))) {
         stmt->get_rpc_arg().action_ = obrpc::ObUpgradeJobArg::UPGRADE_SYSTEM_TABLE;
+      } else if (0 == str.case_compare(rootserver::ObRsJobTableOperator::get_job_type_str(
+                 rootserver::JOB_TYPE_UPGRADE_VIRTUAL_SCHEMA))) {
+        stmt->get_rpc_arg().action_ = obrpc::ObUpgradeJobArg::UPGRADE_VIRTUAL_SCHEMA;
+      } else if (0 == str.case_compare(rootserver::ObRsJobTableOperator::get_job_type_str(
+                 rootserver::JOB_TYPE_UPGRADE_SYSTEM_PACKAGE))) {
+        stmt->get_rpc_arg().action_ = obrpc::ObUpgradeJobArg::UPGRADE_SYSTEM_PACKAGE;
+      } else if (0 == str.case_compare(rootserver::ObRsJobTableOperator::get_job_type_str(
+                 rootserver::JOB_TYPE_UPGRADE_ALL_POST_ACTION))) {
+        stmt->get_rpc_arg().action_ = obrpc::ObUpgradeJobArg::UPGRADE_ALL_POST_ACTION;
+      } else if (0 == str.case_compare(rootserver::ObRsJobTableOperator::get_job_type_str(
+                 rootserver::JOB_TYPE_UPGRADE_INSPECTION))) {
+        stmt->get_rpc_arg().action_ = obrpc::ObUpgradeJobArg::UPGRADE_INSPECTION;
+      } else if (0 == str.case_compare(rootserver::ObRsJobTableOperator::get_job_type_str(
+                 rootserver::JOB_TYPE_UPGRADE_END))) {
+        stmt->get_rpc_arg().action_ = obrpc::ObUpgradeJobArg::UPGRADE_END;
+      } else if (0 == str.case_compare(rootserver::ObRsJobTableOperator::get_job_type_str(
+                 rootserver::JOB_TYPE_UPGRADE_ALL))) {
+        stmt->get_rpc_arg().action_ = obrpc::ObUpgradeJobArg::UPGRADE_ALL;
       } else {
         // UPGRADE_POST_ACTION
         if (OB_FAIL(ObClusterVersion::get_version(str, version))) {
@@ -2616,6 +2641,38 @@ int ObRunUpgradeJobResolver::resolve(const ParseNode &parse_tree)
           stmt->get_rpc_arg().version_ = static_cast<int64_t>(version);
         }
       }
+      // 2. parse tenant_ids
+      if (OB_SUCC(ret)
+          && parse_tree.num_child_ >= 2
+          && OB_NOT_NULL(parse_tree.children_[1])) {
+        ParseNode *tenants_node = parse_tree.children_[1];
+        bool affect_all = false;
+        const int64_t child_num = tenants_node->num_child_;
+        const uint64_t cur_tenant_id = session_info_->get_effective_tenant_id();
+        ObSArray<uint64_t> &tenant_ids = stmt->get_rpc_arg().tenant_ids_;
+        if (T_TENANT_LIST != tenants_node->type_) {
+          ret = OB_ERR_UNEXPECTED;
+          LOG_WARN("type is not T_TENANT_LIST", KR(ret),
+                   "type", get_type_name(tenants_node->type_));
+        } else if (OB_ISNULL(tenants_node->children_)
+                   || OB_UNLIKELY(0 == child_num)) {
+          ret = OB_ERR_UNEXPECTED;
+          LOG_WARN("children of tenant should not be null", KR(ret), K(child_num));
+        } else if (OB_FAIL(Util::resolve_tenant(
+                   *tenants_node, cur_tenant_id, tenant_ids, affect_all))) {
+          LOG_WARN("fail to resolve tenant", KR(ret), K(cur_tenant_id));
+        } else if (affect_all && 0 != tenant_ids.count()) {
+          ret = OB_ERR_UNEXPECTED;
+          LOG_WARN("tenant_ids should be empty when specify tenant = all", KR(ret));
+        } else if (obrpc::ObUpgradeJobArg::UPGRADE_SYSTEM_PACKAGE == stmt->get_rpc_arg().action_) {
+          if ((tenant_ids.count() > 1)
+               || (1 == tenant_ids.count() && !is_sys_tenant(tenant_ids.at(0)))) {
+            ret = OB_NOT_SUPPORTED;
+            LOG_WARN("Only system tenant can run UPGRADE_SYSTEM_PACKAGE Job", KR(ret), K(tenant_ids));
+            LOG_USER_ERROR(OB_NOT_SUPPORTED, "Non-sys tenant run UPGRADE_SYSTEM_PACKAGE job");
+          }
+        }
+      }
     }
   }
   return ret;

src/storage/blocksstable/encoding/ob_micro_block_encoder.cpp

@@ -349,7 +349,7 @@ void ObMicroBlockEncoder::update_estimate_size_limit(const ObMicroBlockEncodingC
                                   ? (2 * ctx.micro_block_size_ - header_size_)
                                   : (2 * ctx.micro_block_size_);
 //TODO huronghui.hrh@oceanbase.com use 4.1.0.0 for version judgment
-  if(ctx.major_working_cluster_version_ >= CLUSTER_VERSION_4_1_0_0 ) {
+  if(ctx.major_working_cluster_version_ >= DATA_VERSION_4_1_0_0 ) {
     data_size_limit = MAX(data_size_limit, DEFAULT_MICRO_MAX_SIZE);
   }
   estimate_size_limit_ = std::min(data_size_limit * expand_pct_ / 100, ctx.macro_block_size_);

src/storage/blocksstable/ob_index_block_builder.cpp

@@ -863,7 +863,7 @@ int ObSSTableIndexBuilder::close(const int64_t column_cnt, ObSSTableMergeRes &re
   } else if (OB_FAIL(sort_roots())) {
     STORAGE_LOG(WARN, "fail to sort roots", K(ret));
   } else if (!index_store_desc_.is_major_merge()
-      || index_store_desc_.major_working_cluster_version_ >= CLUSTER_VERSION_4_1_0_0) {
+      || index_store_desc_.major_working_cluster_version_ >= DATA_VERSION_4_1_0_0) {
     const bool is_single_block = check_single_block();
     if (is_single_block) {
       switch (optimization_mode_) {

src/storage/blocksstable/ob_macro_block.cpp

@@ -239,7 +239,7 @@ int ObDataStoreDesc::init(
       STORAGE_LOG(WARN, "Failed to generate multi version column ids", K(ret));
     } else if (OB_FAIL(datum_utils_.init(col_desc_array_, schema_rowkey_col_cnt_, lib::is_oracle_mode(), allocator_))) {
       STORAGE_LOG(WARN, "Failed to init datum utils", K(ret));
-    } else if (is_major && major_working_cluster_version_ <= CLUSTER_VERSION_4_0_0_0) {
+    } else if (is_major && major_working_cluster_version_ <= DATA_VERSION_4_0_0_0) {
       micro_block_size_ = merge_schema.get_block_size();
     } else {
       micro_block_size_ = MAX(merge_schema.get_block_size(), MIN_MICRO_BLOCK_SIZE);

src/storage/blocksstable/ob_macro_block_writer.cpp

@@ -472,7 +472,7 @@ int ObMacroBlockWriter::open(
       } else {
         //TODO huronghui.hrh@oceanbase.com use 4.1.0.0 for version judgment
         const bool is_use_adaptive = !data_store_desc_->is_major_merge()
-         || data_store_desc_->major_working_cluster_version_ >= CLUSTER_VERSION_4_1_0_0;
+         || data_store_desc_->major_working_cluster_version_ >= DATA_VERSION_4_1_0_0;
         if (OB_FAIL(micro_block_adaptive_splitter_.init(data_store_desc.macro_store_size_, is_use_adaptive))) {
           STORAGE_LOG(WARN, "Failed to init micro block adaptive split", K(ret), K(data_store_desc.macro_store_size_));
         }

src/storage/compaction/ob_partition_merger.cpp

@@ -130,7 +130,7 @@ int ObPartitionMerger::open_macro_writer(ObMergeParameter &merge_param)
     STORAGE_LOG(WARN, "failed to set sstable seq", K(ret), K(merge_param.sstable_logic_seq_));
   } else {
     data_store_desc_.end_scn_ = merge_ctx_->scn_range_.end_scn_;
-    if ((data_store_desc_.is_major_merge() && data_store_desc_.major_working_cluster_version_ <= CLUSTER_VERSION_4_0_0_0)
+    if ((data_store_desc_.is_major_merge() && data_store_desc_.major_working_cluster_version_ <= DATA_VERSION_4_0_0_0)
       || !data_store_desc_.is_use_pct_free()) {
       macro_writer_ = alloc_merge_helper<ObMacroBlockWriter>();
     } else {

src/storage/compaction/ob_tenant_freeze_info_mgr.cpp

@@ -768,7 +768,7 @@ int ObTenantFreezeInfoMgr::ReloadTask::get_freeze_info(
     for (int64_t i = 0; i < tmp.count() && OB_SUCC(ret); ++i) {
       ObSimpleFrozenStatus &status = tmp.at(i);
       if (OB_FAIL(freeze_info.push_back(
-            FreezeInfo(status.frozen_scn_.get_val_for_tx(), status.schema_version_, status.cluster_version_)))) {
+            FreezeInfo(status.frozen_scn_.get_val_for_tx(), status.schema_version_, status.data_version_)))) {
         STORAGE_LOG(WARN, "fail to push back freeze info", K(ret), K(status));
       }
     }

src/storage/tx/ob_multi_data_source.cpp

@@ -20,6 +20,7 @@
 #include "storage/tablelock/ob_table_lock_common.h"
 #include "storage/tablet/ob_tablet_binding_helper.h"
 #include "share/ls/ob_ls_operator.h"
+#include "share/ob_standby_upgrade.h"  // ObStandbyUpgrade
 
 namespace oceanbase
 {
@@ -162,6 +163,10 @@ int ObMulSourceTxDataNotifier::notify(const ObTxBufferNodeArray &array,
         ret = notify_modify_tablet_binding(notify_type, buf, len, tmp_notify_arg);
         break;
       }
+      case ObTxDataSourceType::STANDBY_UPGRADE: {
+        ret = notify_standby_upgrade(notify_type, buf, len, tmp_notify_arg);
+        break;
+      }
       default: {
         ret = OB_ERR_UNEXPECTED;
         break;
@@ -268,6 +273,30 @@ int ObMulSourceTxDataNotifier::notify_ls_table(const NotifyType type,
   return ret;
 }
 
+int ObMulSourceTxDataNotifier::notify_standby_upgrade(const NotifyType type,
+                                               const char *buf, const int64_t len,
+                                               const ObMulSourceDataNotifyArg &arg)
+{
+  int ret = OB_SUCCESS;
+  share::ObStandbyUpgrade data_version;
+  int64_t pos = 0;
+
+  if (OB_ISNULL(buf) || OB_UNLIKELY(len <= 0)) {
+    ret = OB_INVALID_ARGUMENT;
+    TRANS_LOG(WARN, "invalid argument", KR(ret), K(buf), K(len), K(type));
+  } else if (OB_FAIL(data_version.deserialize(buf, len, pos))) {
+    TRANS_LOG(WARN, "failed to deserialize data_version", KR(ret), K(buf), K(len));
+  } else if (pos > len) {
+    ret = OB_ERR_UNEXPECTED;
+    TRANS_LOG(WARN, "deserialize error", KR(ret), K(pos), K(len));
+  }
+  TRANS_LOG(INFO, "standby upgrade notify", KR(ret), K(data_version), K(len), K(type));
+
+  ob_abort_log_cb_notify_(type, ret, arg.for_replay_);
+
+  return ret;
+}
+
 int ObMulSourceTxDataNotifier::notify_create_tablet(const NotifyType type,
                                                       const char *buf, const int64_t len,
                                                       const ObMulSourceDataNotifyArg & arg)

src/storage/tx/ob_multi_data_source.h

@@ -57,6 +57,8 @@ enum class ObTxDataSourceType : int64_t
   DDL_TRANS = 6,
   // for unbind hidden tablet and reuse index tablet
   MODIFY_TABLET_BINDING = 7,
+  // for standby upgrade
+  STANDBY_UPGRADE = 8,
   MAX_TYPE = 100
 };
 
@@ -172,6 +174,9 @@ private:
   static int notify_ls_table(const NotifyType type,
                              const char *buf, const int64_t len,
                              const ObMulSourceDataNotifyArg &arg);
+  static int notify_standby_upgrade(const NotifyType type,
+                             const char *buf, const int64_t len,
+                             const ObMulSourceDataNotifyArg &arg);
   static int notify_create_tablet(const NotifyType type,
                                   const char *buf, const int64_t len,
                                   const ObMulSourceDataNotifyArg &arg);

tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_upgrade_inspection.result

@@ -8,4 +8,5 @@ zone_check	succeed
 sys_stat_check	succeed
 sys_param_check	succeed
 sys_table_schema_check	succeed
+data_version_check	succeed
 all_check	succeed

tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result

@@ -216,6 +216,7 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr
 412	__all_service_epoch	0	201001	1
 413	__all_spatial_reference_systems	0	201001	1
 416	__all_column_checksum_error_info	0	201001	1
+445	__all_cluster_event_history	0	201001	1
 10001	__tenant_virtual_all_table	2	201001	1
 10002	__tenant_virtual_table_column	2	201001	1
 10003	__tenant_virtual_table_index	2	201001	1
@@ -556,6 +557,7 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr
 12337	__all_virtual_schema_slot	2	201001	1
 12338	__all_virtual_minor_freeze_info	2	201001	1
 12340	__all_virtual_ha_diagnose	2	201001	1
+12362	__all_virtual_core_table	2	201001	1
 20001	GV$OB_PLAN_CACHE_STAT	1	201001	1
 20002	GV$OB_PLAN_CACHE_PLAN_STAT	1	201001	1
 20003	SCHEMATA	1	201002	1
@@ -816,6 +818,7 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr
 21337	CDB_OB_USERS	1	201001	1
 21338	DBA_OB_DATABASE_PRIVILEGE	1	201001	1
 21339	CDB_OB_DATABASE_PRIVILEGE	1	201001	1
+21345	DBA_OB_CLUSTER_EVENT_HISTORY	1	201001	1
 check sys table count and table_id range success
 check count and table_id range for virtual table success
 select * from  information_schema.CHARACTER_SETS limit 1;

tools/upgrade/actions.py

@@ -18,9 +18,17 @@ class SqlItem:
     self.action_sql = action_sql
     self.rollback_sql = rollback_sql
 
+current_cluster_version = "4.1.0.0"
+current_data_version = "4.1.0.0"
 g_succ_sql_list = []
 g_commit_sql_list = []
 
+def get_current_cluster_version():
+  return current_cluster_version
+
+def get_current_data_version():
+  return current_data_version
+
 def refresh_commit_sql_list():
   global g_succ_sql_list
   global g_commit_sql_list
@@ -97,15 +105,15 @@ def check_is_update_sql(sql):
         and word_list[1].lower().startswith('@') and ':=' == word_list[2].lower()):
       raise MyError('sql must be update, key_word="{0}", sql="{1}"'.format(key_word, sql))
 
-def set_parameter(cur, parameter, value):
+def set_parameter(cur, parameter, value, timeout = 0):
   sql = """alter system set {0} = '{1}'""".format(parameter, value)
   logging.info(sql)
   cur.execute(sql)
-  wait_parameter_sync(cur, parameter, value)
+  wait_parameter_sync(cur, parameter, value, timeout)
 
-def get_ori_enable_ddl(cur):
+def get_ori_enable_ddl(cur, timeout):
   ori_value_str = fetch_ori_enable_ddl(cur)
-  wait_parameter_sync(cur, 'enable_ddl', ori_value_str)
+  wait_parameter_sync(cur, 'enable_ddl', ori_value_str, timeout)
   ori_value = (ori_value_str == 'True')
   return ori_value
 
@@ -132,11 +140,63 @@ def fetch_ori_enable_ddl(cur):
     raise e
   return ori_value
 
-def wait_parameter_sync(cur, key, value):
-  sql = """select count(*) as cnt from oceanbase.__all_virtual_sys_parameter_stat
+# print version like "x.x.x.x"
+def print_version(version):
+  version = int(version)
+  major = (version >> 32) & 0xffffffff
+  minor = (version >> 16) & 0xffff
+  major_patch = (version >> 8) & 0xff
+  minor_patch = version & 0xff
+  version_str = "{0}.{1}.{2}.{3}".format(major, minor, major_patch, minor_patch)
+
+# version str should like "x.x.x.x"
+def get_version(version_str):
+  versions = version_str.split(".")
+
+  if len(versions) != 4:
+    logging.exception("""version:{0} is invalid""".format(version_str))
+    raise e
+
+  major = int(versions[0])
+  minor = int(versions[1])
+  major_patch = int(versions[2])
+  minor_patch = int(versions[3])
+
+  if major > 0xffffffff or minor > 0xffff or major_patch > 0xff or minor_patch > 0xff:
+    logging.exception("""version:{0} is invalid""".format(version_str))
+    raise e
+
+  version = (major << 32) | (minor << 16) | (major_patch << 8) | (minor_patch)
+  return version
+
+def check_server_version_by_cluster(cur):
+  sql = """select distinct(substring_index(build_version, '_', 1)) from __all_server""";
+  logging.info(sql)
+  cur.execute(sql)
+  result = cur.fetchall()
+  if len(result) != 1:
+    raise MyError("servers build_version not match")
+  else:
+    logging.info("check server version success")
+
+def check_parameter(cur, key, value):
+  sql = """select * from oceanbase.GV$OB_PARAMETERS
+           where name = '{0}' and value = '{1}'""".format(key, value)
+  logging.info(sql)
+  cur.execute(sql)
+  result = cur.fetchall()
+  bret = False
+  if len(result) > 0:
+    bret = True
+  else:
+    bret = False
+  return bret
+
+def wait_parameter_sync(cur, key, value, timeout):
+  sql = """select count(*) as cnt from oceanbase.GV$OB_PARAMETERS
            where name = '{0}' and value != '{1}'""".format(key, value)
-  times = 10
-  while times > 0:
+  times = (timeout if timeout > 0 else 60) / 5
+  while times >= 0:
     logging.info(sql)
     cur.execute(sql)
     result = cur.fetchall()
@@ -150,11 +210,66 @@ def wait_parameter_sync(cur, key, value):
       logging.info("""{0} is not sync, value should be {1}""".format(key, value))
 
     times -= 1
-    if times == 0:
+    if times == -1:
       logging.exception("""check {0}:{1} sync timeout""".format(key, value))
       raise e
     time.sleep(5)
 
+def do_begin_upgrade(cur, timeout):
+  action_sql = "alter system begin upgrade"
+  rollback_sql = "alter system end upgrade"
+
+  if not check_parameter(cur, "enable_upgrade_mode", "True"):
+    logging.info(action_sql)
+    cur.execute(action_sql)
+
+  wait_parameter_sync(cur, "enable_upgrade_mode", "True", timeout)
+
+  global g_succ_sql_list
+  g_succ_sql_list.append(SqlItem(action_sql, rollback_sql))
+
+def do_begin_rolling_upgrade(cur, timeout):
+  action_sql = "alter system begin rolling upgrade"
+  rollback_sql = "alter system end upgrade"
+
+  if not check_parameter(cur, "_upgrade_stage", "DBUPGRADE"):
+    logging.info(action_sql)
+    cur.execute(action_sql)
+
+  wait_parameter_sync(cur, "_upgrade_stage", "DBUPGRADE", timeout)
+
+  global g_succ_sql_list
+  g_succ_sql_list.append(SqlItem(action_sql, rollback_sql))
+
+def do_end_rolling_upgrade(cur, timeout):
+  current_cluster_version = get_current_cluster_version()
+
+  action_sql = "alter system end rolling upgrade"
+  rollback_sql = "alter system end upgrade"
+
+  if not check_parameter(cur, "_upgrade_stage", "POSTUPGRADE") or not check_parameter(cur, "min_observer_version", current_cluster_version):
+    logging.info(action_sql)
+    cur.execute(action_sql)
+
+  wait_parameter_sync(cur, "min_observer_version", current_data_version, timeout)
+  wait_parameter_sync(cur, "_upgrade_stage", "POSTUPGRADE", timeout)
+
+  global g_succ_sql_list
+  g_succ_sql_list.append(SqlItem(action_sql, rollback_sql))
+
+def do_end_upgrade(cur, timeout):
+  action_sql = "alter system end upgrade"
+  rollback_sql = ""
+
+  if not check_parameter(cur, "enable_upgrade_mode", "False"):
+    logging.info(action_sql)
+    cur.execute(action_sql)
+
+  wait_parameter_sync(cur, "enable_upgrade_mode", "False", timeout)
+
+  global g_succ_sql_list
+  g_succ_sql_list.append(SqlItem(action_sql, rollback_sql))
+
 class Cursor:
   __cursor = None
   def __init__(self, cursor):
@@ -269,14 +384,6 @@ class BaseEachTenantDMLAction():
     rollback_sql = self.get_each_tenant_rollback_sql(tenant_id)
     self.__dml_cursor.exec_update(action_sql)
     g_succ_sql_list.append(SqlItem(action_sql, rollback_sql))
-  def change_tenant(self, tenant_id):
-    self._cursor.exec_sql("alter system change tenant tenant_id = {0}".format(tenant_id))
-    (desc, results) = self._query_cursor.exec_query("select effective_tenant_id()")
-    if (1 != len(results) or 1 != len(results[0])):
-      raise MyError("results cnt not match")
-    elif (tenant_id != results[0][0]):
-      raise MyError("change tenant failed, effective_tenant_id:{0}, tenant_id:{1}"
-                    .format(results[0][0], tenant_id))
 
 class BaseEachTenantDDLAction():
   __dml_cursor = None
@@ -321,13 +428,16 @@ def reflect_action_cls_list(action_module, action_name_prefix):
   action_cls_list.sort(actions_cls_compare)
   return action_cls_list
 
-def fetch_observer_version(query_cur):
-  (desc, results) = query_cur.exec_query("""select distinct value from __all_virtual_sys_parameter_stat where name='min_observer_version'""")
-  if len(results) != 1:
+def fetch_observer_version(cur):
+  sql = """select distinct value from __all_virtual_sys_parameter_stat where name='min_observer_version'"""
+  logging.info(sql)
+  cur.execute(sql)
+  result = cur.fetchall()
+  if len(result) != 1:
     raise MyError('query results count is not 1')
   else:
-    logging.info('get observer version success, version = {0}'.format(results[0][0]))
-  return results[0][0]
+    logging.info('get observer version success, version = {0}'.format(result[0][0]))
+  return result[0][0]
 
 def fetch_tenant_ids(query_cur):
   try:

tools/upgrade/do_upgrade_post.py

@@ -11,7 +11,9 @@ import config
 import opts
 import run_modules
 import actions
-import special_upgrade_action_post
+import upgrade_health_checker
+import tenant_upgrade_action
+import upgrade_post_checker
 
 # 由于用了/*+read_consistency(WEAK) */来查询，因此升级期间不能允许创建或删除租户
 
@@ -37,43 +39,6 @@ def config_logging_module(log_filenamme):
   # 将定义好的stdout_handler日志handler添加到root logger
   logging.getLogger('').addHandler(stdout_handler)
 
-
-def dump_sql_to_file(dump_filename, tenant_id_list):
-  normal_ddls_str = normal_ddl_actions_post.get_normal_ddl_actions_sqls_str()
-  normal_dmls_str = normal_dml_actions_post.get_normal_dml_actions_sqls_str()
-  each_tenant_dmls_str = each_tenant_dml_actions_post.get_each_tenant_dml_actions_sqls_str(tenant_id_list)
-  dump_file = open(dump_filename, 'w')
-  dump_file.write('# 以下是upgrade_post.py脚本中的步骤\n')
-  dump_file.write('# 仅供upgrade_post.py脚本运行失败需要人肉的时候参考\n')
-  dump_file.write('\n\n')
-  dump_file.write('# normal ddl\n')
-  dump_file.write(normal_ddls_str + '\n')
-  dump_file.write('\n\n')
-  dump_file.write('# normal dml\n')
-  dump_file.write(normal_dmls_str + '\n')
-  dump_file.write('\n\n')
-  dump_file.write('# each tenant dml\n')
-  dump_file.write(each_tenant_dmls_str + '\n')
-  dump_file.write('\n\n')
-  dump_file.write('# do special upgrade actions\n')
-  dump_file.write('# please run ./upgrade_post.py -h [host] -P [port] -u [user] -p [password] -m special_action\n')
-  dump_file.write('\n\n')
-  dump_file.close()
-
-def check_before_upgrade(query_cur, upgrade_params):
-  check_server_version(query_cur)
-  return
-
-# 混部阶段执行POST脚本，会导致租户级系统表创建可能多数派处于旧binary而被GC,
-# 需要规避OCP升级流程异常导致的混部阶段执行POST脚本的问题
-def check_server_version(query_cur):
-  sql = """select distinct(substring_index(build_version, '_', 1)) from __all_server""";
-  (desc, results) = query_cur.exec_query(sql);
-  if len(results) != 1:
-    raise MyError("servers build_version not match")
-  else:
-    logging.info("check server version success")
-
 def print_stats():
   logging.info('==================================================================================')
   logging.info('============================== STATISTICS BEGIN ==================================')
@@ -84,7 +49,7 @@ def print_stats():
   logging.info('=============================== STATISTICS END ===================================')
   logging.info('==================================================================================')
 
-def do_upgrade(my_host, my_port, my_user, my_passwd, my_module_set, upgrade_params):
+def do_upgrade(my_host, my_port, my_user, my_passwd, timeout, my_module_set, upgrade_params):
   try:
     conn = mysql.connector.connect(user = my_user,
                                    password = my_passwd,
@@ -95,8 +60,7 @@ def do_upgrade(my_host, my_port, my_user, my_passwd, my_module_set, upgrade_para
     cur = conn.cursor(buffered=True)
     try:
       query_cur = actions.QueryCursor(cur)
-      # 开始升级前的检查
-      check_before_upgrade(query_cur, upgrade_params)
+      actions.check_server_version_by_cluster(cur)
       # 获取租户id列表
       tenant_id_list = actions.fetch_tenant_ids(query_cur)
       if len(tenant_id_list) <= 0:
@@ -104,13 +68,45 @@ def do_upgrade(my_host, my_port, my_user, my_passwd, my_module_set, upgrade_para
         raise MyError('no tenant id')
       logging.info('there has %s distinct tenant ids: [%s]', len(tenant_id_list), ','.join(str(tenant_id) for tenant_id in tenant_id_list))
       conn.commit()
-      if run_modules.MODULE_SPECIAL_ACTION in my_module_set:
-        logging.info('================begin to run special action===============')
+
+      if run_modules.MODULE_HEALTH_CHECK in my_module_set:
+        logging.info('================begin to run health check action ===============')
+        upgrade_health_checker.do_check(my_host, my_port, my_user, my_passwd, upgrade_params, timeout)
+        logging.info('================succeed to run health check action ===============')
+
+      if run_modules.MODULE_END_ROLLING_UPGRADE in my_module_set:
+        logging.info('================begin to run end rolling upgrade action ===============')
         conn.autocommit = True
-        special_upgrade_action_post.do_special_upgrade(conn, cur, tenant_id_list, my_user, my_passwd)
+        actions.do_end_rolling_upgrade(cur, timeout)
         conn.autocommit = False
         actions.refresh_commit_sql_list()
-        logging.info('================succeed to commit special action===============')
+        logging.info('================succeed to run end rolling upgrade action ===============')
+
+      if run_modules.MODULE_TENANT_UPRADE in my_module_set:
+        logging.info('================begin to run tenant upgrade action ===============')
+        conn.autocommit = True
+        tenant_upgrade_action.do_upgrade(conn, cur, tenant_id_list, timeout, my_user, my_passwd)
+        conn.autocommit = False
+        actions.refresh_commit_sql_list()
+        logging.info('================succeed to run tenant upgrade action ===============')
+
+      if run_modules.MODULE_END_UPRADE in my_module_set:
+        logging.info('================begin to run end upgrade action ===============')
+        conn.autocommit = True
+        actions.do_end_upgrade(cur, timeout)
+        conn.autocommit = False
+        actions.refresh_commit_sql_list()
+        logging.info('================succeed to run end upgrade action ===============')
+
+      if run_modules.MODULE_POST_CHECK in my_module_set:
+        logging.info('================begin to run post check action ===============')
+        conn.autocommit = True
+        actions.do_end_upgrade(cur, timeout)
+        upgrade_post_checker.do_check(conn, cur, query_cur, timeout)
+        conn.autocommit = False
+        actions.refresh_commit_sql_list()
+        logging.info('================succeed to run post check action ===============')
+
     except Exception, e:
       logging.exception('run error')
       raise e
@@ -118,7 +114,7 @@ def do_upgrade(my_host, my_port, my_user, my_passwd, my_module_set, upgrade_para
       # 打印统计信息
       print_stats()
       # 将回滚sql写到文件中
-      actions.dump_rollback_sql_to_file(upgrade_params.rollback_sql_filename)
+      # actions.dump_rollback_sql_to_file(upgrade_params.rollback_sql_filename)
       cur.close()
       conn.close()
   except mysql.connector.Error, e:
@@ -133,7 +129,7 @@ def do_upgrade_by_argv(argv):
   opts.change_opt_defult_value('log-file', upgrade_params.log_filename)
   opts.parse_options(argv)
   if not opts.has_no_local_opts():
-    opts.deal_with_local_opts()
+    opts.deal_with_local_opts('upgrade_post')
   else:
     opts.check_db_client_opts()
     log_filename = opts.get_opt_log_file()
@@ -145,6 +141,7 @@ def do_upgrade_by_argv(argv):
       port = int(opts.get_opt_port())
       user = opts.get_opt_user()
       password = opts.get_opt_password()
+      timeout = int(opts.get_opt_timeout())
       cmd_module_str = opts.get_opt_module()
       module_set = set([])
       all_module_set = run_modules.get_all_module_set()
@@ -156,9 +153,9 @@ def do_upgrade_by_argv(argv):
           module_set.add(cmd_module)
         else:
           raise MyError('invalid module: {0}'.format(cmd_module))
-      logging.info('parameters from cmd: host=\"%s\", port=%s, user=\"%s\", password=\"%s\", module=\"%s\", log-file=\"%s\"',\
-          host, port, user, password, module_set, log_filename)
-      do_upgrade(host, port, user, password, module_set, upgrade_params)
+      logging.info('parameters from cmd: host=\"%s\", port=%s, user=\"%s\", password=\"%s\", timeout=\"%s\", module=\"%s\", log-file=\"%s\"',\
+          host, port, user, password, timeout, module_set, log_filename)
+      do_upgrade(host, port, user, password, timeout, module_set, upgrade_params)
     except mysql.connector.Error, e:
       logging.exception('mysql connctor error')
       logging.exception('run error, maybe you can reference ' + upgrade_params.rollback_sql_filename + ' to rollback it')

tools/upgrade/do_upgrade_pre.py

@@ -12,6 +12,7 @@ import opts
 import run_modules
 import actions
 import special_upgrade_action_pre
+import upgrade_health_checker
 
 # 由于用了/*+read_consistency(WEAK) */来查询，因此升级期间不能允许创建或删除租户
 
@@ -37,34 +38,6 @@ def config_logging_module(log_filenamme):
   # 将定义好的stdout_handler日志handler添加到root logger
   logging.getLogger('').addHandler(stdout_handler)
 
-def dump_sql_to_file(cur, query_cur, dump_filename, tenant_id_list, update_sys_var_list, add_sys_var_list):
-  normal_ddls_str = normal_ddl_actions_pre.get_normal_ddl_actions_sqls_str(query_cur)
-  normal_dmls_str = normal_dml_actions_pre.get_normal_dml_actions_sqls_str()
-  each_tenant_dmls_str = each_tenant_dml_actions_pre.get_each_tenant_dml_actions_sqls_str(tenant_id_list)
-  dump_file = open(dump_filename, 'w')
-  dump_file.write('# 以下是upgrade_pre.py脚本中的步骤\n')
-  dump_file.write('# 仅供upgrade_pre.py脚本运行失败需要人肉的时候参考\n')
-  dump_file.write('\n\n')
-  dump_file.write('# normal ddl\n')
-  dump_file.write(normal_ddls_str + '\n')
-  dump_file.write('\n\n')
-  dump_file.write('# normal dml\n')
-  dump_file.write(normal_dmls_str + '\n')
-  dump_file.write('\n\n')
-  dump_file.write('# each tenant dml\n')
-  dump_file.write(each_tenant_dmls_str + '\n')
-  dump_file.write('\n\n')
-  dump_file.write('# upgrade sys vars\n')
-  dump_file.write(sys_vars_upgrade_dmls_str + '\n')
-  dump_file.write('\n\n')
-  dump_file.write('# do special upgrade actions\n')
-  dump_file.write('# please run ./upgrade_pre.py -h [host] -P [port] -u [user] -p [password] -m special_action\n')
-  dump_file.write('\n\n')
-  dump_file.close()
-
-def check_before_upgrade(query_cur, upgrade_params):
-  return
-
 def print_stats():
   logging.info('==================================================================================')
   logging.info('============================== STATISTICS BEGIN ==================================')
@@ -75,7 +48,7 @@ def print_stats():
   logging.info('=============================== STATISTICS END ===================================')
   logging.info('==================================================================================')
 
-def do_upgrade(my_host, my_port, my_user, my_passwd, my_module_set, upgrade_params):
+def do_upgrade(my_host, my_port, my_user, my_passwd, timeout, my_module_set, upgrade_params):
   try:
     conn = mysql.connector.connect(user = my_user,
                                    password = my_passwd,
@@ -86,10 +59,7 @@ def do_upgrade(my_host, my_port, my_user, my_passwd, my_module_set, upgrade_para
     cur = conn.cursor(buffered=True)
     try:
       query_cur = actions.QueryCursor(cur)
-      # 开始升级前的检查
-      check_before_upgrade(query_cur, upgrade_params)
-      # get min_observer_version
-      version = actions.fetch_observer_version(query_cur)
+      actions.check_server_version_by_cluster(cur)
       # 获取租户id列表
       tenant_id_list = actions.fetch_tenant_ids(query_cur)
       if len(tenant_id_list) <= 0:
@@ -97,13 +67,35 @@ def do_upgrade(my_host, my_port, my_user, my_passwd, my_module_set, upgrade_para
         raise MyError('no tenant id')
       logging.info('there has %s distinct tenant ids: [%s]', len(tenant_id_list), ','.join(str(tenant_id) for tenant_id in tenant_id_list))
 
+      if run_modules.MODULE_BEGIN_UPGRADE in my_module_set:
+        logging.info('================begin to run begin upgrade action===============')
+        conn.autocommit = True
+        actions.do_begin_upgrade(cur, timeout)
+        conn.autocommit = False
+        actions.refresh_commit_sql_list()
+        logging.info('================succeed to run begin upgrade action===============')
+
+      if run_modules.MODULE_BEGIN_ROLLING_UPGRADE in my_module_set:
+        logging.info('================begin to run begin rolling upgrade action===============')
+        conn.autocommit = True
+        actions.do_begin_rolling_upgrade(cur, timeout)
+        conn.autocommit = False
+        actions.refresh_commit_sql_list()
+        logging.info('================succeed to run begin rolling upgrade action===============')
+
       if run_modules.MODULE_SPECIAL_ACTION in my_module_set:
         logging.info('================begin to run special action===============')
         conn.autocommit = True
-        special_upgrade_action_pre.do_special_upgrade(conn, cur, tenant_id_list, my_user, my_passwd)
+        special_upgrade_action_pre.do_special_upgrade(conn, cur, tenant_id_list, timeout, my_user, my_passwd)
         conn.autocommit = False
         actions.refresh_commit_sql_list()
-        logging.info('================succeed to commit special action===============')
+        logging.info('================succeed to run special action===============')
+
+      if run_modules.MODULE_HEALTH_CHECK in my_module_set:
+        logging.info('================begin to run health check action ===============')
+        upgrade_health_checker.do_check(my_host, my_port, my_user, my_passwd, upgrade_params, timeout)
+        logging.info('================succeed to run health check action ===============')
+
     except Exception, e:
       logging.exception('run error')
       raise e
@@ -111,7 +103,7 @@ def do_upgrade(my_host, my_port, my_user, my_passwd, my_module_set, upgrade_para
       # 打印统计信息
       print_stats()
       # 将回滚sql写到文件中
-      actions.dump_rollback_sql_to_file(upgrade_params.rollback_sql_filename)
+      # actions.dump_rollback_sql_to_file(upgrade_params.rollback_sql_filename)
       cur.close()
       conn.close()
   except mysql.connector.Error, e:
@@ -126,7 +118,7 @@ def do_upgrade_by_argv(argv):
   opts.change_opt_defult_value('log-file', upgrade_params.log_filename)
   opts.parse_options(argv)
   if not opts.has_no_local_opts():
-    opts.deal_with_local_opts()
+    opts.deal_with_local_opts('upgrade_pre')
   else:
     opts.check_db_client_opts()
     log_filename = opts.get_opt_log_file()
@@ -138,6 +130,7 @@ def do_upgrade_by_argv(argv):
       port = int(opts.get_opt_port())
       user = opts.get_opt_user()
       password = opts.get_opt_password()
+      timeout = int(opts.get_opt_timeout())
       cmd_module_str = opts.get_opt_module()
       module_set = set([])
       all_module_set = run_modules.get_all_module_set()
@@ -149,9 +142,9 @@ def do_upgrade_by_argv(argv):
           module_set.add(cmd_module)
         else:
           raise MyError('invalid module: {0}'.format(cmd_module))
-      logging.info('parameters from cmd: host=\"%s\", port=%s, user=\"%s\", password=\"%s\", module=\"%s\", log-file=\"%s\"',\
-          host, port, user, password, module_set, log_filename)
-      do_upgrade(host, port, user, password, module_set, upgrade_params)
+      logging.info('parameters from cmd: host=\"%s\", port=%s, user=\"%s\", password=\"%s\", timeout=\"%s\", module=\"%s\", log-file=\"%s\"',\
+          host, port, user, password, timeout, module_set, log_filename)
+      do_upgrade(host, port, user, password, timeout, module_set, upgrade_params)
     except mysql.connector.Error, e:
       logging.exception('mysql connctor error')
       logging.exception('run error, maybe you can reference ' + upgrade_params.rollback_sql_filename + ' to rollback it')

tools/upgrade/opts.py

@@ -6,7 +6,7 @@ import sys
 import os
 import getopt
 
-help_str = \
+pre_help_str = \
 """
 Help:
 """ +\
@@ -19,11 +19,45 @@ sys.argv[0] + """ [OPTIONS]""" +\
 '-u, --user=name     User for login.\n' +\
 '-p, --password=name Password to use when connecting to server. If password is\n' +\
 '                    not given it\'s empty string "".\n' +\
+'-t, --timeout=name  Cmd/Query/Inspection execute timeout(s).\n' +\
 '-m, --module=name   Modules to run. Modules should be a string combined by some of\n' +\
-'                    the following strings: ddl, normal_dml, each_tenant_dml,\n' +\
-'                    system_variable_dml, special_action, all. "all" represents\n' +\
+'                    the following strings:\n' +\
+'                    1. begin_upgrade \n' +\
+'                    2. begin_rolling_upgrade \n' +\
+'                    3. special_action \n' +\
+'                    4. health_check \n' +\
+'                    5. all: default value, run all sub modules above \n' +\
 '                    that all modules should be run. They are splitted by ",".\n' +\
-'                    For example: -m all, or --module=ddl,normal_dml,special_action\n' +\
+'                    For example: -m all, or --module=begin_upgrade,begin_rolling_upgrade,special_action\n' +\
+'-l, --log-file=name Log file path. If log file path is not given it\'s ' + os.path.splitext(sys.argv[0])[0] + '.log\n' +\
+'\n\n' +\
+'Maybe you want to run cmd like that:\n' +\
+sys.argv[0] + ' -h 127.0.0.1 -P 3306 -u admin -p admin\n'
+
+post_help_str = \
+"""
+Help:
+""" +\
+sys.argv[0] + """ [OPTIONS]""" +\
+'\n\n' +\
+'-I, --help          Display this help and exit.\n' +\
+'-V, --version       Output version information and exit.\n' +\
+'-h, --host=name     Connect to host.\n' +\
+'-P, --port=name     Port number to use for connection.\n' +\
+'-u, --user=name     User for login.\n' +\
+'-p, --password=name Password to use when connecting to server. If password is\n' +\
+'                    not given it\'s empty string "".\n' +\
+'-t, --timeout=name  Cmd/Query/Inspection execute timeout(s).\n' +\
+'-m, --module=name   Modules to run. Modules should be a string combined by some of\n' +\
+'                    the following strings:\n' +\
+'                    1. health_check \n' +\
+'                    2. end_rolling_upgrade \n' +\
+'                    3. tenant_upgrade \n' +\
+'                    4. end_upgrade \n' +\
+'                    5. post_check \n' +\
+'                    6. all: default value, run all sub modules above \n' +\
+'                    that all modules should be run. They are splitted by ",".\n' +\
+'                    For example: -m all, or --module=health_check,end_rolling_upgrade\n' +\
 '-l, --log-file=name Log file path. If log file path is not given it\'s ' + os.path.splitext(sys.argv[0])[0] + '.log\n' +\
 '\n\n' +\
 'Maybe you want to run cmd like that:\n' +\
@@ -79,6 +113,7 @@ Option('V', 'version', False, True),\
 Option('h', 'host', True, False),\
 Option('P', 'port', True, False),\
 Option('u', 'user', True, False),\
+Option('t', 'timeout', True, False, 0),\
 Option('p', 'password', True, False, ''),\
 # 要跑哪个模块，默认全跑
 Option('m', 'module', True, False, 'all'),\
@@ -134,22 +169,28 @@ def parse_options(argv):
   if has_no_local_opts():
     check_db_client_opts()
 
-def deal_with_local_opt(opt):
+def deal_with_local_opt(opt, filename):
   if 'help' == opt.get_long_name():
-    global help_str
-    print help_str
+    if 'upgrade_pre' == filename:
+      global pre_help_str
+      print pre_help_str
+    elif 'upgrade_post' == filename:
+      global post_help_str
+      print post_help_str
+    else:
+            raise MyError('not supported filename:{0} for help option'.format(filename))
   elif 'version' == opt.get_long_name():
     global version_str
     print version_str
 
-def deal_with_local_opts():
+def deal_with_local_opts(filename):
   global g_opts
   if has_no_local_opts():
     raise MyError('no local options, can not deal with local options')
   else:
     for opt in g_opts:
       if opt.is_local_opt() and opt.has_value():
-        deal_with_local_opt(opt)
+        deal_with_local_opt(opt, filename)
         # 只处理一个
         return
 
@@ -177,6 +218,12 @@ def get_opt_password():
     if 'password' == opt.get_long_name():
       return opt.get_value()
 
+def get_opt_timeout():
+  global g_opts
+  for opt in g_opts:
+    if 'timeout' == opt.get_long_name():
+      return opt.get_value()
+
 def get_opt_module():
   global g_opts
   for opt in g_opts:

tools/upgrade/run_modules.py

@@ -3,12 +3,18 @@
 
 ALL_MODULE = 'all'
 
-MODULE_DDL = 'ddl'
-MODULE_NORMAL_DML = 'normal_dml'
-MODULE_EACH_TENANT_DML = 'each_tenant_dml'
-MODULE_EACH_TENANT_DDL = 'each_tenant_ddl'
-MODULE_SYSTEM_VARIABLE_DML = 'system_variable_dml'
-MODULE_SPECIAL_ACTION = 'special_action'
+# module for upgrade_pre.py
+MODULE_BEGIN_UPGRADE          = 'begin_upgrade'
+MODULE_BEGIN_ROLLING_UPGRADE  = 'begin_rolling_upgrade'
+MODULE_SPECIAL_ACTION         = 'special_action'
+#MODULE_HEALTH_CHECK          = 'health_check'
+
+# module for upgrade_post.py
+MODULE_HEALTH_CHECK           = 'health_check'
+MODULE_END_ROLLING_UPGRADE    = 'end_rolling_upgrade'
+MODULE_TENANT_UPRADE          = 'tenant_upgrade'
+MODULE_END_UPRADE             = 'end_upgrade'
+MODULE_POST_CHECK             = 'post_check'
 
 def get_all_module_set():
   import run_modules

tools/upgrade/special_upgrade_action_post.py

@@ -1,323 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import logging
-import time
-from actions import Cursor
-from actions import DMLCursor
-from actions import QueryCursor
-import mysql.connector
-from mysql.connector import errorcode
-import actions
-
-def do_special_upgrade(conn, cur, tenant_id_list, user, pwd):
-  # special upgrade action
-#升级语句对应的action要写在下面的actions begin和actions end这两行之间，
-#因为基准版本更新的时候会调用reset_upgrade_scripts.py来清空actions begin和actions end
-#这两行之间的这些代码，如果不写在这两行之间的话会导致清空不掉相应的代码。
-  upgrade_system_package(conn, cur)
-####========******####======== actions begin ========####******========####
-  return
-####========******####========= actions end =========####******========####
-
-def query(cur, sql):
-  cur.execute(sql)
-  results = cur.fetchall()
-  return results
-
-def get_tenant_names(cur):
-  return [_[0] for _ in query(cur, 'select tenant_name from oceanbase.__all_tenant')]
-
-def run_create_inner_schema_job(conn, cur):
-  try:
-    ###### enable ddl
-    ori_enable_ddl = actions.get_ori_enable_ddl(cur)
-    if ori_enable_ddl == 0:
-      actions.set_parameter(cur, 'enable_ddl', 'True')
-
-    # check record in rs_job
-    count_sql = """select count(*) from oceanbase.__all_rootservice_job
-            where job_type = 'CREATE_INNER_SCHEMA';"""
-    result = query(cur, count_sql)
-    job_count = 0
-    if (1 != len(result)) :
-      logging.warn("unexpected sql output")
-      raise e
-    else :
-      job_count = result[0][0]
-      # run job
-      sql = "alter system run job 'CREATE_INNER_SCHEMA';"
-      logging.info(sql)
-      cur.execute(sql)
-
-      # wait job finish
-      times = 180
-      ## 先检查job count变化
-      count_check = False
-      while times > 0:
-        result = query(cur, count_sql)
-        if (1 != len(result)):
-          logging.warn("unexpected sql output")
-          raise e
-        elif (result[0][0] > job_count):
-          count_check = True
-          logging.info('create_inner_schema job detected')
-          break
-        time.sleep(10)
-        times -= 1
-        if times == 0:
-          raise MyError('check create_inner_schema job failed!')
-
-      ## 继续检查job status状态
-      status_sql = """select job_status from oceanbase.__all_rootservice_job
-                      where job_type = 'CREATE_INNER_SCHEMA' order by job_id desc limit 1;"""
-      status_check = False
-      while times > 0 and count_check == True:
-        result = query(cur, status_sql)
-        if (0 == len(result)):
-          logging.warn("unexpected sql output")
-          raise e
-        elif (1 != len(result) or 1 != len(result[0])):
-          logging.warn("result len not match")
-          raise e
-        elif result[0][0] == "FAILED":
-          logging.warn("run create_inner_schema job faild")
-          raise e
-        elif result[0][0] == "INPROGRESS":
-          logging.info('create_inner_schema job is still running')
-        elif result[0][0] == "SUCCESS":
-          status_check = True
-          break;
-        else:
-          logging.warn("invalid result: {0}" % (result[0][0]))
-          raise e
-        time.sleep(10)
-        times -= 1
-        if times == 0:
-          raise MyError('check create_inner_schema job failed!')
-
-      if (status_check == True and count_check == True):
-        logging.info('check create_inner_schema job success')
-      else:
-        logging.warn("run create_inner_schema job faild")
-        raise e
-
-    # disable ddl
-    if ori_enable_ddl == 0:
-      actions.set_parameter(cur, 'enable_ddl', 'False')
-  except Exception, e:
-    logging.warn("run create_inner_schema job failed")
-    raise e
-  logging.info("run create_inner_schema job success")
-
-def disable_major_freeze(conn, cur):
-  try:
-    actions.set_parameter(cur, "enable_major_freeze", 'False')
-  except Exception, e:
-    logging.warn("disable enable_major_freeze failed")
-    raise e
-  logging.info("disable enable_major_freeze finish")
-
-def get_max_used_job_id(cur):
-  try:
-    max_job_id = 0
-    sql = "select job_id from oceanbase.__all_rootservice_job order by job_id desc limit 1"
-    results = query(cur, sql)
-
-    if (len(results) == 0):
-      max_job_id = 0
-    elif (len(results) != 1 or len(results[0]) != 1):
-      logging.warn("row cnt not match")
-      raise e
-    else:
-      max_job_id = results[0][0]
-
-    logging.info("get max_used_job_id:{0}".format(max_job_id))
-
-    return max_job_id
-  except Exception, e:
-    logging.warn("failed to get max_used_job_id")
-    raise e
-
-def check_can_run_upgrade_job(cur, version):
-  try:
-    sql = """select job_status from oceanbase.__all_rootservice_job
-             where job_type = 'UPGRADE_POST_ACTION' and extra_info = '{0}'
-             order by job_id desc limit 1""".format(version)
-    results = query(cur, sql)
-
-    bret = True
-    if (len(results) == 0):
-      bret = True
-      logging.info("upgrade job not created yet, should run upgrade job")
-    elif (len(results) != 1 or len(results[0]) != 1):
-      logging.warn("row cnt not match")
-      raise e
-    elif ("INPROGRESS" == results[0][0]):
-      logging.warn("upgrade job still running, should wait")
-      raise e
-    elif ("SUCCESS" == results[0][0]):
-      bret = False
-      logging.info("execute upgrade job successfully, skip run upgrade job")
-    elif ("FAILED" == results[0][0]):
-      bret = True
-      logging.info("execute upgrade job failed, should run again")
-    else:
-      logging.warn("invalid job status: {0}".format(results[0][0]))
-      raise e
-
-    return bret
-  except Exception, e:
-    logging.warn("failed to check if upgrade job can run")
-    raise e
-
-
-def check_upgrade_job_result(cur, version, max_used_job_id):
-  try:
-    times = 0
-    while (times <= 180):
-      sql = """select job_status from oceanbase.__all_rootservice_job
-               where job_type = 'RUN_UPGRADE_POST_JOB' and extra_info = '{0}'
-               and job_id > {1} order by job_id desc limit 1""".format(version, max_used_job_id)
-      results = query(cur, sql)
-
-      if (len(results) == 0):
-        logging.info("upgrade job not created yet")
-      elif (len(results) != 1 or len(results[0]) != 1):
-        logging.warn("row cnt not match")
-        raise e
-      elif ("INPROGRESS" == results[0][0]):
-        logging.info("upgrade job is still running")
-      elif ("SUCCESS" == results[0][0]):
-        logging.info("execute upgrade job successfully")
-        break;
-      elif ("FAILED" == results[0][0]):
-        logging.warn("execute upgrade job failed")
-        raise e
-      else:
-        logging.warn("invalid job status: {0}".format(results[0][0]))
-        raise e
-
-      if times >= 180:
-        logging.warn("run upgrade job timeout")
-        raise e
-      times = times + 1
-      time.sleep(10)
-  except Exception, e:
-    logging.warn("failed to check upgrade job result")
-    raise e
-
-
-def run_upgrade_job(conn, cur, version):
-  try:
-    logging.info("start to run upgrade job, version:{0}".format(version))
-    # pre check
-    if (check_can_run_upgrade_job(cur, version) == True):
-      conn.autocommit = True
-      # disable enable_ddl
-      ori_enable_ddl = actions.get_ori_enable_ddl(cur)
-      if ori_enable_ddl == 1:
-        actions.set_parameter(cur, 'enable_ddl', 'False')
-      # get max_used_job_id
-      max_used_job_id = get_max_used_job_id(cur)
-      # run upgrade job
-      sql = """alter system run upgrade job '{0}'""".format(version)
-      logging.info(sql)
-      cur.execute(sql)
-      # check upgrade job result
-      check_upgrade_job_result(cur, version, max_used_job_id)
-      # reset enable_ddl
-      if ori_enable_ddl == 1:
-        actions.set_parameter(cur, 'enable_ddl', 'True')
-  except Exception, e:
-    logging.warn("run upgrade job failed, version:{0}".format(version))
-    raise e
-  logging.info("run upgrade job success, version:{0}".format(version))
-
-
-def upgrade_system_package(conn, cur):
-  try:
-    ori_enable_ddl = actions.get_ori_enable_ddl(cur)
-    if ori_enable_ddl == 0:
-      actions.set_parameter(cur, 'enable_ddl', 'True')
-
-    sql = "set ob_query_timeout = 300000000;"
-    logging.info(sql)
-    cur.execute(sql)
-
-    sql = """
-      CREATE OR REPLACE PACKAGE __DBMS_UPGRADE
-        PROCEDURE UPGRADE(package_name VARCHAR(1024));
-        PROCEDURE UPGRADE_ALL();
-      END;
-    """
-    logging.info(sql)
-    cur.execute(sql)
-
-    sql = """
-      CREATE OR REPLACE PACKAGE BODY __DBMS_UPGRADE
-        PROCEDURE UPGRADE(package_name VARCHAR(1024));
-          PRAGMA INTERFACE(c, UPGRADE_SINGLE);
-        PROCEDURE UPGRADE_ALL();
-          PRAGMA INTERFACE(c, UPGRADE_ALL);
-      END;
-    """
-    logging.info(sql)
-    cur.execute(sql)
-
-    sql = """
-      CALL __DBMS_UPGRADE.UPGRADE_ALL();
-    """
-    logging.info(sql)
-    cur.execute(sql)
-
-    ###### alter session compatibility mode
-    sql = "set ob_compatibility_mode='oracle';"
-    logging.info(sql)
-    cur.execute(sql)
-
-    sql = "set ob_query_timeout = 300000000;"
-    logging.info(sql)
-    cur.execute(sql)
-
-    sql = """
-      CREATE OR REPLACE PACKAGE "__DBMS_UPGRADE" IS
-        PROCEDURE UPGRADE(package_name VARCHAR2);
-        PROCEDURE UPGRADE_ALL;
-      END;
-    """
-    logging.info(sql)
-    cur.execute(sql)
-
-    sql = """
-      CREATE OR REPLACE PACKAGE BODY "__DBMS_UPGRADE" IS
-        PROCEDURE UPGRADE(package_name VARCHAR2);
-          PRAGMA INTERFACE(c, UPGRADE_SINGLE);
-        PROCEDURE UPGRADE_ALL;
-          PRAGMA INTERFACE(c, UPGRADE_ALL);
-      END;
-    """
-    logging.info(sql)
-    cur.execute(sql)
-
-    ########## update system package ##########
-    sql = """
-      CALL "__DBMS_UPGRADE".UPGRADE_ALL();
-    """
-    logging.info(sql)
-    cur.execute(sql)
-
-    ###### alter session compatibility mode
-    sql = "set ob_compatibility_mode='mysql';"
-    logging.info(sql)
-    cur.execute(sql)
-    time.sleep(10)
-
-    if ori_enable_ddl == 0:
-      actions.set_parameter(cur, 'enable_ddl', 'False')
-
-    logging.info("upgrade package finished!")
-  except Exception, e:
-    logging.warn("upgrade package failed!")
-    raise e
-

tools/upgrade/special_upgrade_action_pre.py

@@ -16,44 +16,18 @@ import my_utils
 import actions
 import sys
 
-#def modify_schema_history(conn, cur, tenant_ids):
-#  try:
-#    conn.autocommit = True
-#
-#    # disable ddl
-#    ori_enable_ddl = actions.get_ori_enable_ddl(cur)
-#    if ori_enable_ddl == 1:
-#      actions.set_parameter(cur, 'enable_ddl', 'False')
-#    log('tenant_ids: {0}'.format(tenant_ids))
-#    for tenant_id in tenant_ids:
-#      sql = """alter system change tenant tenant_id = {0}""".format(tenant_id)
-#      log(sql)
-#      cur.execute(sql)
-
-#    #####implement#####
-#
-#    # 还原默认 tenant
-#    sys_tenant_id = 1
-#    sql = """alter system change tenant tenant_id = {0}""".format(sys_tenant_id)
-#    log(sql)
-#    cur.execute(sql)
-#
-#    # enable ddl
-#    if ori_enable_ddl == 1:
-#      actions.set_parameter(cur, 'enable_ddl', 'True')
-#  except Exception, e:
-#    logging.warn("exec modify trigger failed")
-#    raise e
-#  logging.info('exec modify trigger finish')
-
 # 主库需要执行的升级动作
-def do_special_upgrade(conn, cur, tenant_id_list, user, passwd):
+def do_special_upgrade(conn, cur, tenant_id_list, timeout, user, passwd):
   # special upgrade action
 #升级语句对应的action要写在下面的actions begin和actions end这两行之间，
 #因为基准版本更新的时候会调用reset_upgrade_scripts.py来清空actions begin和actions end
 #这两行之间的这些代码，如果不写在这两行之间的话会导致清空不掉相应的代码。
-# 主库升级流程没加滚动升级步骤，或混部阶段DDL测试有相关case覆盖前，混部开始禁DDL
-  actions.set_parameter(cur, 'enable_ddl', 'False')
+  current_version = actions.fetch_observer_version(cur)
+  target_version = actions.get_current_cluster_version()
+  # when upgrade across version, disable enable_ddl/major_freeze
+  if current_version != target_version:
+    actions.set_parameter(cur, 'enable_ddl', 'False', timeout)
+    actions.set_parameter(cur, 'enable_major_freeze', 'False', timeout)
 ####========******####======== actions begin ========####******========####
   return
 ####========******####========= actions end =========####******========####