From eb103e94b04e8ea97ae51ab2efa219092d37a641 Mon Sep 17 00:00:00 2001 From: tino247 Date: Fri, 5 Jan 2024 06:12:44 +0000 Subject: [PATCH] [UPGRADE] Make sure data_version persisted before create unit --- src/observer/omt/ob_tenant_config_mgr.cpp | 5 + src/observer/omt/ob_tenant_node_balancer.cpp | 40 ++++-- src/observer/omt/ob_tenant_node_balancer.h | 4 +- src/rootserver/ob_bootstrap.cpp | 6 +- src/rootserver/ob_ddl_service.cpp | 6 +- src/rootserver/ob_unit_manager.cpp | 123 ++++++++++++++++--- src/rootserver/ob_unit_manager.h | 18 ++- 7 files changed, 165 insertions(+), 37 deletions(-) diff --git a/src/observer/omt/ob_tenant_config_mgr.cpp b/src/observer/omt/ob_tenant_config_mgr.cpp index 6c177ad5b4..f73864b91b 100644 --- a/src/observer/omt/ob_tenant_config_mgr.cpp +++ b/src/observer/omt/ob_tenant_config_mgr.cpp @@ -322,6 +322,11 @@ int ObTenantConfigMgr::del_tenant_config(uint64_t tenant_id) } else if (!config->is_ref_clear()) { ret = OB_EAGAIN; LOG_INFO("something hold config ref, try delete later..."); + } else if (OB_ISNULL(GCTX.omt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("omt is null", KR(ret)); + } else if (GCTX.omt_->has_tenant(tenant_id)) { + LOG_WARN("local tenant resource still exist, try to delete tenant config later", K(tenant_id)); } else { config->set_deleting(); if (OB_FAIL(wait(config->get_update_task()))) { diff --git a/src/observer/omt/ob_tenant_node_balancer.cpp b/src/observer/omt/ob_tenant_node_balancer.cpp index 4422ab4343..d971335743 100644 --- a/src/observer/omt/ob_tenant_node_balancer.cpp +++ b/src/observer/omt/ob_tenant_node_balancer.cpp @@ -205,7 +205,7 @@ int ObTenantNodeBalancer::notify_create_tenant(const obrpc::TenantServerUnitConf } else if (is_user_tenant(tenant_id) && OB_FAIL(basic_tenant_unit.divide_meta_tenant(meta_tenant_unit))) { LOG_WARN("divide meta tenant failed", KR(ret), K(unit), K(basic_tenant_unit)); - } else if (OB_FAIL(check_new_tenant(basic_tenant_unit, create_tenant_timeout_ts))) { + } else if (OB_FAIL(check_new_tenant(basic_tenant_unit, false /*check_data_version*/, create_tenant_timeout_ts))) { LOG_WARN("failed to create new tenant", KR(ret), K(basic_tenant_unit), K(create_tenant_timeout_ts)); } else { ret = OB_SUCCESS; @@ -233,7 +233,7 @@ int ObTenantNodeBalancer::notify_create_tenant(const obrpc::TenantServerUnitConf #endif // create meta tenant if (OB_SUCC(ret) && is_user_tenant(tenant_id)) { - if (OB_FAIL(check_new_tenant(meta_tenant_unit, create_tenant_timeout_ts))) { + if (OB_FAIL(check_new_tenant(meta_tenant_unit, false /*check_data_version*/, create_tenant_timeout_ts))) { LOG_WARN("failed to create meta tenant", KR(ret), K(meta_tenant_unit), K(create_tenant_timeout_ts)); } else { ret = OB_SUCCESS; @@ -350,20 +350,25 @@ int ObTenantNodeBalancer::check_del_tenants(const TenantUnits &local_units, Tena int ObTenantNodeBalancer::check_new_tenants(TenantUnits &units) { int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; DEBUG_SYNC(CHECK_NEW_TENANT); + const bool check_data_version = true; // check all units of tenants. for (TenantUnits::iterator it = units.begin(); it != units.end(); it++) { - if (OB_FAIL(check_new_tenant(*it))) { - LOG_WARN("failed to check new tenant", K(ret)); + if (OB_TMP_FAIL(check_new_tenant(*it, check_data_version))) { + LOG_WARN("failed to check new tenant", KR(tmp_ret)); + ret = OB_SUCC(ret) ? tmp_ret : ret; } } - return ret; } -int ObTenantNodeBalancer::check_new_tenant(const ObUnitInfoGetter::ObTenantConfig &unit, const int64_t abs_timeout_us) +int ObTenantNodeBalancer::check_new_tenant( + const ObUnitInfoGetter::ObTenantConfig &unit, + const bool check_data_version, + const int64_t abs_timeout_us) { int ret = OB_SUCCESS; @@ -378,7 +383,16 @@ int ObTenantNodeBalancer::check_new_tenant(const ObUnitInfoGetter::ObTenantConfi ret = OB_SUCCESS; ObTenantMeta tenant_meta; ObTenantSuperBlock super_block(tenant_id, false /*is_hidden*/); // empty super block - if (OB_FAIL(tenant_meta.build(unit, super_block))) { + const bool should_check_data_version = check_data_version && is_user_tenant(tenant_id); + uint64_t data_version = 0; + if (should_check_data_version && OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) { + if (OB_ENTRY_NOT_EXIST == ret) { + ret = OB_EAGAIN; + LOG_WARN("data_version not refreshed yet, create tenant later", KR(ret), K(tenant_id)); + } else { + LOG_WARN("fail to get data_version", KR(ret), K(tenant_id)); + } + } else if (OB_FAIL(tenant_meta.build(unit, super_block))) { LOG_WARN("fail to build tenant meta", K(ret)); } else if (OB_FAIL(omt_->create_tenant(tenant_meta, true /* write_slog */, abs_timeout_us))) { LOG_WARN("fail to create new tenant", K(ret), K(tenant_id)); @@ -603,10 +617,16 @@ int ObTenantNodeBalancer::refresh_tenant(TenantUnits &units) if (OB_SUCC(ret)) { if (OB_FAIL(check_new_tenants(units))) { LOG_WARN("check and add new tenant fail", K(ret)); - } else if (FALSE_IT(omt_->set_synced())) { - } else if (OB_FAIL(check_del_tenants(local_units, units))) { + ret = OB_SUCCESS; // just don't affect the following process in run1(). + } else { + omt_->set_synced(); + } + + if (OB_FAIL(check_del_tenants(local_units, units))) { // overwrite ret LOG_WARN("check delete tenant fail", K(ret)); - } else if (OB_FAIL(refresh_hidden_sys_memory())) { + } + + if (OB_FAIL(refresh_hidden_sys_memory())) { // overwrite ret LOG_WARN("refresh hidden sys memory failed", K(ret)); } } diff --git a/src/observer/omt/ob_tenant_node_balancer.h b/src/observer/omt/ob_tenant_node_balancer.h index 43e62ef2e8..cc7ccc1409 100644 --- a/src/observer/omt/ob_tenant_node_balancer.h +++ b/src/observer/omt/ob_tenant_node_balancer.h @@ -80,7 +80,9 @@ private: ~ObTenantNodeBalancer(); int check_new_tenants(share::TenantUnits &units); - int check_new_tenant(const share::ObUnitInfoGetter::ObTenantConfig &unit, const int64_t abs_timeout_us = INT64_MAX); + int check_new_tenant(const share::ObUnitInfoGetter::ObTenantConfig &unit, + const bool check_data_version, + const int64_t abs_timeout_us = INT64_MAX); int check_del_tenants(const share::TenantUnits &local_units, share::TenantUnits &units); int refresh_hidden_sys_memory(); void periodically_check_tenant(); diff --git a/src/rootserver/ob_bootstrap.cpp b/src/rootserver/ob_bootstrap.cpp index 0328d758e0..2a4b23ea8f 100644 --- a/src/rootserver/ob_bootstrap.cpp +++ b/src/rootserver/ob_bootstrap.cpp @@ -1505,8 +1505,9 @@ int ObBootstrap::create_sys_resource_pool() ObArray sys_units; ObArray pool_names; share::ObResourcePool pool; - bool is_bootstrap = true; + const bool is_bootstrap = true; const bool if_not_exist = false; + const bool check_data_version = false; common::ObMySQLTransaction trans; common::ObArray new_ug_id_array; if (OB_FAIL(check_inner_stat())) { @@ -1528,7 +1529,8 @@ int ObBootstrap::create_sys_resource_pool() } else if (OB_FAIL(unit_mgr_.grant_pools( trans, new_ug_id_array, lib::Worker::CompatMode::MYSQL, pool_names, - OB_SYS_TENANT_ID, is_bootstrap, OB_INVALID_TENANT_ID/*source_tenant_id*/))) { + OB_SYS_TENANT_ID, is_bootstrap, OB_INVALID_TENANT_ID/*source_tenant_id*/, + check_data_version))) { LOG_WARN("grant_pools_to_tenant failed", K(pool_names), "tenant_id", static_cast(OB_SYS_TENANT_ID), K(ret)); } else { diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index 401e98d5f9..a3a6ee3997 100755 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -24430,7 +24430,8 @@ int ObDDLService::create_tenant_schema( compat_mode, pools, user_tenant_id, false/*is_bootstrap*/, - arg.source_tenant_id_))) { + arg.source_tenant_id_, + false/*check_data_version*/))) { LOG_WARN("grant_pools_to_tenant failed", KR(ret), K(arg), K(pools), K(user_tenant_id)); } LOG_INFO("[CREATE_TENANT] STEP 1.2. finish grant pools", KR(ret), K(user_tenant_id), @@ -26423,7 +26424,8 @@ int ObDDLService::modify_and_cal_resource_pool_diff( LOG_WARN("fail to grant pool", K(ret), K(diff_pools)); } else if (OB_FAIL(unit_mgr_->grant_pools( trans, new_ug_id_array, compat_mode, diff_pools, tenant_id, - false/*is_bootstrap*/, OB_INVALID_TENANT_ID/*source_tenant_id*/))) { + false/*is_bootstrap*/, OB_INVALID_TENANT_ID/*source_tenant_id*/, + true/*check_data_version*/))) { LOG_WARN("fail to grant pools", K(ret)); } } else if (new_pool_name_list.count() + 1 == old_pool_name_list.count()) { diff --git a/src/rootserver/ob_unit_manager.cpp b/src/rootserver/ob_unit_manager.cpp index b5af63320f..a0d0873250 100644 --- a/src/rootserver/ob_unit_manager.cpp +++ b/src/rootserver/ob_unit_manager.cpp @@ -850,7 +850,7 @@ int ObUnitManager::clone_resource_pool( LOG_WARN("fail to construct resource pool to clone", KR(ret), K(source_tenant_id), K(resource_pool)); } else if (OB_FAIL(inner_get_all_unit_infos_by_tenant_(source_tenant_id, source_units))) { LOG_WARN("fail to get units by source tenant", KR(ret), K(source_tenant_id)); - } else if (OB_FAIL(inner_create_resource_pool( + } else if (OB_FAIL(inner_create_resource_pool_( resource_pool, unit_config_name, if_not_exist, @@ -871,13 +871,13 @@ int ObUnitManager::create_resource_pool( int ret = OB_SUCCESS; SpinWLockGuard guard(lock_); common::ObArray source_units; // not used - if (OB_FAIL(inner_create_resource_pool(resource_pool, config_name, if_not_exist, OB_INVALID_TENANT_ID/*source_tenant_id*/, source_units))) { + if (OB_FAIL(inner_create_resource_pool_(resource_pool, config_name, if_not_exist, OB_INVALID_TENANT_ID/*source_tenant_id*/, source_units))) { LOG_WARN("fail to inner create resource pool", KR(ret), K(resource_pool), K(config_name), K(if_not_exist)); } return ret; } -int ObUnitManager::inner_create_resource_pool( +int ObUnitManager::inner_create_resource_pool_( share::ObResourcePool &resource_pool, const ObUnitConfigName &config_name, const bool if_not_exist, @@ -3982,8 +3982,9 @@ int ObUnitManager::grant_pools(common::ObMySQLTransaction &trans, const ObIArray &pool_names, const uint64_t tenant_id, const bool is_bootstrap, - const uint64_t source_tenant_id - /*arg "const bool skip_offline_server" is no longer supported*/) + const uint64_t source_tenant_id, + /*arg "const bool skip_offline_server" is no longer supported*/ + const bool check_data_version) { int ret = OB_SUCCESS; SpinWLockGuard guard(lock_); @@ -4033,8 +4034,9 @@ int ObUnitManager::grant_pools(common::ObMySQLTransaction &trans, is_bootstrap, grant, tenant_id, legal_unit_num, new_unit_group_id_array))) { LOG_WARN("fail to generate new unit group id", KR(ret), K(tenant_id), K(legal_unit_num)); } else if (OB_FAIL(do_grant_pools_( - trans, new_unit_group_id_array, compat_mode, - pool_names, tenant_id, is_bootstrap, source_tenant_id))) { + trans, new_unit_group_id_array, compat_mode, + pool_names, tenant_id, is_bootstrap, + source_tenant_id, check_data_version))) { LOG_WARN("do grant pools failed", KR(ret), K(grant), K(pool_names), K(tenant_id), K(compat_mode), K(is_bootstrap), K(source_tenant_id)); } @@ -5188,7 +5190,8 @@ int ObUnitManager::try_notify_tenant_server_unit_resource_( const lib::Worker::CompatMode compat_mode, const share::ObUnit &unit, const bool if_not_grant, - const bool skip_offline_server) + const bool skip_offline_server, + const bool check_data_version) { int ret = OB_SUCCESS; bool is_alive = false; @@ -5207,10 +5210,14 @@ int ObUnitManager::try_notify_tenant_server_unit_resource_( // STEP 1: Get and init notifying arg obrpc::TenantServerUnitConfig tenant_unit_server_config; if (!is_delete) { + const bool should_check_data_version = check_data_version && is_user_tenant(tenant_id); if (OB_FAIL(build_notify_create_unit_resource_rpc_arg_( tenant_id, unit, compat_mode, unit_config_id, if_not_grant, tenant_unit_server_config))) { LOG_WARN("fail to init tenant_unit_server_config", KR(ret), K(tenant_id), K(is_delete)); + } else if (should_check_data_version + && OB_FAIL(check_dest_data_version_is_loaded_(tenant_id, unit.server_))) { + LOG_WARN("fail to check dest data_version is loaded", KR(ret), K(tenant_id), "dst", unit.server_); } } else { if (OB_FAIL(tenant_unit_server_config.init_for_dropping(tenant_id, is_delete))) { @@ -5232,6 +5239,83 @@ int ObUnitManager::try_notify_tenant_server_unit_resource_( return ret; } +int ObUnitManager::check_dest_data_version_is_loaded_( + const uint64_t tenant_id, const ObAddr &addr) +{ + int ret = OB_SUCCESS; + ObTimeoutCtx ctx; + const int64_t DEFTAULT_TIMEOUT_TS = 5 * GCONF.rpc_timeout; + char ip_buf[OB_IP_STR_BUFF] = ""; + if (OB_UNLIKELY(!check_inner_stat())) { + ret = OB_INNER_STAT_ERROR; + LOG_WARN("check_inner_stat failed", KR(ret), K(inited_), K(loaded_)); + } else if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id + || !addr.is_valid() + || OB_ISNULL(proxy_))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", KR(ret), K(tenant_id), K(addr), KP_(proxy)); + } else if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, DEFTAULT_TIMEOUT_TS))) { + LOG_WARN("fail to set default timeout ctx", KR(ret)); + } else if (OB_UNLIKELY(!addr.ip_to_string(ip_buf, sizeof(ip_buf)))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to convert ip to string", KR(ret), K(tenant_id), K(addr)); + } else { + const int64_t start_timeout_ts = ObTimeUtility::current_time(); + const int64_t CHECK_INTERVAL_TS = 500 * 1000L; // 500ms + const int64_t SLEEP_TS = 100 * 1000L; // 100ms + ObSqlString sql; + if (OB_FAIL(sql.assign_fmt("SELECT IF(value = '0.0.0.0', 0, 1) AS loaded " + "FROM %s WHERE tenant_id = %lu AND name = 'compatible' " + "AND svr_ip = '%s' AND svr_port = %d", + OB_ALL_VIRTUAL_TENANT_PARAMETER_INFO_TNAME, + tenant_id, ip_buf, addr.get_port()))) { + LOG_WARN("fail to assign fmt", KR(ret), K(tenant_id), K(addr)); + } + while (OB_SUCC(ret)) { + if (OB_UNLIKELY(ctx.is_timeouted())) { + ret = OB_TIMEOUT; + LOG_WARN("check dest data version timeout", KR(ret), + K(start_timeout_ts), "abs_timeout", ctx.get_abs_timeout()); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + sqlclient::ObMySQLResult *result = NULL; + if (OB_FAIL(proxy_->read(res, OB_SYS_TENANT_ID, sql.ptr()))) { + LOG_WARN("fail to read by sql", KR(ret), K(sql)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result is null", KR(ret)); + } else if (OB_FAIL(result->next())) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + if (REACH_TIME_INTERVAL(CHECK_INTERVAL_TS)) { + LOG_WARN_RET(OB_EAGAIN, "check data_version is loaded, but result is empty, try later", + K(tenant_id), K(addr)); + } + } else { + LOG_WARN("fail to get next row", KR(ret)); + } + } else { + int64_t loaded = 0; + EXTRACT_INT_FIELD_MYSQL(*result, "loaded", loaded, int64_t); + if (OB_SUCC(ret)) { + if (1 == loaded) { + break; + } else if (REACH_TIME_INTERVAL(CHECK_INTERVAL_TS)) + LOG_WARN_RET(OB_EAGAIN, "check data_version is loaded, but it's not refreshed yet, try later", + K(tenant_id), K(addr)); + } + } + } // end SMART_VAR + + if (OB_SUCC(ret)) { + ob_usleep(SLEEP_TS); + } + } + } // end while + } + return ret; +} + int ObUnitManager::build_notify_create_unit_resource_rpc_arg_( const uint64_t tenant_id, const share::ObUnit &unit, @@ -5331,7 +5415,8 @@ int ObUnitManager::rollback_persistent_units_( if (OB_TMP_FAIL(try_notify_tenant_server_unit_resource_( pool.tenant_id_, is_delete, notify_proxy, pool.unit_config_id_, dummy_mode, unit, - false/*if_not_grant*/, false/*skip_offline_server*/))) { + false/*if_not_grant*/, false/*skip_offline_server*/, + false /*check_data_version*/))) { ret = OB_SUCC(ret) ? tmp_ret : ret; LOG_WARN("fail to try notify server unit resource", KR(ret), KR(tmp_ret), K(is_delete), K(pool), K(dummy_mode), K(unit)); @@ -5578,7 +5663,7 @@ int ObUnitManager::try_persist_unit_info_( } else if (OB_FAIL(try_notify_tenant_server_unit_resource_( pool.tenant_id_, is_delete, notify_proxy, pool.unit_config_id_, compat_mode, unit, false/*if not grant*/, - false/*skip offline server*/))) { + false/*skip offline server*/, true /*check_data_version*/))) { LOG_WARN("fail to try notify server unit resource", KR(ret), K(pool), K(is_delete), K(unit)); } else if (OB_FAIL(add_unit(client, unit))) { LOG_WARN("add_unit failed", KR(ret), K(unit), K(unit)); @@ -8497,7 +8582,8 @@ int ObUnitManager::construct_pool_units_to_grant_( const lib::Worker::CompatMode &compat_mode, ObNotifyTenantServerResourceProxy ¬ify_proxy, const uint64_t source_tenant_id, - ObIArray &pool_units) + ObIArray &pool_units, + const bool check_data_version) { int ret = OB_SUCCESS; pool_units.reset(); @@ -8531,7 +8617,8 @@ int ObUnitManager::construct_pool_units_to_grant_( } else if (OB_FAIL(try_notify_tenant_server_unit_resource_( tenant_id, false /*is_delete*/, notify_proxy, new_pool.unit_config_id_, compat_mode, *unit, - false/*if_not_grant*/, false/*skip_offline_server*/))) { + false/*if_not_grant*/, false/*skip_offline_server*/, + check_data_version))) { LOG_WARN("fail to try notify server unit resource", KR(ret), K(tenant_id), K(compat_mode), KPC(unit)); } else if (FALSE_IT(new_unit = *unit)) { @@ -8687,7 +8774,8 @@ int ObUnitManager::do_grant_pools_( const ObIArray &pool_names, const uint64_t tenant_id, const bool is_bootstrap, - const uint64_t source_tenant_id) + const uint64_t source_tenant_id, + const bool check_data_version) { int ret = OB_SUCCESS; if (!check_inner_stat()) { @@ -8739,7 +8827,8 @@ int ObUnitManager::do_grant_pools_( compat_mode, notify_proxy, source_tenant_id, - pool_units))) { + pool_units, + check_data_version))) { LOG_WARN("fail to construct pool units to grant", KR(ret), K(tenant_id), K(new_pool), K(zone_sorted_unit_array), K(new_ug_ids), K(compat_mode), K(source_tenant_id)); } else if (OB_FAIL(all_pool_units.push_back(pool_units))) { @@ -8829,7 +8918,8 @@ int ObUnitManager::do_revoke_pools_( } else if (OB_FAIL(try_notify_tenant_server_unit_resource_( tenant_id, true /*is_delete*/, notify_proxy, new_pool.unit_config_id_, dummy_mode, *unit, - false/*if_not_grant*/, false/*skip_offline_server*/))) { + false/*if_not_grant*/, false/*skip_offline_server*/, + false /*check_data_version*/))) { LOG_WARN("fail to try notify server unit resource", KR(ret)); } else if (FALSE_IT(new_unit = *unit)) { // shall never be here @@ -9417,7 +9507,8 @@ int ObUnitManager::do_migrate_unit_notify_resource_(const share::ObResourcePool if (OB_FAIL(try_notify_tenant_server_unit_resource_( pool.tenant_id_, false/*is_delete*/, notify_proxy, // is_delete is false when migrate unit pool.unit_config_id_, compat_mode, new_unit, false/*if not grant*/, - false/*skip offline server*/))) { + false/*skip offline server*/, + true /*check_data_version*/))) { LOG_WARN("fail to try notify server unit resource", K(ret)); } int tmp_ret = OB_SUCCESS; diff --git a/src/rootserver/ob_unit_manager.h b/src/rootserver/ob_unit_manager.h index 05e42acafa..ed646d170d 100644 --- a/src/rootserver/ob_unit_manager.h +++ b/src/rootserver/ob_unit_manager.h @@ -198,8 +198,9 @@ public: const lib::Worker::CompatMode compat_mode, const common::ObIArray &pool_names, const uint64_t tenant_id, - const bool is_bootstrap = false, - const uint64_t source_tenant_id = OB_INVALID_TENANT_ID); + const bool is_bootstrap, + const uint64_t source_tenant_id, + const bool check_data_version); virtual int revoke_pools( common::ObMySQLTransaction &trans, common::ObIArray &new_ug_id_array, @@ -739,7 +740,8 @@ private: const lib::Worker::CompatMode &compat_mode, ObNotifyTenantServerResourceProxy ¬ify_proxy, const uint64_t source_tenant_id, - ObIArray &pool_units); + ObIArray &pool_units, + const bool check_data_version); int construct_unit_group_id_for_unit_( const uint64_t source_tenant_id, @@ -764,7 +766,8 @@ private: const common::ObIArray &pool_names, const uint64_t tenant_id, const bool is_bootstrap, - const uint64_t source_tenant_id); + const uint64_t source_tenant_id, + const bool check_data_version); int do_revoke_pools_(common::ObMySQLTransaction &trans, const common::ObIArray &new_unit_group_id_array, @@ -852,7 +855,8 @@ private: const lib::Worker::CompatMode compat_mode, const share::ObUnit &unit, const bool if_not_grant, - const bool skip_offline_server); + const bool skip_offline_server, + const bool check_data_version); int build_notify_create_unit_resource_rpc_arg_( const uint64_t tenant_id, const share::ObUnit &unit, @@ -860,6 +864,8 @@ private: const uint64_t unit_config_id, const bool if_not_grant, obrpc::TenantServerUnitConfig &rpc_arg) const; + int check_dest_data_version_is_loaded_( + const uint64_t tenant_id, const ObAddr &addr); int do_notify_unit_resource_( const common::ObAddr server, const obrpc::TenantServerUnitConfig ¬ify_arg, @@ -960,7 +966,7 @@ private: int inner_create_unit_config_( const share::ObUnitConfig &unit_config, const bool if_not_exist); - int inner_create_resource_pool( + int inner_create_resource_pool_( share::ObResourcePool &resource_pool, const share::ObUnitConfigName &config_name, const bool if_not_exist,